mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 01:29:56 +00:00
fix(update): keep retained deleted-agent databases from blocking upgrades (#162056)
* fix(update): hold retained deleted-agent databases * fix(update): fence resumed retained database writes * fix(update): fence pending agent deletions * fix(update): distinguish pending agent deletion holds * fix(update): preserve partial deletion fences * fix(update): fence unavailable deletion history * test(doctor): include deletion journal in legacy fixture * test(cron): configure fallback runtime fixture
This commit is contained in:
Dallin Romney
GitHub
parent
310927d0f6
commit
d21cb744af
12 changed files with 938 additions and 60 deletions
|
|
@ -261,6 +261,12 @@ the container normally.
|
|||
|
||||
`openclaw doctor --fix` is the only owner for persistent file-to-SQLite migrations. It validates and claims each recognized source, writes and verifies canonical rows, records a migration receipt, then removes the retired source. Runtime code does not perform lazy imports or fallback reads.
|
||||
|
||||
Completed agent deletions that intentionally retained their database are held back from
|
||||
legacy state migrations. Doctor reports a nonblocking note naming the retained path and
|
||||
manual restore-or-move guidance, while a configured or registered surviving owner of the
|
||||
same physical database still migrates normally. If deletion history is missing or unreadable,
|
||||
migration fails closed instead of guessing that an unowned database is active.
|
||||
|
||||
Doctor reports interrupted auth-profile archive recovery even when no new migration remains or you decline another migration. If recovery cannot finish, its warning includes the failure cause and leaves the pending source for recovery; do not delete it to silence the warning.
|
||||
|
||||
`doctor --fix` also repairs an inconsistent completed auth migration only when its old receipt has no credential fingerprints, none of the migrated credentials remain in the current canonical store, and the preserved archive still matches the recorded source hash. Doctor reimports through the normal verified migration flow. Completed receipts with fingerprints, surviving migrated credentials, or no archive remain untouched, so removing credentials after a verified migration does not restore them from backup.
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { resolveConfiguredAgentDatabaseTargets } from "../config/sessions/target
|
|||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { requireNodeSqlite } from "../infra/node-sqlite.js";
|
||||
import { OPENCLAW_AGENT_SCHEMA_VERSION } from "../state/openclaw-agent-db-contract.js";
|
||||
import { ensureAgentDeletionJournalSchema } from "../state/openclaw-state-db-schema-additive.js";
|
||||
import { beginDoctorMaintenance } from "./doctor-maintenance.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
|
@ -36,6 +37,7 @@ function createLegacyRegistryFixture() {
|
|||
PRIMARY KEY (agent_id, path)
|
||||
);
|
||||
`);
|
||||
ensureAgentDeletionJournalSchema(database);
|
||||
database.close();
|
||||
const config: OpenClawConfig = {
|
||||
agents: { ownership: "explicit", entries: { main: {} } },
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import {
|
|||
isCliProviderMock,
|
||||
loadRunCronIsolatedAgentTurn,
|
||||
mockRunCronFallbackPassthrough,
|
||||
resolveCliRuntimeExecutionProviderMock,
|
||||
resolveConfiguredModelRefMock,
|
||||
resolveSessionAuthSelectionMock,
|
||||
runCliAgentMock,
|
||||
|
|
@ -56,6 +57,9 @@ function getCliAgentParams(): {
|
|||
}
|
||||
|
||||
function setupClaudeCliBackend(): void {
|
||||
resolveCliRuntimeExecutionProviderMock.mockImplementation(({ modelId }: { modelId: string }) =>
|
||||
modelId === "claude-sonnet-4-6" ? "claude-cli" : undefined,
|
||||
);
|
||||
cliBackendsTesting.setDepsForTest({
|
||||
resolveRuntimeCliBackends: () => [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1207,6 +1207,7 @@ export async function autoMigrateLegacyState(params: {
|
|||
? await migrateHistoricalTranscriptDirectives(agentMigrationOptions)
|
||||
: { changes: [], warnings: [] };
|
||||
if (transcriptDirectives.warnings.length > 0 || mediaPersistence.warnings.length > 0) {
|
||||
const notices = mergeNotices([stateDirResult, transcriptDirectives, mediaPersistence]);
|
||||
return {
|
||||
migrated:
|
||||
stateDirResult.migrated ||
|
||||
|
|
@ -1226,7 +1227,7 @@ export async function autoMigrateLegacyState(params: {
|
|||
...transcriptDirectives.warnings,
|
||||
...mediaPersistence.warnings,
|
||||
],
|
||||
...(stateDirResult.notices?.length ? { notices: stateDirResult.notices } : {}),
|
||||
...(notices.length > 0 ? { notices } : {}),
|
||||
};
|
||||
}
|
||||
const profileWorkspace =
|
||||
|
|
@ -1404,6 +1405,8 @@ export async function autoMigrateLegacyState(params: {
|
|||
const notices = mergeNotices([
|
||||
stateDirResult,
|
||||
profileWorkspace,
|
||||
transcriptDirectives,
|
||||
mediaPersistence,
|
||||
detected,
|
||||
...alwaysRunSources,
|
||||
deviceAuth,
|
||||
|
|
@ -1442,6 +1445,8 @@ export async function autoMigrateLegacyState(params: {
|
|||
const notices = mergeNotices([
|
||||
stateDirResult,
|
||||
profileWorkspace,
|
||||
transcriptDirectives,
|
||||
mediaPersistence,
|
||||
detected,
|
||||
...migrations.sharedNoticeSources,
|
||||
deviceAuth,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,10 @@ import { afterEach, describe, expect, it } from "vitest";
|
|||
import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js";
|
||||
import { resolveSessionStorePathCore } from "../config/sessions/paths.js";
|
||||
import { resolveSqliteTargetFromSessionStorePath } from "../config/sessions/session-sqlite-target.js";
|
||||
import {
|
||||
beginAgentDeletionJournal,
|
||||
completeAgentDeletionJournal,
|
||||
} from "../state/agent-deletion-journal.js";
|
||||
import {
|
||||
registerOpenClawAgentDatabase,
|
||||
unregisterOpenClawAgentDatabase,
|
||||
|
|
@ -24,11 +28,16 @@ import { migrateLegacyMediaPersistence } from "./state-migrations.media-persiste
|
|||
|
||||
const tempDirs: string[] = [];
|
||||
const PREVIOUS_VERSION = 16;
|
||||
const RETAINED_NOTICE = (databasePath: string, agentId: string) =>
|
||||
`Held retained database ${databasePath} for deleted agent ${agentId}; restore that agent from backup or move this database out of the active state directory, then rerun openclaw doctor --fix.`;
|
||||
const PENDING_NOTICE = (databasePath: string, agentId: string) =>
|
||||
`Held database ${databasePath} while deletion of agent ${agentId} is pending; finish or retry that agent deletion, then rerun openclaw doctor --fix.`;
|
||||
|
||||
function createLegacyAgentDatabase(params: {
|
||||
agentId?: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
path?: string;
|
||||
version?: number;
|
||||
}): string {
|
||||
const agentId = params.agentId ?? "main";
|
||||
const opened = openOpenClawAgentDatabase({
|
||||
|
|
@ -41,10 +50,14 @@ function createLegacyAgentDatabase(params: {
|
|||
const { DatabaseSync } = requireNodeSqlite();
|
||||
const database = new DatabaseSync(databasePath);
|
||||
try {
|
||||
database.exec(`DROP TABLE session_participants; PRAGMA user_version = ${PREVIOUS_VERSION};`);
|
||||
const version = params.version ?? PREVIOUS_VERSION;
|
||||
if (version === PREVIOUS_VERSION) {
|
||||
database.exec("DROP TABLE session_participants");
|
||||
}
|
||||
database.exec(`PRAGMA user_version = ${version};`);
|
||||
database
|
||||
.prepare("UPDATE schema_meta SET schema_version = ? WHERE meta_key = 'primary'")
|
||||
.run(PREVIOUS_VERSION);
|
||||
.run(version);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
|
|
@ -68,6 +81,379 @@ afterEach(() => {
|
|||
});
|
||||
|
||||
describe("media persistence migration targets", () => {
|
||||
it("skips a retained database after its agent deletion completes", async () => {
|
||||
const stateDir = fs.realpathSync.native(
|
||||
makeTempDir(tempDirs, "media-persistence-retained-deletion-"),
|
||||
);
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agentId = "retired";
|
||||
const operationId = "retained-deletion";
|
||||
const databasePath = createLegacyAgentDatabase({ agentId, env });
|
||||
const agentDir = path.dirname(databasePath);
|
||||
beginAgentDeletionJournal(
|
||||
{
|
||||
agentId,
|
||||
operationId,
|
||||
agentDir,
|
||||
workspaceDir: path.join(stateDir, "workspaces", agentId),
|
||||
sessionsDir: path.join(agentDir, "sessions"),
|
||||
databasePaths: [databasePath],
|
||||
deleteFiles: false,
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
expect(completeAgentDeletionJournal(agentId, operationId, { env })).toBe(true);
|
||||
unregisterOpenClawAgentDatabase({ agentId, env, path: databasePath });
|
||||
|
||||
const result = await migrateLegacyMediaPersistence({ env });
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.notices).toEqual([RETAINED_NOTICE(databasePath, agentId)]);
|
||||
expect(readUserVersion(databasePath)).toBe(PREVIOUS_VERSION);
|
||||
});
|
||||
|
||||
it("reports pending deletion separately from retained-store recovery", async () => {
|
||||
const stateDir = fs.realpathSync.native(
|
||||
makeTempDir(tempDirs, "media-persistence-pending-deletion-"),
|
||||
);
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agentId = "deleting";
|
||||
const databasePath = createLegacyAgentDatabase({ agentId, env });
|
||||
beginAgentDeletionJournal(
|
||||
{
|
||||
agentId,
|
||||
operationId: "pending-deletion",
|
||||
agentDir: path.dirname(databasePath),
|
||||
workspaceDir: path.join(stateDir, "workspaces", agentId),
|
||||
sessionsDir: path.join(stateDir, "agents", agentId, "sessions"),
|
||||
databasePaths: [databasePath],
|
||||
deleteFiles: true,
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
|
||||
const result = await migrateLegacyMediaPersistence({ env });
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.notices).toEqual([PENDING_NOTICE(databasePath, agentId)]);
|
||||
expect(readUserVersion(databasePath)).toBe(PREVIOUS_VERSION);
|
||||
});
|
||||
|
||||
it("migrates a retained path claimed by a surviving configured owner", async () => {
|
||||
const stateDir = fs.realpathSync.native(
|
||||
makeTempDir(tempDirs, "media-persistence-retained-surviving-owner-"),
|
||||
);
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const databasePath = path.join(stateDir, "agents", "retired", "agent", "openclaw-agent.sqlite");
|
||||
createLegacyAgentDatabase({ agentId: "survivor", env, path: databasePath });
|
||||
const operationId = "retained-surviving-owner";
|
||||
beginAgentDeletionJournal(
|
||||
{
|
||||
agentId: "retired",
|
||||
operationId,
|
||||
agentDir: path.dirname(databasePath),
|
||||
workspaceDir: path.join(stateDir, "workspaces", "retired"),
|
||||
sessionsDir: path.join(stateDir, "agents", "retired", "sessions"),
|
||||
databasePaths: [databasePath],
|
||||
deleteFiles: false,
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
expect(completeAgentDeletionJournal("retired", operationId, { env })).toBe(true);
|
||||
|
||||
const result = await migrateLegacyMediaPersistence({
|
||||
configuredAgentDatabaseTargets: [{ agentId: "survivor", path: databasePath }],
|
||||
env,
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.notices).toBeUndefined();
|
||||
expect(readUserVersion(databasePath)).toBe(OPENCLAW_AGENT_SCHEMA_VERSION);
|
||||
expect(
|
||||
listOpenClawRegisteredAgentDatabases({
|
||||
env,
|
||||
includeIncompatibleSchemaVersions: true,
|
||||
}),
|
||||
).toEqual([expect.objectContaining({ agentId: "survivor", path: databasePath })]);
|
||||
});
|
||||
|
||||
it("migrates a surviving registered owner through a hardlink to a retained path", async () => {
|
||||
const stateDir = fs.realpathSync.native(
|
||||
makeTempDir(tempDirs, "media-persistence-retained-hardlink-owner-"),
|
||||
);
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const retainedPath = path.join(stateDir, "agents", "retired", "agent", "openclaw-agent.sqlite");
|
||||
createLegacyAgentDatabase({ agentId: "survivor", env, path: retainedPath });
|
||||
unregisterOpenClawAgentDatabase({ agentId: "survivor", env, path: retainedPath });
|
||||
const survivingPath = path.join(stateDir, "stores", "survivor.sqlite");
|
||||
fs.mkdirSync(path.dirname(survivingPath), { recursive: true });
|
||||
fs.linkSync(retainedPath, survivingPath);
|
||||
registerOpenClawAgentDatabase({
|
||||
agentId: "survivor",
|
||||
env,
|
||||
path: survivingPath,
|
||||
schemaVersion: PREVIOUS_VERSION,
|
||||
});
|
||||
const operationId = "retained-hardlink-owner";
|
||||
beginAgentDeletionJournal(
|
||||
{
|
||||
agentId: "retired",
|
||||
operationId,
|
||||
agentDir: path.dirname(retainedPath),
|
||||
workspaceDir: path.join(stateDir, "workspaces", "retired"),
|
||||
sessionsDir: path.join(stateDir, "agents", "retired", "sessions"),
|
||||
databasePaths: [retainedPath],
|
||||
deleteFiles: false,
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
expect(completeAgentDeletionJournal("retired", operationId, { env })).toBe(true);
|
||||
|
||||
const result = await migrateLegacyMediaPersistence({ env });
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.notices).toBeUndefined();
|
||||
expect(readUserVersion(retainedPath)).toBe(OPENCLAW_AGENT_SCHEMA_VERSION);
|
||||
expect(readUserVersion(survivingPath)).toBe(OPENCLAW_AGENT_SCHEMA_VERSION);
|
||||
});
|
||||
|
||||
it("holds an unowned database when deletion journal history is unavailable", async () => {
|
||||
const stateDir = fs.realpathSync.native(
|
||||
makeTempDir(tempDirs, "media-persistence-missing-deletion-history-"),
|
||||
);
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const databasePath = createLegacyAgentDatabase({ agentId: "unknown", env });
|
||||
const state = openOpenClawStateDatabase({ env });
|
||||
state.db.exec("DROP TABLE agent_deletion_journal");
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
|
||||
const result = await migrateLegacyMediaPersistence({ env });
|
||||
|
||||
expect(result.warnings).toEqual([
|
||||
expect.stringContaining("Agent database maintenance deferred"),
|
||||
]);
|
||||
expect(readUserVersion(databasePath)).toBe(PREVIOUS_VERSION);
|
||||
});
|
||||
|
||||
it("fences a configured database when deletion history becomes unavailable before final I/O", async () => {
|
||||
const stateDir = fs.realpathSync.native(
|
||||
makeTempDir(tempDirs, "media-persistence-unavailable-history-race-"),
|
||||
);
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agentId = "deleting-configured";
|
||||
const databasePath = createLegacyAgentDatabase({ agentId, env });
|
||||
const bytesBefore = fs.readFileSync(databasePath);
|
||||
|
||||
const result = await migrateLegacyMediaPersistence({
|
||||
configuredAgentDatabaseTargets: [{ agentId, path: databasePath }],
|
||||
env,
|
||||
hooks: {
|
||||
beforeDatabaseWrite: () => {
|
||||
const state = openOpenClawStateDatabase({ env });
|
||||
state.db.exec("DROP TABLE agent_deletion_journal");
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringContaining("deletion journal history is unavailable"),
|
||||
expect.stringContaining(`Skipped agent database ${databasePath}`),
|
||||
]),
|
||||
);
|
||||
expect(fs.readFileSync(databasePath)).toEqual(bytesBefore);
|
||||
expect(readUserVersion(databasePath)).toBe(PREVIOUS_VERSION);
|
||||
});
|
||||
|
||||
it("holds an unowned database when retained deletion history is malformed", async () => {
|
||||
const stateDir = fs.realpathSync.native(
|
||||
makeTempDir(tempDirs, "media-persistence-malformed-deletion-history-"),
|
||||
);
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agentId = "unknown";
|
||||
const operationId = "malformed-deletion-history";
|
||||
const databasePath = createLegacyAgentDatabase({ agentId, env });
|
||||
beginAgentDeletionJournal(
|
||||
{
|
||||
agentId,
|
||||
operationId,
|
||||
agentDir: path.dirname(databasePath),
|
||||
workspaceDir: path.join(stateDir, "workspaces", agentId),
|
||||
sessionsDir: path.join(stateDir, "agents", agentId, "sessions"),
|
||||
databasePaths: [databasePath],
|
||||
deleteFiles: false,
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
expect(completeAgentDeletionJournal(agentId, operationId, { env })).toBe(true);
|
||||
const state = openOpenClawStateDatabase({ env });
|
||||
state.db
|
||||
.prepare("UPDATE agent_deletion_journal SET database_paths_json = ? WHERE agent_id = ?")
|
||||
.run("{}", agentId);
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
|
||||
const result = await migrateLegacyMediaPersistence({ env });
|
||||
|
||||
expect(result.warnings).toContain(
|
||||
`Could not read database paths for deleted agent ${agentId}; its known database identity remains held.`,
|
||||
);
|
||||
expect(result.notices).toEqual([RETAINED_NOTICE(databasePath, agentId)]);
|
||||
expect(readUserVersion(databasePath)).toBe(PREVIOUS_VERSION);
|
||||
expect(
|
||||
listOpenClawRegisteredAgentDatabases({
|
||||
env,
|
||||
includeIncompatibleSchemaVersions: true,
|
||||
}),
|
||||
).toEqual([expect.objectContaining({ agentId, path: databasePath })]);
|
||||
});
|
||||
|
||||
it("migrates a configured active database when deletion history is malformed", async () => {
|
||||
const stateDir = fs.realpathSync.native(
|
||||
makeTempDir(tempDirs, "media-persistence-malformed-active-history-"),
|
||||
);
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const databasePath = createLegacyAgentDatabase({ agentId: "active", env });
|
||||
beginAgentDeletionJournal(
|
||||
{
|
||||
agentId: "retired",
|
||||
operationId: "malformed-active-history",
|
||||
agentDir: path.join(stateDir, "agents", "retired", "agent"),
|
||||
workspaceDir: path.join(stateDir, "workspaces", "retired"),
|
||||
sessionsDir: path.join(stateDir, "agents", "retired", "sessions"),
|
||||
databasePaths: [path.join(stateDir, "agents", "retired", "agent.sqlite")],
|
||||
deleteFiles: false,
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
expect(completeAgentDeletionJournal("retired", "malformed-active-history", { env })).toBe(true);
|
||||
const state = openOpenClawStateDatabase({ env });
|
||||
state.db
|
||||
.prepare("UPDATE agent_deletion_journal SET database_paths_json = ? WHERE agent_id = ?")
|
||||
.run("{}", "retired");
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
|
||||
const result = await migrateLegacyMediaPersistence({
|
||||
configuredAgentDatabaseTargets: [{ agentId: "active", path: databasePath }],
|
||||
env,
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringContaining("Could not read database paths for deleted agent retired"),
|
||||
]),
|
||||
);
|
||||
expect(readUserVersion(databasePath)).toBe(OPENCLAW_AGENT_SCHEMA_VERSION);
|
||||
});
|
||||
|
||||
it("fences a configured pending deletion with malformed paths before final I/O", async () => {
|
||||
const stateDir = fs.realpathSync.native(
|
||||
makeTempDir(tempDirs, "media-persistence-malformed-pending-race-"),
|
||||
);
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agentId = "deleting-configured";
|
||||
const operationId = "malformed-pending-race";
|
||||
const databasePath = createLegacyAgentDatabase({ agentId, env });
|
||||
const bytesBefore = fs.readFileSync(databasePath);
|
||||
|
||||
const result = await migrateLegacyMediaPersistence({
|
||||
configuredAgentDatabaseTargets: [{ agentId, path: databasePath }],
|
||||
env,
|
||||
hooks: {
|
||||
beforeDatabaseWrite: () => {
|
||||
beginAgentDeletionJournal(
|
||||
{
|
||||
agentId,
|
||||
operationId,
|
||||
agentDir: path.dirname(databasePath),
|
||||
workspaceDir: path.join(stateDir, "workspaces", agentId),
|
||||
sessionsDir: path.join(stateDir, "agents", agentId, "sessions"),
|
||||
databasePaths: [databasePath],
|
||||
deleteFiles: true,
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
openOpenClawStateDatabase({ env })
|
||||
.db.prepare(
|
||||
"UPDATE agent_deletion_journal SET database_paths_json = ? WHERE agent_id = ?",
|
||||
)
|
||||
.run("{}", agentId);
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.warnings).toContain(
|
||||
`Could not read database paths for deleted agent ${agentId}; its known database identity remains held.`,
|
||||
);
|
||||
expect(result.notices).toEqual([PENDING_NOTICE(databasePath, agentId)]);
|
||||
expect(fs.readFileSync(databasePath)).toEqual(bytesBefore);
|
||||
expect(readUserVersion(databasePath)).toBe(PREVIOUS_VERSION);
|
||||
});
|
||||
|
||||
it.each([PREVIOUS_VERSION, 18])(
|
||||
"rechecks deletion ownership before version %i database writes",
|
||||
async (version) => {
|
||||
const stateDir = fs.realpathSync.native(
|
||||
makeTempDir(tempDirs, "media-persistence-deletion-race-"),
|
||||
);
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agentId = "retired-during-migration";
|
||||
const operationId = "retired-during-migration";
|
||||
const databasePath = createLegacyAgentDatabase({ agentId, env, version });
|
||||
const bytesBefore = fs.readFileSync(databasePath);
|
||||
let deletionCompleted = false;
|
||||
|
||||
const result = await migrateLegacyMediaPersistence({
|
||||
env,
|
||||
hooks: {
|
||||
beforeDatabaseWrite: () => {
|
||||
if (deletionCompleted) {
|
||||
return;
|
||||
}
|
||||
beginAgentDeletionJournal(
|
||||
{
|
||||
agentId,
|
||||
operationId,
|
||||
agentDir: path.dirname(databasePath),
|
||||
workspaceDir: path.join(stateDir, "workspaces", agentId),
|
||||
sessionsDir: path.join(stateDir, "agents", agentId, "sessions"),
|
||||
databasePaths: [databasePath],
|
||||
deleteFiles: false,
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
expect(completeAgentDeletionJournal(agentId, operationId, { env })).toBe(true);
|
||||
deletionCompleted = true;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.notices).toEqual([RETAINED_NOTICE(databasePath, agentId)]);
|
||||
expect(fs.readFileSync(databasePath)).toEqual(bytesBefore);
|
||||
expect(readUserVersion(databasePath)).toBe(version);
|
||||
expect(
|
||||
listOpenClawRegisteredAgentDatabases({
|
||||
env,
|
||||
includeIncompatibleSchemaVersions: true,
|
||||
}),
|
||||
).toEqual([expect.objectContaining({ agentId, path: databasePath })]);
|
||||
},
|
||||
);
|
||||
|
||||
it("upgrades an active version 18 database through the same write boundary", async () => {
|
||||
const stateDir = fs.realpathSync.native(makeTempDir(tempDirs, "media-persistence-active-v18-"));
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const databasePath = createLegacyAgentDatabase({ agentId: "active", env, version: 18 });
|
||||
|
||||
const result = await migrateLegacyMediaPersistence({ env });
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.notices).toBeUndefined();
|
||||
expect(readUserVersion(databasePath)).toBe(OPENCLAW_AGENT_SCHEMA_VERSION);
|
||||
});
|
||||
|
||||
it("migrates and registers an unregistered default-layout agent database", async () => {
|
||||
const stateDir = fs.realpathSync.native(makeTempDir(tempDirs, "media-persistence-disk-scan-"));
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
|
|
|
|||
|
|
@ -1,9 +1,17 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { resolveAgentSessionDirsFromAgentsDirSync } from "../agents/session-dirs.js";
|
||||
import { formatCliCommand } from "../cli/command-format.js";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import { isSessionArchiveArtifactName } from "../config/sessions/artifacts.js";
|
||||
import { normalizeAgentId } from "../routing/session-key.js";
|
||||
import {
|
||||
readRetainedAgentDeletions,
|
||||
retainedAgentDeletionHistoryAbsent,
|
||||
retainedAgentDeletionHistoryUnavailable,
|
||||
retainedAgentDeletionReadWarning,
|
||||
type RetainedAgentDeletionDisposition,
|
||||
} from "../state/agent-deletion-journal.read.js";
|
||||
import {
|
||||
createOpenClawAgentDatabasePathMatcher,
|
||||
isPersistentOpenClawAgentDatabasePath,
|
||||
|
|
@ -22,6 +30,119 @@ type AgentDatabaseMigrationTarget = {
|
|||
|
||||
type CandidateTarget = Omit<AgentDatabaseMigrationTarget, "realPath">;
|
||||
|
||||
type RetainedAgentDatabaseHold = {
|
||||
kind: "notice" | "silent" | "warning";
|
||||
message: string;
|
||||
};
|
||||
|
||||
export class RetainedAgentDatabaseHoldError extends Error {
|
||||
constructor(readonly hold: RetainedAgentDatabaseHold) {
|
||||
super(hold.message);
|
||||
this.name = "RetainedAgentDatabaseHoldError";
|
||||
}
|
||||
|
||||
record(notices: string[], warnings: string[]): void {
|
||||
if (this.hold.kind === "silent") {
|
||||
return;
|
||||
}
|
||||
const messages = this.hold.kind === "notice" ? notices : warnings;
|
||||
if (!messages.includes(this.hold.message)) {
|
||||
messages.push(this.hold.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function classifyRetainedAgentDatabaseHold(params: {
|
||||
candidate: { agentId: string; path: string };
|
||||
configuredAgentDatabaseTargets: readonly { agentId: string; path: string }[];
|
||||
registeredAgentDatabases: readonly { agentId: string; path: string }[];
|
||||
retainedDeletions: RetainedAgentDeletionDisposition;
|
||||
env: NodeJS.ProcessEnv;
|
||||
}): RetainedAgentDatabaseHold | undefined {
|
||||
const pathMatcher = createOpenClawAgentDatabasePathMatcher();
|
||||
const sameDatabasePath = (left: string, right: string) => {
|
||||
try {
|
||||
return pathMatcher(left, right);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
const hasConfiguredOwner = (deletedAgentIds: ReadonlySet<string>) =>
|
||||
params.configuredAgentDatabaseTargets.some(
|
||||
(owner) =>
|
||||
!deletedAgentIds.has(normalizeAgentId(owner.agentId)) &&
|
||||
sameDatabasePath(owner.path, params.candidate.path),
|
||||
);
|
||||
if (retainedAgentDeletionHistoryUnavailable(params.retainedDeletions)) {
|
||||
return {
|
||||
kind: "warning",
|
||||
message: `Skipped agent database ${params.candidate.path}; deletion journal history is unavailable.`,
|
||||
};
|
||||
}
|
||||
if (retainedAgentDeletionHistoryAbsent(params.retainedDeletions)) {
|
||||
return undefined;
|
||||
}
|
||||
const deletedAgentIds = new Set(
|
||||
params.retainedDeletions.map((entry) => normalizeAgentId(entry.agentId)),
|
||||
);
|
||||
const deletion = params.retainedDeletions.find(
|
||||
(entry) =>
|
||||
normalizeAgentId(entry.agentId) === normalizeAgentId(params.candidate.agentId) ||
|
||||
entry.databasePaths.some((databasePath) =>
|
||||
sameDatabasePath(databasePath, params.candidate.path),
|
||||
),
|
||||
);
|
||||
if (!deletion) {
|
||||
return undefined;
|
||||
}
|
||||
const hasSurvivingRegisteredOwner = params.registeredAgentDatabases.some(
|
||||
(owner) =>
|
||||
!deletedAgentIds.has(normalizeAgentId(owner.agentId)) &&
|
||||
sameDatabasePath(owner.path, params.candidate.path),
|
||||
);
|
||||
if (hasConfiguredOwner(deletedAgentIds) || hasSurvivingRegisteredOwner) {
|
||||
return normalizeAgentId(params.candidate.agentId) === normalizeAgentId(deletion.agentId)
|
||||
? { kind: "silent", message: "" }
|
||||
: undefined;
|
||||
}
|
||||
if (deletion.state === "pending") {
|
||||
return {
|
||||
kind: "notice",
|
||||
message: `Held database ${params.candidate.path} while deletion of agent ${deletion.agentId} is pending; finish or retry that agent deletion, then rerun ${formatCliCommand("openclaw doctor --fix", params.env)}.`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
kind: "notice",
|
||||
message: `Held retained database ${params.candidate.path} for deleted agent ${deletion.agentId}; restore that agent from backup or move this database out of the active state directory, then rerun ${formatCliCommand("openclaw doctor --fix", params.env)}.`,
|
||||
};
|
||||
}
|
||||
|
||||
export function assertRetainedAgentDatabaseWritable(params: {
|
||||
candidate: { agentId: string; path: string };
|
||||
configuredAgentDatabaseTargets: readonly { agentId: string; path: string }[];
|
||||
env: NodeJS.ProcessEnv;
|
||||
warnings: string[];
|
||||
}): void {
|
||||
const retainedDeletions = readRetainedAgentDeletions(params.env);
|
||||
const readWarning = retainedAgentDeletionReadWarning(retainedDeletions);
|
||||
if (readWarning && !params.warnings.includes(readWarning)) {
|
||||
params.warnings.push(readWarning);
|
||||
}
|
||||
const hold = classifyRetainedAgentDatabaseHold({
|
||||
candidate: params.candidate,
|
||||
configuredAgentDatabaseTargets: params.configuredAgentDatabaseTargets,
|
||||
registeredAgentDatabases: listOpenClawRegisteredAgentDatabases({
|
||||
env: params.env,
|
||||
includeIncompatibleSchemaVersions: true,
|
||||
}),
|
||||
retainedDeletions,
|
||||
env: params.env,
|
||||
});
|
||||
if (hold) {
|
||||
throw new RetainedAgentDatabaseHoldError(hold);
|
||||
}
|
||||
}
|
||||
|
||||
function listDefaultAgentDatabaseTargets(
|
||||
env: NodeJS.ProcessEnv,
|
||||
failure: (pathname: string, reason: string) => void,
|
||||
|
|
@ -46,9 +167,11 @@ function listDefaultAgentDatabaseTargets(
|
|||
export function discoverAgentDatabaseMigrationTargets(params: {
|
||||
configuredAgentDatabaseTargets: readonly { agentId: string; path: string }[];
|
||||
registeredAgentDatabases: readonly { agentId: string; path: string }[];
|
||||
retainedDeletions?: RetainedAgentDeletionDisposition;
|
||||
env: NodeJS.ProcessEnv;
|
||||
}) {
|
||||
const warnings: string[] = [];
|
||||
const notices: string[] = [];
|
||||
const failures: Array<{ path: string; reason: string }> = [];
|
||||
const registryRemovals: Array<{ agentId: string; path: string; change?: string }> = [];
|
||||
const failure = (pathname: string, reason: string) => {
|
||||
|
|
@ -88,12 +211,37 @@ export function discoverAgentDatabaseMigrationTargets(params: {
|
|||
}
|
||||
}
|
||||
const configuredPathMatcher = createOpenClawAgentDatabasePathMatcher();
|
||||
const retainedDeletions = params.retainedDeletions ?? readRetainedAgentDeletions(params.env);
|
||||
const retainedDeletionWarning = retainedAgentDeletionReadWarning(retainedDeletions);
|
||||
if (retainedDeletionWarning) {
|
||||
warnings.push(retainedDeletionWarning);
|
||||
}
|
||||
const targets: AgentDatabaseMigrationTarget[] = [];
|
||||
const seenRealPaths = new Set<string>();
|
||||
for (const candidate of candidates) {
|
||||
// Preserve the original locator: lexical normalization of `link/../file`
|
||||
// can select a different file than filesystem symlink traversal does.
|
||||
const pathname = candidate.path;
|
||||
const retainedHold = classifyRetainedAgentDatabaseHold({
|
||||
candidate,
|
||||
configuredAgentDatabaseTargets: params.configuredAgentDatabaseTargets,
|
||||
registeredAgentDatabases: params.registeredAgentDatabases,
|
||||
retainedDeletions,
|
||||
env: params.env,
|
||||
});
|
||||
if (retainedHold) {
|
||||
if (retainedHold.kind === "silent") {
|
||||
continue;
|
||||
}
|
||||
const messages = retainedHold.kind === "notice" ? notices : warnings;
|
||||
if (!messages.includes(retainedHold.message)) {
|
||||
messages.push(retainedHold.message);
|
||||
}
|
||||
if (retainedHold.kind === "warning") {
|
||||
failures.push({ path: pathname, reason: retainedHold.message });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!isPersistentOpenClawAgentDatabasePath(pathname, params.env)) {
|
||||
discard(
|
||||
candidate,
|
||||
|
|
@ -164,7 +312,7 @@ export function discoverAgentDatabaseMigrationTargets(params: {
|
|||
seenRealPaths.add(realPath);
|
||||
targets.push({ ...candidate, path: pathname, realPath });
|
||||
}
|
||||
return { targets, registryRemovals, warnings, failures };
|
||||
return { targets, registryRemovals, warnings, notices, failures };
|
||||
}
|
||||
|
||||
/** Migration alone owns cleanup of stale registry entries discovered above. */
|
||||
|
|
@ -172,6 +320,8 @@ export function resolveAgentDatabaseMigrationTargets(params: {
|
|||
changes: string[];
|
||||
configuredAgentDatabaseTargets: readonly { agentId: string; path: string }[];
|
||||
env: NodeJS.ProcessEnv;
|
||||
notices?: string[];
|
||||
retainedDeletions?: RetainedAgentDeletionDisposition;
|
||||
warnings: string[];
|
||||
}): AgentDatabaseMigrationTarget[] {
|
||||
let registeredAgentDatabases: ReturnType<typeof listOpenClawRegisteredAgentDatabases> = [];
|
||||
|
|
@ -193,6 +343,7 @@ export function resolveAgentDatabaseMigrationTargets(params: {
|
|||
}
|
||||
}
|
||||
params.warnings.push(...discovery.warnings);
|
||||
params.notices?.push(...discovery.notices);
|
||||
return discovery.targets;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -49,7 +49,9 @@ import {
|
|||
} from "./sqlite-transaction.js";
|
||||
import { readSqliteUserVersion } from "./sqlite-user-version.js";
|
||||
import {
|
||||
assertRetainedAgentDatabaseWritable,
|
||||
listTranscriptArchives,
|
||||
RetainedAgentDatabaseHoldError,
|
||||
resolveAgentDatabaseMigrationTargets,
|
||||
} from "./state-migrations.media-persistence-targets.js";
|
||||
import type { MigrationMessages } from "./state-migrations.types.js";
|
||||
|
|
@ -343,27 +345,9 @@ function mediaSourceDriftMessage(
|
|||
return `${pathname} source changed before migration transaction`;
|
||||
}
|
||||
|
||||
function createMigrationDatabaseHandle(
|
||||
database: DatabaseSync,
|
||||
agentId: string,
|
||||
pathname: string,
|
||||
): OpenClawAgentDatabase {
|
||||
return {
|
||||
agentId,
|
||||
db: database,
|
||||
path: pathname,
|
||||
walMaintenance: { checkpoint: () => false, close: () => false },
|
||||
};
|
||||
}
|
||||
|
||||
function refreshAgentDatabasePlannerStatistics(database: DatabaseSync): void {
|
||||
// Doctor owns a stopped-writer maintenance window here. Explicitly analyze every
|
||||
// table because the supported pre-3.46 SQLite floor lacks optimize's all-table bit.
|
||||
database.exec("PRAGMA analysis_limit=1000; ANALYZE main;");
|
||||
}
|
||||
|
||||
function migrateAgentDatabase(params: {
|
||||
agentId: string;
|
||||
assertWritable?: () => void;
|
||||
beforeTransaction?: () => void;
|
||||
pathname: string;
|
||||
}) {
|
||||
|
|
@ -376,6 +360,7 @@ function migrateAgentDatabase(params: {
|
|||
});
|
||||
let userVersion = readSqliteUserVersion(database);
|
||||
const initialVersion = userVersion;
|
||||
params.assertWritable?.();
|
||||
if (userVersion <= PREVIOUS_MEDIA_SCHEMA_VERSION) {
|
||||
migrateOpenClawAgentDatabaseToMediaPrerequisiteSchema(database, {
|
||||
agentId: params.agentId,
|
||||
|
|
@ -429,14 +414,21 @@ function migrateAgentDatabase(params: {
|
|||
{ databaseLabel: params.pathname, operationLabel: "media-persistence-detection" },
|
||||
);
|
||||
if (detected.rewrittenSessions === 0 && detected.rewrittenTrajectoryRows === 0) {
|
||||
refreshAgentDatabasePlannerStatistics(database);
|
||||
params.assertWritable?.();
|
||||
database.exec("PRAGMA analysis_limit=1000; ANALYZE main;");
|
||||
return { ...detected, initialVersion, finalVersion: userVersion };
|
||||
}
|
||||
}
|
||||
|
||||
const sourceVersion = readMediaSourceVersion(database);
|
||||
params.beforeTransaction?.();
|
||||
const owner = createMigrationDatabaseHandle(database, params.agentId, params.pathname);
|
||||
params.assertWritable?.();
|
||||
const owner: OpenClawAgentDatabase = {
|
||||
agentId: params.agentId,
|
||||
db: database,
|
||||
path: params.pathname,
|
||||
walMaintenance: { checkpoint: () => false, close: () => false },
|
||||
};
|
||||
const rewritten = runSqliteImmediateTransactionSync(
|
||||
database,
|
||||
() => {
|
||||
|
|
@ -479,8 +471,11 @@ function migrateAgentDatabase(params: {
|
|||
operationLabel: "media-persistence-retirement",
|
||||
},
|
||||
);
|
||||
params.assertWritable?.();
|
||||
ensureOpenClawAgentDatabaseSchema(database, { agentId: params.agentId, path: params.pathname });
|
||||
refreshAgentDatabasePlannerStatistics(database);
|
||||
// Doctor owns a stopped-writer maintenance window. Explicitly analyze every
|
||||
// table because the supported pre-3.46 SQLite floor lacks optimize's all-table bit.
|
||||
database.exec("PRAGMA analysis_limit=1000; ANALYZE main;");
|
||||
return {
|
||||
...rewritten,
|
||||
initialVersion,
|
||||
|
|
@ -614,6 +609,7 @@ export async function migrateLegacyMediaPersistence(
|
|||
configuredAgentDatabaseTargets?: readonly { agentId: string; path: string }[];
|
||||
hooks?: {
|
||||
beforeArchiveReplace?: (archivePath: string) => void;
|
||||
beforeDatabaseWrite?: (databasePath: string) => void;
|
||||
beforeDatabaseTransaction?: (databasePath: string) => void;
|
||||
};
|
||||
env?: NodeJS.ProcessEnv;
|
||||
|
|
@ -622,12 +618,14 @@ export async function migrateLegacyMediaPersistence(
|
|||
const env = params.env ?? process.env;
|
||||
const changes: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
const notices: string[] = [];
|
||||
try {
|
||||
await withAgentDatabaseMaintenanceLease({ env }, async () => {
|
||||
const targets = resolveAgentDatabaseMigrationTargets({
|
||||
changes,
|
||||
configuredAgentDatabaseTargets: params.configuredAgentDatabaseTargets ?? [],
|
||||
env,
|
||||
notices,
|
||||
warnings,
|
||||
});
|
||||
const seenPaths = new Set<string>();
|
||||
|
|
@ -635,24 +633,33 @@ export async function migrateLegacyMediaPersistence(
|
|||
const archiveDirectories = new Set<string>();
|
||||
for (const entry of targets) {
|
||||
const pathname = entry.path;
|
||||
archiveDirectories.add(
|
||||
resolveSqliteTranscriptArchiveDirectory({
|
||||
agentId: entry.agentId,
|
||||
path: pathname,
|
||||
}),
|
||||
);
|
||||
if (seenPaths.has(entry.realPath)) {
|
||||
continue;
|
||||
}
|
||||
seenPaths.add(entry.realPath);
|
||||
try {
|
||||
const result = migrateAgentDatabase({
|
||||
agentId: entry.agentId,
|
||||
assertWritable: () => {
|
||||
params.hooks?.beforeDatabaseWrite?.(pathname);
|
||||
assertRetainedAgentDatabaseWritable({
|
||||
candidate: entry,
|
||||
configuredAgentDatabaseTargets: params.configuredAgentDatabaseTargets ?? [],
|
||||
env,
|
||||
warnings,
|
||||
});
|
||||
},
|
||||
beforeTransaction: params.hooks?.beforeDatabaseTransaction
|
||||
? () => params.hooks?.beforeDatabaseTransaction?.(pathname)
|
||||
: undefined,
|
||||
pathname,
|
||||
});
|
||||
seenPaths.add(entry.realPath);
|
||||
archiveDirectories.add(
|
||||
resolveSqliteTranscriptArchiveDirectory({
|
||||
agentId: entry.agentId,
|
||||
path: pathname,
|
||||
}),
|
||||
);
|
||||
const schemaAdvanced = result.finalVersion > result.initialVersion;
|
||||
if (entry.source !== "registry" || schemaAdvanced) {
|
||||
registerOpenClawAgentDatabase({ agentId: entry.agentId, env, path: pathname });
|
||||
|
|
@ -668,6 +675,10 @@ export async function migrateLegacyMediaPersistence(
|
|||
);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof RetainedAgentDatabaseHoldError) {
|
||||
error.record(notices, warnings);
|
||||
continue;
|
||||
}
|
||||
databaseMigrationFailed = true;
|
||||
warnings.push(`Skipped agent database migration for ${pathname}: ${String(error)}`);
|
||||
}
|
||||
|
|
@ -717,5 +728,5 @@ export async function migrateLegacyMediaPersistence(
|
|||
} catch (error) {
|
||||
warnings.push(`Agent database maintenance deferred: ${String(error)}`);
|
||||
}
|
||||
return { changes, warnings };
|
||||
return notices.length > 0 ? { changes, warnings, notices } : { changes, warnings };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,12 +26,17 @@ import type {
|
|||
} from "../plugins/doctor-contract-module.js";
|
||||
import { EMPTY_LEGACY_SESSION_SURFACES } from "../plugins/legacy-session-surfaces.types.js";
|
||||
import { resetPluginRuntimeStateForTest, setActivePluginRegistry } from "../plugins/runtime.js";
|
||||
import {
|
||||
beginAgentDeletionJournal,
|
||||
completeAgentDeletionJournal,
|
||||
} from "../state/agent-deletion-journal.js";
|
||||
import { readConfigMachineState, writeConfigMachineState } from "../state/config-machine-state.js";
|
||||
import { listOpenClawRegisteredAgentDatabases } from "../state/openclaw-agent-db-registry.js";
|
||||
import {
|
||||
closeOpenClawAgentDatabasesForTest,
|
||||
ensureOpenClawAgentDatabaseSchema,
|
||||
OPENCLAW_AGENT_SCHEMA_VERSION,
|
||||
openOpenClawAgentDatabase,
|
||||
runOpenClawAgentWriteTransaction,
|
||||
} from "../state/openclaw-agent-db.js";
|
||||
import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db-contract.js";
|
||||
|
|
@ -1939,6 +1944,80 @@ describe("state migrations", () => {
|
|||
expect(migrateLegacyState).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("carries retained database holds through the Doctor migration result", async () => {
|
||||
const root = await createTempDir();
|
||||
const stateDir = path.join(root, ".openclaw");
|
||||
const env = createEnv(stateDir);
|
||||
const agentId = "retired";
|
||||
const operationId = "retained-database-doctor-notice";
|
||||
const databasePath = openOpenClawAgentDatabase({ agentId, env }).path;
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
beginAgentDeletionJournal(
|
||||
{
|
||||
agentId,
|
||||
operationId,
|
||||
agentDir: path.dirname(databasePath),
|
||||
workspaceDir: path.join(stateDir, "workspaces", agentId),
|
||||
sessionsDir: path.join(stateDir, "agents", agentId, "sessions"),
|
||||
databasePaths: [databasePath],
|
||||
deleteFiles: false,
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
expect(completeAgentDeletionJournal(agentId, operationId, { env })).toBe(true);
|
||||
|
||||
const result = await autoMigrateLegacyState({
|
||||
cfg: createConfig(),
|
||||
env,
|
||||
homedir: () => root,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.notices).toContain(
|
||||
`Held retained database ${databasePath} for deleted agent ${agentId}; restore that agent from backup or move this database out of the active state directory, then rerun openclaw doctor --fix.`,
|
||||
);
|
||||
expect(
|
||||
listOpenClawRegisteredAgentDatabases({
|
||||
env,
|
||||
includeIncompatibleSchemaVersions: true,
|
||||
}),
|
||||
).toEqual([expect.objectContaining({ agentId, path: databasePath })]);
|
||||
});
|
||||
|
||||
it("reports pending deletion through the Doctor migration result", async () => {
|
||||
const root = await createTempDir();
|
||||
const stateDir = path.join(root, ".openclaw");
|
||||
const env = createEnv(stateDir);
|
||||
const agentId = "deleting";
|
||||
const databasePath = openOpenClawAgentDatabase({ agentId, env }).path;
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
beginAgentDeletionJournal(
|
||||
{
|
||||
agentId,
|
||||
operationId: "pending-deletion-doctor-notice",
|
||||
agentDir: path.dirname(databasePath),
|
||||
workspaceDir: path.join(stateDir, "workspaces", agentId),
|
||||
sessionsDir: path.join(stateDir, "agents", agentId, "sessions"),
|
||||
databasePaths: [databasePath],
|
||||
deleteFiles: true,
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
|
||||
const result = await autoMigrateLegacyState({
|
||||
cfg: createConfig(),
|
||||
env,
|
||||
homedir: () => root,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.notices).toContain(
|
||||
`Held database ${databasePath} while deletion of agent ${agentId} is pending; finish or retry that agent deletion, then rerun openclaw doctor --fix.`,
|
||||
);
|
||||
});
|
||||
|
||||
it("checks automatic migrations independently for each state directory", async () => {
|
||||
const root = await createTempDir();
|
||||
const stateDirs = [path.join(root, "state-a"), path.join(root, "state-b")];
|
||||
|
|
|
|||
|
|
@ -236,6 +236,7 @@ function rewriteArchiveRow(database: DatabaseSync, planned: ArchiveRowPlan): boo
|
|||
|
||||
function repairPublishedArchiveFile(params: {
|
||||
archiveDirectory: string;
|
||||
assertWritable: () => void;
|
||||
planned: ArchiveRowPlan;
|
||||
}): boolean {
|
||||
const archiveDirectory = path.resolve(params.archiveDirectory);
|
||||
|
|
@ -256,6 +257,7 @@ function repairPublishedArchiveFile(params: {
|
|||
return true;
|
||||
}
|
||||
assertAgentDatabaseMaintenanceAuthority();
|
||||
params.assertWritable();
|
||||
replaceFileAtomicSync({
|
||||
beforeRename: ({ tempPath }) => {
|
||||
const stagedHash = createHash("sha256").update(fs.readFileSync(tempPath)).digest("hex");
|
||||
|
|
@ -265,6 +267,7 @@ function repairPublishedArchiveFile(params: {
|
|||
// Staging and fsync can outlive the timer-driven lease heartbeat. Recheck
|
||||
// at the atomic publication boundary so an expired owner cannot rename.
|
||||
assertAgentDatabaseMaintenanceAuthority();
|
||||
params.assertWritable();
|
||||
},
|
||||
content: params.planned.nextBytes,
|
||||
filePath: archivePath,
|
||||
|
|
@ -326,6 +329,7 @@ function finalizeArchiveCursor(params: {
|
|||
|
||||
export async function migrateTranscriptDirectiveArchives(params: {
|
||||
agentId: string;
|
||||
assertWritable: () => void;
|
||||
database: DatabaseSync;
|
||||
pathname: string;
|
||||
start: ArchiveCursor;
|
||||
|
|
@ -342,6 +346,7 @@ export async function migrateTranscriptDirectiveArchives(params: {
|
|||
if (batch.length === 0) {
|
||||
runSqliteImmediateTransactionSync(params.database, () => {
|
||||
assertAgentDatabaseMaintenanceAuthority();
|
||||
params.assertWritable();
|
||||
params.writeCursor({ phase: "complete" });
|
||||
assertAgentDatabaseMaintenanceAuthority();
|
||||
});
|
||||
|
|
@ -352,6 +357,7 @@ export async function migrateTranscriptDirectiveArchives(params: {
|
|||
params.database,
|
||||
() => {
|
||||
assertAgentDatabaseMaintenanceAuthority();
|
||||
params.assertWritable();
|
||||
const currentRowPresent = rewriteArchiveRow(params.database, planned);
|
||||
assertAgentDatabaseMaintenanceAuthority();
|
||||
return currentRowPresent;
|
||||
|
|
@ -363,12 +369,17 @@ export async function migrateTranscriptDirectiveArchives(params: {
|
|||
},
|
||||
);
|
||||
const fileCurrent = rowPresent
|
||||
? repairPublishedArchiveFile({ archiveDirectory, planned })
|
||||
? repairPublishedArchiveFile({
|
||||
archiveDirectory,
|
||||
assertWritable: params.assertWritable,
|
||||
planned,
|
||||
})
|
||||
: false;
|
||||
runSqliteImmediateTransactionSync(
|
||||
params.database,
|
||||
() => {
|
||||
assertAgentDatabaseMaintenanceAuthority();
|
||||
params.assertWritable();
|
||||
finalizeArchiveCursor({
|
||||
database: params.database,
|
||||
fileCurrent,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js";
|
|||
import { readSessionArchiveContentSync } from "../config/sessions/archive-compression.js";
|
||||
import { resolveSqliteTranscriptArchiveDirectory } from "../config/sessions/session-accessor.sqlite-scope.js";
|
||||
import { reconcileSessionTranscriptIndexInTransaction } from "../config/sessions/session-transcript-index.js";
|
||||
import { beginAgentDeletionJournal } from "../state/agent-deletion-journal.js";
|
||||
import {
|
||||
AGENT_DATABASE_MAINTENANCE_LEASE,
|
||||
assertAgentDatabaseMaintenanceAuthority,
|
||||
|
|
@ -822,7 +823,7 @@ describe("historical transcript directive migration", () => {
|
|||
releaseOpenClawAgentDatabaseLease(competingLeaseId as string, { env });
|
||||
});
|
||||
|
||||
it("preserves a published archive when maintenance expires before rename", async () => {
|
||||
async function expectArchivePreserved(interruption: "lease" | "deletion") {
|
||||
const stateDir = makeTempDir(tempDirs, "transcript-directive-expired-archive-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const opened = openOpenClawAgentDatabase({ agentId: "main", env });
|
||||
|
|
@ -883,19 +884,36 @@ describe("historical transcript directive migration", () => {
|
|||
const authority = vi
|
||||
.spyOn(agentDatabaseLease, "assertAgentDatabaseMaintenanceAuthority")
|
||||
.mockImplementation(() => {
|
||||
if (!competingLeaseId && new Error().stack?.includes("beforeRename")) {
|
||||
openOpenClawStateDatabase({ env })
|
||||
.db.prepare("UPDATE state_leases SET expires_at = ? WHERE scope = ? AND lease_key = ?")
|
||||
.run(
|
||||
Date.now() - 1,
|
||||
AGENT_DATABASE_MAINTENANCE_LEASE.scope,
|
||||
AGENT_DATABASE_MAINTENANCE_LEASE.key,
|
||||
if (new Error().stack?.includes("beforeRename")) {
|
||||
if (interruption === "deletion") {
|
||||
beginAgentDeletionJournal(
|
||||
{
|
||||
agentId: "main",
|
||||
operationId: "retained-before-archive-rename",
|
||||
agentDir: path.dirname(opened.path),
|
||||
workspaceDir: path.join(stateDir, "workspace-main"),
|
||||
sessionsDir: path.join(stateDir, "agents", "main", "sessions"),
|
||||
databasePaths: [opened.path],
|
||||
deleteFiles: false,
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
competingLeaseId = claimOpenClawAgentDatabaseLease({
|
||||
agentId: "competitor",
|
||||
path: path.join(stateDir, "competitor.sqlite"),
|
||||
env,
|
||||
});
|
||||
} else {
|
||||
openOpenClawStateDatabase({ env })
|
||||
.db.prepare(
|
||||
"UPDATE state_leases SET expires_at = ? WHERE scope = ? AND lease_key = ?",
|
||||
)
|
||||
.run(
|
||||
Date.now() - 1,
|
||||
AGENT_DATABASE_MAINTENANCE_LEASE.scope,
|
||||
AGENT_DATABASE_MAINTENANCE_LEASE.key,
|
||||
);
|
||||
competingLeaseId = claimOpenClawAgentDatabaseLease({
|
||||
agentId: "competitor",
|
||||
path: path.join(stateDir, "competitor.sqlite"),
|
||||
env,
|
||||
});
|
||||
}
|
||||
}
|
||||
originalAssert();
|
||||
});
|
||||
|
|
@ -904,20 +922,76 @@ describe("historical transcript directive migration", () => {
|
|||
authority.mockRestore();
|
||||
});
|
||||
|
||||
expect(result.warnings.length).toBeGreaterThanOrEqual(1);
|
||||
expect(
|
||||
result.warnings.every(
|
||||
(warning) => warning.includes("maintenance lease") && warning.includes("was lost"),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(competingLeaseId).toBeDefined();
|
||||
expect(result.warnings).toEqual(
|
||||
interruption === "deletion"
|
||||
? []
|
||||
: expect.arrayContaining([expect.stringMatching(/maintenance lease.*was lost/u)]),
|
||||
);
|
||||
if (interruption === "deletion") {
|
||||
expect((result.notices ?? []).join("\n")).toContain("deletion of agent main is pending");
|
||||
}
|
||||
expect(fs.readFileSync(archivePath)).toEqual(archiveBytes);
|
||||
expect(readMigrationCursor(opened.path)).toEqual({
|
||||
generation: "",
|
||||
phase: "archives",
|
||||
sessionId: "",
|
||||
});
|
||||
releaseOpenClawAgentDatabaseLease(competingLeaseId as string, { env });
|
||||
if (competingLeaseId) {
|
||||
releaseOpenClawAgentDatabaseLease(competingLeaseId, { env });
|
||||
}
|
||||
}
|
||||
|
||||
it.each(["lease", "deletion"] as const)(
|
||||
"preserves a published archive when %s interrupts before rename",
|
||||
expectArchivePreserved,
|
||||
);
|
||||
it("stops resumed transcript writes when agent deletion begins between batches", async () => {
|
||||
const stateDir = makeTempDir(tempDirs, "transcript-directive-retained-resume-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agentId = "main";
|
||||
const opened = openOpenClawAgentDatabase({ agentId, env });
|
||||
for (let index = 0; index <= 32; index += 1) {
|
||||
insertSession(opened.db, {
|
||||
events: [
|
||||
messageEvent({
|
||||
content: [{ type: "text", text: "[[reply_to_current]] Migrating" }],
|
||||
id: `assistant-retained-${index}`,
|
||||
role: "assistant",
|
||||
timestamp: index + 1,
|
||||
}),
|
||||
],
|
||||
generation: "before",
|
||||
sessionId: `session-${String(index).padStart(2, "0")}`,
|
||||
});
|
||||
}
|
||||
const finalSessionId = "session-32";
|
||||
const finalEventJson = readEventJson(opened.path, finalSessionId, 0);
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
const operationId = "retained-during-transcript-resume";
|
||||
vi.spyOn(globalThis, "setImmediate").mockImplementationOnce((callback) => {
|
||||
beginAgentDeletionJournal(
|
||||
{
|
||||
agentId,
|
||||
operationId,
|
||||
agentDir: path.dirname(opened.path),
|
||||
workspaceDir: path.join(stateDir, "workspace-main"),
|
||||
sessionsDir: path.join(stateDir, "agents", agentId, "sessions"),
|
||||
databasePaths: [opened.path],
|
||||
deleteFiles: false,
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
callback();
|
||||
return 0 as unknown as NodeJS.Immediate;
|
||||
});
|
||||
const result = await migrateHistoricalTranscriptDirectives({ env });
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect((result.notices ?? []).join("\n")).toContain(`deletion of agent ${agentId} is pending`);
|
||||
expect(readEventJson(opened.path, finalSessionId, 0)).toBe(finalEventJson);
|
||||
expect(readMigrationCursor(opened.path)).toEqual({
|
||||
phase: "transcripts",
|
||||
sessionId: "session-31",
|
||||
});
|
||||
});
|
||||
|
||||
it("renews maintenance beyond its original lifetime and fences writers through final mutation", async () => {
|
||||
|
|
|
|||
|
|
@ -26,7 +26,11 @@ import {
|
|||
} from "./kysely-sync.js";
|
||||
import { openNodeSqliteDatabase } from "./node-sqlite.js";
|
||||
import { runSqliteImmediateTransactionSync } from "./sqlite-transaction.js";
|
||||
import { resolveAgentDatabaseMigrationTargets } from "./state-migrations.media-persistence-targets.js";
|
||||
import {
|
||||
assertRetainedAgentDatabaseWritable,
|
||||
resolveAgentDatabaseMigrationTargets,
|
||||
RetainedAgentDatabaseHoldError,
|
||||
} from "./state-migrations.media-persistence-targets.js";
|
||||
import {
|
||||
migrateTranscriptDirectiveArchives,
|
||||
TRANSCRIPT_DIRECTIVE_MIGRATION_BATCH_SIZE,
|
||||
|
|
@ -265,6 +269,7 @@ async function yieldToMaintenanceHeartbeat(): Promise<void> {
|
|||
|
||||
async function migrateTranscriptSessions(params: {
|
||||
agentId: string;
|
||||
assertWritable: () => void;
|
||||
database: DatabaseSync;
|
||||
owner: OpenClawAgentDatabase;
|
||||
pathname: string;
|
||||
|
|
@ -277,6 +282,7 @@ async function migrateTranscriptSessions(params: {
|
|||
if (sessionIds.length === 0) {
|
||||
runSqliteImmediateTransactionSync(params.database, () => {
|
||||
assertAgentDatabaseMaintenanceAuthority();
|
||||
params.assertWritable();
|
||||
writeMigrationCursor(params.database, params.agentId, {
|
||||
generation: "",
|
||||
phase: "archives",
|
||||
|
|
@ -293,6 +299,7 @@ async function migrateTranscriptSessions(params: {
|
|||
params.database,
|
||||
() => {
|
||||
assertAgentDatabaseMaintenanceAuthority();
|
||||
params.assertWritable();
|
||||
assertTranscriptSessionSourceUnchanged(params.database, sessionId, planned);
|
||||
updateSqliteTranscriptEventJsonInTransaction(
|
||||
params.owner,
|
||||
|
|
@ -325,8 +332,10 @@ async function migrateTranscriptSessions(params: {
|
|||
|
||||
async function migrateAgentDatabase(params: {
|
||||
agentId: string;
|
||||
assertWritable: () => void;
|
||||
pathname: string;
|
||||
}): Promise<DatabaseMigrationResult> {
|
||||
params.assertWritable();
|
||||
migrateOpenClawAgentDatabaseForMaintenance(params);
|
||||
const database = openNodeSqliteDatabase(params.pathname);
|
||||
try {
|
||||
|
|
@ -341,6 +350,7 @@ async function migrateAgentDatabase(params: {
|
|||
cursor.phase === "transcripts"
|
||||
? await migrateTranscriptSessions({
|
||||
agentId: params.agentId,
|
||||
assertWritable: params.assertWritable,
|
||||
database,
|
||||
owner,
|
||||
pathname: params.pathname,
|
||||
|
|
@ -352,6 +362,7 @@ async function migrateAgentDatabase(params: {
|
|||
archiveCursor.phase === "archives"
|
||||
? await migrateTranscriptDirectiveArchives({
|
||||
agentId: params.agentId,
|
||||
assertWritable: params.assertWritable,
|
||||
database,
|
||||
pathname: params.pathname,
|
||||
start: archiveCursor,
|
||||
|
|
@ -423,11 +434,13 @@ export async function migrateHistoricalTranscriptDirectives(
|
|||
const env = params.env ?? process.env;
|
||||
const changes: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
const notices: string[] = [];
|
||||
try {
|
||||
const discoveredTargets = resolveAgentDatabaseMigrationTargets({
|
||||
changes,
|
||||
configuredAgentDatabaseTargets: params.configuredAgentDatabaseTargets ?? [],
|
||||
env,
|
||||
notices,
|
||||
warnings,
|
||||
});
|
||||
const targets: typeof discoveredTargets = [];
|
||||
|
|
@ -449,13 +462,28 @@ export async function migrateHistoricalTranscriptDirectives(
|
|||
}
|
||||
}
|
||||
if (targets.length === 0) {
|
||||
return { changes, warnings };
|
||||
return notices.length > 0 ? { changes, warnings, notices } : { changes, warnings };
|
||||
}
|
||||
await withAgentDatabaseMaintenanceLease({ env }, async () => {
|
||||
for (const target of targets) {
|
||||
const approvedRealPaths = new Set(targets.map((target) => target.realPath));
|
||||
const currentTargets = resolveAgentDatabaseMigrationTargets({
|
||||
changes,
|
||||
configuredAgentDatabaseTargets: params.configuredAgentDatabaseTargets ?? [],
|
||||
env,
|
||||
notices,
|
||||
warnings,
|
||||
}).filter((target) => approvedRealPaths.has(target.realPath));
|
||||
for (const target of currentTargets) {
|
||||
try {
|
||||
const result = await migrateAgentDatabase({
|
||||
agentId: target.agentId,
|
||||
assertWritable: () =>
|
||||
assertRetainedAgentDatabaseWritable({
|
||||
candidate: target,
|
||||
configuredAgentDatabaseTargets: params.configuredAgentDatabaseTargets ?? [],
|
||||
env,
|
||||
warnings,
|
||||
}),
|
||||
pathname: target.path,
|
||||
});
|
||||
if (result.transcriptSessions > 0 || result.archivedTranscripts > 0) {
|
||||
|
|
@ -464,6 +492,10 @@ export async function migrateHistoricalTranscriptDirectives(
|
|||
);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof RetainedAgentDatabaseHoldError) {
|
||||
error.record(notices, warnings);
|
||||
continue;
|
||||
}
|
||||
warnings.push(
|
||||
`Skipped historical transcript directive migration for ${target.path}: ${String(error)}`,
|
||||
);
|
||||
|
|
@ -473,5 +505,5 @@ export async function migrateHistoricalTranscriptDirectives(
|
|||
} catch (error) {
|
||||
warnings.push(`Skipped historical transcript directive migration: ${String(error)}`);
|
||||
}
|
||||
return { changes, warnings };
|
||||
return notices.length > 0 ? { changes, warnings, notices } : { changes, warnings };
|
||||
}
|
||||
|
|
|
|||
117
src/state/agent-deletion-journal.read.ts
Normal file
117
src/state/agent-deletion-journal.read.ts
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
import path from "node:path";
|
||||
import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js";
|
||||
import {
|
||||
isSqliteCorruptionError,
|
||||
runSqliteDeferredTransactionSync,
|
||||
} from "../infra/sqlite-transaction.js";
|
||||
import { withExistingOpenClawStateDatabaseReadOnly } from "./openclaw-state-db-readonly.js";
|
||||
import { tableExists } from "./openclaw-state-db-schema-helpers.js";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js";
|
||||
|
||||
type RetainedAgentDeletion = {
|
||||
agentId: string;
|
||||
databasePaths: readonly string[];
|
||||
state: "pending" | "retained";
|
||||
warning?: string;
|
||||
};
|
||||
|
||||
type RetainedAgentDeletionReadFailure = { status: "unavailable"; warning?: string };
|
||||
|
||||
export type RetainedAgentDeletionDisposition =
|
||||
| readonly RetainedAgentDeletion[]
|
||||
| { status: "absent" }
|
||||
| RetainedAgentDeletionReadFailure;
|
||||
|
||||
export function retainedAgentDeletionHistoryUnavailable(
|
||||
disposition: RetainedAgentDeletionDisposition,
|
||||
): disposition is RetainedAgentDeletionReadFailure {
|
||||
return "status" in disposition && disposition.status === "unavailable";
|
||||
}
|
||||
|
||||
export function retainedAgentDeletionHistoryAbsent(
|
||||
disposition: RetainedAgentDeletionDisposition,
|
||||
): disposition is { status: "absent" } {
|
||||
return "status" in disposition && disposition.status === "absent";
|
||||
}
|
||||
|
||||
export function retainedAgentDeletionReadWarning(
|
||||
disposition: RetainedAgentDeletionDisposition,
|
||||
): string | undefined {
|
||||
if (retainedAgentDeletionHistoryUnavailable(disposition)) {
|
||||
return disposition.warning;
|
||||
}
|
||||
if (retainedAgentDeletionHistoryAbsent(disposition)) {
|
||||
return undefined;
|
||||
}
|
||||
const warnings = [...new Set(disposition.flatMap((entry) => entry.warning ?? []))];
|
||||
return warnings.length > 0 ? warnings.join(" ") : undefined;
|
||||
}
|
||||
|
||||
function parseDatabasePaths(value: string): string[] | undefined {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(value);
|
||||
if (Array.isArray(parsed) && parsed.every((entry) => typeof entry === "string")) {
|
||||
return parsed;
|
||||
}
|
||||
} catch {
|
||||
// The readable identity still fences this deletion below.
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Read deletions that currently revoke database writes without mutating journal history. */
|
||||
export function readRetainedAgentDeletions(
|
||||
env: NodeJS.ProcessEnv,
|
||||
): RetainedAgentDeletionDisposition {
|
||||
try {
|
||||
return (
|
||||
withExistingOpenClawStateDatabaseReadOnly(
|
||||
({ db }) => {
|
||||
if (!tableExists(db, "agent_deletion_journal")) {
|
||||
return { status: "unavailable" as const };
|
||||
}
|
||||
const database =
|
||||
getNodeSqliteKysely<Pick<OpenClawStateKyselyDatabase, "agent_deletion_journal">>(db);
|
||||
return runSqliteDeferredTransactionSync(db, () =>
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
database
|
||||
.selectFrom("agent_deletion_journal")
|
||||
.select(["agent_id", "agent_dir", "database_paths_json", "cleanup_completed"])
|
||||
.where((expression) =>
|
||||
expression.or([
|
||||
expression("cleanup_completed", "=", 0),
|
||||
expression("delete_files", "=", 0),
|
||||
]),
|
||||
)
|
||||
.orderBy("agent_id", "asc"),
|
||||
).rows.map((row) => {
|
||||
const databasePaths = parseDatabasePaths(row.database_paths_json);
|
||||
const entry: RetainedAgentDeletion = {
|
||||
agentId: row.agent_id,
|
||||
databasePaths: [
|
||||
path.join(row.agent_dir, "openclaw-agent.sqlite"),
|
||||
...(databasePaths ?? []),
|
||||
],
|
||||
state: row.cleanup_completed === 0 ? ("pending" as const) : ("retained" as const),
|
||||
};
|
||||
if (!databasePaths) {
|
||||
entry.warning = `Could not read database paths for deleted agent ${row.agent_id}; its known database identity remains held.`;
|
||||
}
|
||||
return entry;
|
||||
}),
|
||||
);
|
||||
},
|
||||
{ env },
|
||||
) ?? { status: "absent" as const }
|
||||
);
|
||||
} catch (error) {
|
||||
if (isSqliteCorruptionError(error)) {
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
status: "unavailable",
|
||||
warning: `Could not read retained agent deletion history: ${String(error)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue