mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 01:29:56 +00:00
fix(memory): reduce CPU work during index sync (#150602)
This commit is contained in:
parent
902071456d
commit
6fa64d9f71
44 changed files with 345 additions and 270 deletions
|
|
@ -710,7 +710,7 @@ extensions/memory-core/src/dreaming-phases.ts 2
|
|||
extensions/memory-core/src/dreaming.ts 1
|
||||
extensions/memory-core/src/memory/embeddings.ts 2
|
||||
extensions/memory-core/src/memory/hybrid.ts 1
|
||||
extensions/memory-core/src/memory/manager-db.ts 5
|
||||
extensions/memory-core/src/memory/manager-db.ts 2
|
||||
extensions/memory-core/src/memory/manager-embedding-errors.ts 2
|
||||
extensions/memory-core/src/memory/manager-embedding-ops.ts 1
|
||||
extensions/memory-core/src/memory/manager-keyword-retrieval.ts 1
|
||||
|
|
|
|||
|
|
@ -374,6 +374,7 @@ Use `isLoopbackHost(host)` when a plugin must accept only the local machine. It
|
|||
| `plugin-sdk/session-discussion` | External session discussion provider contracts, registration, and canonical Control UI session path building |
|
||||
| `plugin-sdk/session-transcript-runtime` | Private-local after July 2026; Transcript identity, bounded raw and visible cursors, scoped target/read/write helpers, read-only native transcript catalog pages with portable sender attribution, visible message-entry projection, update publishing, write locks, and transcript memory hit keys |
|
||||
| `plugin-sdk/sqlite-runtime` | Private-local after July 2026; SQLite agent-schema, path, transaction, and shared-handle borrowing helpers for first-party runtime. Type-only `Generated` and `Selectable` model generated columns and selected rows in Kysely table definitions. `compileSqliteQueryBindings` compiles fixed Kysely SQL with fresh bindings for caller-owned native statements; statement lifetime stays with the caller. `iterateSqliteQuerySync` streams Kysely query rows for incremental decoding; consume the iterator before closing its database. `sqliteStringSet` binds string membership as one SQLite JSON table-valued query parameter, preserving one query snapshot without a variable-count placeholder list. `borrowOpenClawAgentDatabase` returns `{ db, release }`; active borrows prevent cache eviction, `release()` does not close the handle, and explicit owner disposal still revokes it. `withOpenClawAgentDatabaseAsync` admits a connection asynchronously and retains its original identity through the operation’s settlement; it does not move callback execution off the caller thread. |
|
||||
| `plugin-sdk/sqlite-worker-runtime` | Private-local native SQLite, Kysely query, synchronous transaction, and operation-admission primitives for worker backends and their shared helpers. Imports existing owners directly so workers do not load host database lifecycle code. Host store creation and lifecycle coordination remain on `sqlite-runtime`. |
|
||||
| `plugin-sdk/cron-store-runtime` | Private-local after July 2026; Cron store path/load/save helpers |
|
||||
| `plugin-sdk/state-paths` | State/OAuth dir path helpers |
|
||||
| `plugin-sdk/plugin-state-runtime` | Private-local after July 2026; Plugin-scoped keyed-state and BLOB contracts plus connection pragma, verified WAL maintenance, and atomic STRICT-schema migration helpers. Plugin-state leases were removed; use SQLite transactions and keyed stores instead |
|
||||
|
|
@ -521,7 +522,7 @@ Use `isLoopbackHost(host)` when a plugin must accept only the local machine. It
|
|||
| `plugin-sdk/memory-core-host-engine-fs` | Private-local focused filesystem and user-path helpers for doctor migrations |
|
||||
| `plugin-sdk/memory-core-host-engine-embeddings` | Private-local after July 2026; Memory host embedding contracts and batch/remote helpers. Providers register through the generic embedding provider API. |
|
||||
| `plugin-sdk/memory-core-host-engine-sessions` | Private-local after July 2026; Memory session transcript and query helpers |
|
||||
| `plugin-sdk/memory-core-host-engine-schema` | Private-local focused memory index schema and sqlite-vec helpers for doctor migrations |
|
||||
| `plugin-sdk/memory-core-host-engine-schema` | Private-local memory index schema and sqlite-vec operations shared by Doctor, host maintenance, and native publication workers |
|
||||
| `plugin-sdk/memory-core-host-engine-indexing` | Private-local immutable chunk preparation, annotations, hashes, and embedding input limits for indexing workers |
|
||||
| `plugin-sdk/memory-core-host-engine-knn` | Private-local read-only SQLite ownership checks, sqlite-vec, and text/vector primitives for isolated retrieval workers and children |
|
||||
| `plugin-sdk/memory-core-host-engine-storage` | Private-local after July 2026; Memory host storage engine exports |
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import {
|
|||
getNodeSqliteKysely,
|
||||
openNodeSqliteDatabase,
|
||||
type SqliteWorkerBackend,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
} from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import type { IMessageReceiptDbOperations } from "./send-receipt-db.js";
|
||||
import { normalizeIMessageHandle } from "./targets.js";
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import type { DatabaseSync } from "node:sqlite";
|
|||
import {
|
||||
prepareSqliteQuerySync,
|
||||
type getNodeSqliteKysely,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
} from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import { MAX_FRAMES_PER_CALL } from "./analyze.js";
|
||||
import type { LogbookDatabase, toFrame } from "./store-schema.js";
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import * as fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import type { SqliteWorkerBackend } from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import type { SqliteWorkerBackend } from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import { useAutoCleanupTempDirTracker } from "openclaw/plugin-sdk/test-env";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { LogbookOperations } from "./store-contract.js";
|
||||
|
|
@ -12,8 +12,8 @@ vi.mock("node:fs", async (importOriginal) => {
|
|||
const actual = await importOriginal<typeof import("node:fs")>();
|
||||
return { ...actual, rmSync: vi.fn(actual.rmSync) };
|
||||
});
|
||||
vi.mock("openclaw/plugin-sdk/sqlite-runtime", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("openclaw/plugin-sdk/sqlite-runtime")>();
|
||||
vi.mock("openclaw/plugin-sdk/sqlite-worker-runtime", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("openclaw/plugin-sdk/sqlite-worker-runtime")>();
|
||||
return {
|
||||
...actual,
|
||||
openNodeSqliteDatabase: (...args: Parameters<typeof actual.openNodeSqliteDatabase>) => {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import path from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import type { SqliteWorkerBackend } from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import type { SqliteWorkerBackend } from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import { useAutoCleanupTempDirTracker } from "openclaw/plugin-sdk/test-env";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { LogbookOperations } from "./store-contract.js";
|
||||
|
|
@ -14,8 +14,8 @@ const reads = vi.hoisted(() => ({
|
|||
frameTextBytes: 0,
|
||||
}));
|
||||
const preparations = vi.hoisted(() => new Map<string, number>());
|
||||
vi.mock("openclaw/plugin-sdk/sqlite-runtime", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("openclaw/plugin-sdk/sqlite-runtime")>();
|
||||
vi.mock("openclaw/plugin-sdk/sqlite-worker-runtime", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("openclaw/plugin-sdk/sqlite-worker-runtime")>();
|
||||
return {
|
||||
...actual,
|
||||
openNodeSqliteDatabase: (...args: Parameters<typeof actual.openNodeSqliteDatabase>) => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { Generated, Selectable } from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import type { Generated, Selectable } from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import { asOptionalObjectRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type {
|
||||
LogbookBatch,
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import {
|
|||
prepareSqliteQuerySync,
|
||||
runSqliteImmediateTransactionSync,
|
||||
type SqliteWorkerBackend,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
} from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import { pickKeyframeId } from "./analyze.js";
|
||||
import type {
|
||||
LogbookBatchInput,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import {
|
|||
executeSqliteQuerySync,
|
||||
getNodeSqliteKysely,
|
||||
tableExists,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
} from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
|
||||
type TombstoneDatabase = {
|
||||
memory_session_tombstones: { session_id: string; agent_id: string };
|
||||
|
|
|
|||
|
|
@ -3,11 +3,11 @@ import type {
|
|||
MemoryChunk,
|
||||
MemoryEntryProvenance,
|
||||
MemorySource,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-indexing";
|
||||
import {
|
||||
compileSqliteQueryBindings,
|
||||
getNodeSqliteKysely,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
} from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
|
||||
export type IndexedMemoryChunk = MemoryChunk & {
|
||||
importance: number | null;
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import {
|
|||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { resolveRuntimeWorkerUrl } from "openclaw/plugin-sdk/process-runtime";
|
||||
import {
|
||||
borrowOpenClawAgentDatabase,
|
||||
openSqliteWorkerStore,
|
||||
openOpenClawAgentSqliteWorkerStore,
|
||||
runSqliteWorkerStoreWrite,
|
||||
|
|
@ -19,10 +20,11 @@ import {
|
|||
type StoreWriterQueue,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import { memoryCpuProcessEntrypoints } from "./manager-cpu-entrypoints.js";
|
||||
import { MemoryIndexRevisionConflictError } from "./manager-db-kernel.js";
|
||||
import {
|
||||
MemoryIndexRevisionConflictError,
|
||||
closeMemoryDatabase,
|
||||
openMemoryDatabaseAtPath,
|
||||
openMemoryDatabaseReadOnlyAtPath,
|
||||
} from "./manager-db.js";
|
||||
import type {
|
||||
MemoryPublicationConnection,
|
||||
|
|
@ -53,6 +55,32 @@ export class MemoryIndexDatabase {
|
|||
private releaseInProgress = false;
|
||||
shadowReleased = false;
|
||||
|
||||
static openPublished(params: {
|
||||
agentId: string;
|
||||
writeOptions: Parameters<typeof withOpenClawAgentDatabaseWrite>[0] & { path: string };
|
||||
readOnly: boolean;
|
||||
allowExtension: boolean;
|
||||
maintenanceSource?: MemoryIndexDatabase;
|
||||
}): MemoryIndexDatabase {
|
||||
const connection = params.readOnly
|
||||
? openMemoryDatabaseReadOnlyAtPath(
|
||||
params.writeOptions.path,
|
||||
params.allowExtension,
|
||||
params.agentId,
|
||||
)
|
||||
: borrowOpenClawAgentDatabase(params.writeOptions);
|
||||
if (params.maintenanceSource && connection.db !== params.maintenanceSource.db) {
|
||||
connection.release();
|
||||
throw new Error("Memory maintenance source connection changed");
|
||||
}
|
||||
return new MemoryIndexDatabase(
|
||||
connection.db,
|
||||
connection.release,
|
||||
params.readOnly,
|
||||
params.writeOptions,
|
||||
);
|
||||
}
|
||||
|
||||
static openShadow(filename: string, allowExtension: boolean): MemoryIndexDatabase {
|
||||
let database: MemoryIndexDatabase | undefined;
|
||||
const db = openMemoryDatabaseAtPath(filename, allowExtension, (operation) =>
|
||||
|
|
|
|||
191
extensions/memory-core/src/memory/manager-db-kernel.ts
Normal file
191
extensions/memory-core/src/memory/manager-db-kernel.ts
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
import type { DatabaseSync } from "node:sqlite";
|
||||
import {
|
||||
dropMemoryPathFtsTriggers,
|
||||
ensureMemoryChunkProvenance,
|
||||
ensureMemoryRecallMetadataSchema,
|
||||
ensureMemoryPathFtsTriggers,
|
||||
MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE,
|
||||
MEMORY_INDEX_PATHS_FTS_TABLE,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-schema";
|
||||
import { runSqliteImmediateTransactionSync } from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import { markMemoryVectorIndexClean } from "./manager-vector-rebuild-state.js";
|
||||
|
||||
const MEMORY_REINDEX_SCHEMA = "memory_reindex";
|
||||
export const MEMORY_INDEX_STATE_ID = 1;
|
||||
|
||||
function tableExists(db: DatabaseSync, schema: string, tableName: string): boolean {
|
||||
const row = db
|
||||
.prepare(`SELECT 1 AS ok FROM ${schema}.sqlite_master WHERE type = 'table' AND name = ?`)
|
||||
.get(tableName);
|
||||
return row?.ok === 1;
|
||||
}
|
||||
|
||||
export { tableExists as memoryDatabaseTableExists };
|
||||
|
||||
function readTableSql(db: DatabaseSync, schema: string, tableName: string): string | null {
|
||||
const row = db
|
||||
.prepare(`SELECT sql FROM ${schema}.sqlite_master WHERE type = 'table' AND name = ?`)
|
||||
.get(tableName);
|
||||
return typeof row?.sql === "string" && row.sql.trim() ? row.sql : null;
|
||||
}
|
||||
|
||||
export function readMemoryDatabaseRevision(db: DatabaseSync): number {
|
||||
const row = db
|
||||
.prepare("SELECT revision FROM memory_index_state WHERE id = ?")
|
||||
.get(MEMORY_INDEX_STATE_ID);
|
||||
if (typeof row?.revision !== "number" || !Number.isSafeInteger(row.revision)) {
|
||||
throw new Error("Memory index revision is missing or invalid");
|
||||
}
|
||||
return row.revision;
|
||||
}
|
||||
|
||||
export class MemoryIndexRevisionConflictError extends Error {
|
||||
override name = "MemoryIndexRevisionConflictError";
|
||||
}
|
||||
|
||||
function replaceVirtualTable(params: {
|
||||
db: DatabaseSync;
|
||||
tableName: "memory_index_chunks_fts" | "memory_index_chunks_vec";
|
||||
columns: string;
|
||||
ignoreDropErrorWhenSourceMissing?: boolean;
|
||||
}): void {
|
||||
const { db, tableName, columns } = params;
|
||||
const createSql = readTableSql(db, MEMORY_REINDEX_SCHEMA, tableName);
|
||||
if (!createSql) {
|
||||
try {
|
||||
db.exec(`DROP TABLE IF EXISTS main.${tableName}`);
|
||||
} catch (err) {
|
||||
if (!params.ignoreDropErrorWhenSourceMissing) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
db.exec(`DROP TABLE IF EXISTS main.${tableName}`);
|
||||
db.exec(createSql);
|
||||
db.exec(
|
||||
`INSERT INTO main.${tableName} (${columns}) ` +
|
||||
`SELECT ${columns} FROM ${MEMORY_REINDEX_SCHEMA}.${tableName}`,
|
||||
);
|
||||
}
|
||||
|
||||
function replaceMemoryPathFtsTable(db: DatabaseSync): void {
|
||||
const createSql = readTableSql(db, MEMORY_REINDEX_SCHEMA, MEMORY_INDEX_PATHS_FTS_TABLE);
|
||||
db.exec(`DROP TABLE IF EXISTS main.${MEMORY_INDEX_PATHS_FTS_TABLE}`);
|
||||
if (!createSql) {
|
||||
return;
|
||||
}
|
||||
db.exec(createSql);
|
||||
// Bulk publication already suspends row triggers. Rebuild from the copied
|
||||
// stable source ids so later singleton deletes remain direct rowid lookups.
|
||||
db.exec(
|
||||
`INSERT INTO main.${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source) ` +
|
||||
`SELECT id, path, source FROM main.memory_index_sources`,
|
||||
);
|
||||
}
|
||||
|
||||
/** The native publication owner receives prepared connection and source facts. */
|
||||
type MemoryDatabasePublication = {
|
||||
targetDb: DatabaseSync;
|
||||
sourcePath: string;
|
||||
metaKey: string;
|
||||
expectedRevision: number;
|
||||
onBegin?: () => void;
|
||||
withCommit?: (commit: () => void) => void;
|
||||
vectorIndexComplete?: boolean;
|
||||
};
|
||||
|
||||
/** The admitted connection owns ATTACH, atomic replacement, COMMIT and DETACH. */
|
||||
export function publishMemoryDatabaseTables(params: MemoryDatabasePublication): void {
|
||||
ensureMemoryRecallMetadataSchema(params.targetDb);
|
||||
// Existing pre-provenance databases need this before the publication writes it.
|
||||
ensureMemoryChunkProvenance(params.targetDb);
|
||||
// Admission precedes ATTACH; no shadow attachment or transaction crosses an await.
|
||||
params.targetDb.prepare(`ATTACH DATABASE ? AS ${MEMORY_REINDEX_SCHEMA}`).run(params.sourcePath);
|
||||
try {
|
||||
runSqliteImmediateTransactionSync(
|
||||
params.targetDb,
|
||||
() => {
|
||||
params.onBegin?.();
|
||||
const liveRevision = readMemoryDatabaseRevision(params.targetDb);
|
||||
if (liveRevision !== params.expectedRevision) {
|
||||
throw new MemoryIndexRevisionConflictError(
|
||||
`Memory index changed while full reindex was building ` +
|
||||
`(expected revision ${params.expectedRevision}, found ${liveRevision}); retry the full reindex.`,
|
||||
);
|
||||
}
|
||||
const publishesPathFts = tableExists(
|
||||
params.targetDb,
|
||||
MEMORY_REINDEX_SCHEMA,
|
||||
MEMORY_INDEX_PATHS_FTS_TABLE,
|
||||
);
|
||||
// Bulk source replacement must not fire one FTS5 scan per old row.
|
||||
// Restore the schema-owned triggers only after the derived table is replaced.
|
||||
dropMemoryPathFtsTriggers(params.targetDb);
|
||||
params.targetDb
|
||||
.prepare("DELETE FROM main.memory_index_meta WHERE key = ?")
|
||||
.run(params.metaKey);
|
||||
params.targetDb
|
||||
.prepare(
|
||||
`INSERT INTO main.memory_index_meta (key, value)
|
||||
SELECT key, value FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_meta WHERE key = ?`,
|
||||
)
|
||||
.run(params.metaKey);
|
||||
|
||||
params.targetDb.exec(`
|
||||
DELETE FROM main.memory_index_sources;
|
||||
INSERT INTO main.memory_index_sources (id, path, source, hash, mtime, size)
|
||||
SELECT id, path, source, hash, mtime, size
|
||||
FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_sources;
|
||||
|
||||
DELETE FROM main.memory_index_chunks;
|
||||
INSERT INTO main.memory_index_chunks (
|
||||
id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
|
||||
)
|
||||
SELECT
|
||||
id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
|
||||
FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_chunks;
|
||||
|
||||
DELETE FROM main.${MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE};
|
||||
INSERT INTO main.${MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE} (
|
||||
chunk_id, importance, triggers, project_key
|
||||
)
|
||||
SELECT chunk_id, importance, triggers, project_key
|
||||
FROM ${MEMORY_REINDEX_SCHEMA}.${MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE};
|
||||
|
||||
DELETE FROM main.memory_index_chunk_provenance;
|
||||
INSERT INTO main.memory_index_chunk_provenance (
|
||||
chunk_id, origin_class, session_kind, observed_at, supersedes_key
|
||||
)
|
||||
SELECT chunk_id, origin_class, session_kind, observed_at, supersedes_key
|
||||
FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_chunk_provenance;
|
||||
`);
|
||||
|
||||
replaceVirtualTable({
|
||||
db: params.targetDb,
|
||||
tableName: "memory_index_chunks_fts",
|
||||
columns: "text, id, path, source, model, start_line, end_line",
|
||||
});
|
||||
replaceMemoryPathFtsTable(params.targetDb);
|
||||
if (publishesPathFts) {
|
||||
ensureMemoryPathFtsTriggers(params.targetDb);
|
||||
}
|
||||
replaceVirtualTable({
|
||||
db: params.targetDb,
|
||||
tableName: "memory_index_chunks_vec",
|
||||
columns: "id, embedding",
|
||||
// A vector-disabled connection may not have sqlite-vec loaded and cannot
|
||||
// drop an old virtual table. Missing vector metadata forces a strict
|
||||
// rebuild before that table can be queried again.
|
||||
ignoreDropErrorWhenSourceMissing: true,
|
||||
});
|
||||
if (params.vectorIndexComplete) {
|
||||
markMemoryVectorIndexClean(params.targetDb);
|
||||
}
|
||||
},
|
||||
{ withCommit: params.withCommit },
|
||||
);
|
||||
} finally {
|
||||
params.targetDb.exec(`DETACH DATABASE ${MEMORY_REINDEX_SCHEMA}`);
|
||||
}
|
||||
}
|
||||
|
|
@ -13,12 +13,14 @@ import {
|
|||
resetMemoryCoreDreamingStateForTests,
|
||||
} from "../test-helpers.js";
|
||||
import {
|
||||
cleanupAgedMemoryReindexTempFiles,
|
||||
closeMemoryDatabase,
|
||||
openMemoryDatabaseAtPath,
|
||||
publishMemoryDatabaseTables,
|
||||
readMemoryDatabaseRevision,
|
||||
MemoryIndexRevisionConflictError,
|
||||
} from "./manager-db-kernel.js";
|
||||
import {
|
||||
cleanupAgedMemoryReindexTempFiles,
|
||||
closeMemoryDatabase,
|
||||
openMemoryDatabaseAtPath,
|
||||
resetMemoryDatabase,
|
||||
} from "./manager-db.js";
|
||||
import { waitForMemoryReindexLock } from "./manager-reindex-lock.js";
|
||||
|
|
|
|||
|
|
@ -6,14 +6,8 @@ import type { DatabaseSync } from "node:sqlite";
|
|||
import {
|
||||
closeMemorySqliteWalMaintenance,
|
||||
configureMemorySqliteWalMaintenance,
|
||||
dropMemoryPathFtsTriggers,
|
||||
ensureMemoryChunkProvenance,
|
||||
ensureMemoryIndexSchema,
|
||||
ensureMemoryRecallMetadataSchema,
|
||||
ensureMemoryPathFtsTriggers,
|
||||
loadSqliteVecExtension,
|
||||
MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE,
|
||||
MEMORY_INDEX_PATHS_FTS_TABLE,
|
||||
MEMORY_INDEX_DERIVED_TABLES,
|
||||
MEMORY_INDEX_STATE_TABLE,
|
||||
MEMORY_INDEX_VECTOR_TABLE,
|
||||
|
|
@ -24,12 +18,14 @@ import {
|
|||
runSqliteImmediateTransactionSync,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import { withMemoryWorkspaceLock } from "../memory-workspace-lock.js";
|
||||
import {
|
||||
MEMORY_INDEX_STATE_ID,
|
||||
memoryDatabaseTableExists as tableExists,
|
||||
readMemoryDatabaseRevision,
|
||||
} from "./manager-db-kernel.js";
|
||||
import { withMemoryIndexPublishGeneration } from "./manager-index-generation-lease.js";
|
||||
import { waitForMemoryReindexLock } from "./manager-reindex-lock.js";
|
||||
import { markMemoryVectorIndexClean } from "./manager-vector-rebuild-state.js";
|
||||
|
||||
const MEMORY_REINDEX_SCHEMA = "memory_reindex";
|
||||
const MEMORY_INDEX_STATE_ID = 1;
|
||||
const MEMORY_DATABASE_FILE_SUFFIXES = ["", "-wal", "-shm", "-journal"] as const;
|
||||
const MEMORY_REINDEX_ENTRY_SUFFIXES = ["-wal", "-shm", "-journal", ""] as const;
|
||||
const MEMORY_REINDEX_UUID_PATTERN =
|
||||
|
|
@ -64,22 +60,6 @@ async function isRegularFile(filePath: string): Promise<boolean> {
|
|||
}
|
||||
}
|
||||
|
||||
function tableExists(db: DatabaseSync, schema: string, tableName: string): boolean {
|
||||
const row = db
|
||||
.prepare(`SELECT 1 AS ok FROM ${schema}.sqlite_master WHERE type = 'table' AND name = ?`)
|
||||
.get(tableName) as { ok?: unknown } | undefined;
|
||||
return row?.ok === 1;
|
||||
}
|
||||
|
||||
export { tableExists as memoryDatabaseTableExists };
|
||||
|
||||
function readTableSql(db: DatabaseSync, schema: string, tableName: string): string | null {
|
||||
const row = db
|
||||
.prepare(`SELECT sql FROM ${schema}.sqlite_master WHERE type = 'table' AND name = ?`)
|
||||
.get(tableName) as { sql?: unknown } | undefined;
|
||||
return typeof row?.sql === "string" && row.sql.trim() ? row.sql : null;
|
||||
}
|
||||
|
||||
function hasSqliteVecExtension(db: DatabaseSync): boolean {
|
||||
try {
|
||||
const row = db.prepare("SELECT vec_version() AS version").get() as
|
||||
|
|
@ -91,20 +71,6 @@ function hasSqliteVecExtension(db: DatabaseSync): boolean {
|
|||
}
|
||||
}
|
||||
|
||||
export function readMemoryDatabaseRevision(db: DatabaseSync): number {
|
||||
const row = db
|
||||
.prepare("SELECT revision FROM memory_index_state WHERE id = ?")
|
||||
.get(MEMORY_INDEX_STATE_ID) as { revision?: unknown } | undefined;
|
||||
if (typeof row?.revision !== "number" || !Number.isSafeInteger(row.revision)) {
|
||||
throw new Error("Memory index revision is missing or invalid");
|
||||
}
|
||||
return row.revision;
|
||||
}
|
||||
|
||||
export class MemoryIndexRevisionConflictError extends Error {
|
||||
override name = "MemoryIndexRevisionConflictError";
|
||||
}
|
||||
|
||||
/** Reset derived content without replacing the shared agent database or its schema. */
|
||||
export async function resetMemoryDatabase(params: {
|
||||
targetDb: DatabaseSync;
|
||||
|
|
@ -178,153 +144,6 @@ export async function resetMemoryDatabase(params: {
|
|||
}
|
||||
}
|
||||
|
||||
function replaceVirtualTable(params: {
|
||||
db: DatabaseSync;
|
||||
tableName: "memory_index_chunks_fts" | "memory_index_chunks_vec";
|
||||
columns: string;
|
||||
ignoreDropErrorWhenSourceMissing?: boolean;
|
||||
}): void {
|
||||
const { db, tableName, columns } = params;
|
||||
const createSql = readTableSql(db, MEMORY_REINDEX_SCHEMA, tableName);
|
||||
if (!createSql) {
|
||||
try {
|
||||
db.exec(`DROP TABLE IF EXISTS main.${tableName}`);
|
||||
} catch (err) {
|
||||
if (!params.ignoreDropErrorWhenSourceMissing) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
db.exec(`DROP TABLE IF EXISTS main.${tableName}`);
|
||||
db.exec(createSql);
|
||||
db.exec(
|
||||
`INSERT INTO main.${tableName} (${columns}) ` +
|
||||
`SELECT ${columns} FROM ${MEMORY_REINDEX_SCHEMA}.${tableName}`,
|
||||
);
|
||||
}
|
||||
|
||||
function replaceMemoryPathFtsTable(db: DatabaseSync): void {
|
||||
const createSql = readTableSql(db, MEMORY_REINDEX_SCHEMA, MEMORY_INDEX_PATHS_FTS_TABLE);
|
||||
db.exec(`DROP TABLE IF EXISTS main.${MEMORY_INDEX_PATHS_FTS_TABLE}`);
|
||||
if (!createSql) {
|
||||
return;
|
||||
}
|
||||
db.exec(createSql);
|
||||
// Bulk publication already suspends row triggers. Rebuild from the copied
|
||||
// stable source ids so later singleton deletes remain direct rowid lookups.
|
||||
db.exec(
|
||||
`INSERT INTO main.${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source) ` +
|
||||
`SELECT id, path, source FROM main.memory_index_sources`,
|
||||
);
|
||||
}
|
||||
|
||||
/** The native publication owner receives prepared connection and source facts. */
|
||||
type MemoryDatabasePublication = {
|
||||
targetDb: DatabaseSync;
|
||||
sourcePath: string;
|
||||
metaKey: string;
|
||||
expectedRevision: number;
|
||||
onBegin?: () => void;
|
||||
withCommit?: (commit: () => void) => void;
|
||||
vectorIndexComplete?: boolean;
|
||||
};
|
||||
|
||||
/** The admitted connection owns ATTACH, atomic replacement, COMMIT and DETACH. */
|
||||
export function publishMemoryDatabaseTables(params: MemoryDatabasePublication): void {
|
||||
ensureMemoryRecallMetadataSchema(params.targetDb);
|
||||
// Existing pre-provenance databases need this before the publication writes it.
|
||||
ensureMemoryChunkProvenance(params.targetDb);
|
||||
// Admission precedes ATTACH; no shadow attachment or transaction crosses an await.
|
||||
params.targetDb.prepare(`ATTACH DATABASE ? AS ${MEMORY_REINDEX_SCHEMA}`).run(params.sourcePath);
|
||||
try {
|
||||
runSqliteImmediateTransactionSync(
|
||||
params.targetDb,
|
||||
() => {
|
||||
params.onBegin?.();
|
||||
const liveRevision = readMemoryDatabaseRevision(params.targetDb);
|
||||
if (liveRevision !== params.expectedRevision) {
|
||||
throw new MemoryIndexRevisionConflictError(
|
||||
`Memory index changed while full reindex was building ` +
|
||||
`(expected revision ${params.expectedRevision}, found ${liveRevision}); retry the full reindex.`,
|
||||
);
|
||||
}
|
||||
const publishesPathFts = tableExists(
|
||||
params.targetDb,
|
||||
MEMORY_REINDEX_SCHEMA,
|
||||
MEMORY_INDEX_PATHS_FTS_TABLE,
|
||||
);
|
||||
// Bulk source replacement must not fire one FTS5 scan per old row.
|
||||
// Restore the schema-owned triggers only after the derived table is replaced.
|
||||
dropMemoryPathFtsTriggers(params.targetDb);
|
||||
params.targetDb
|
||||
.prepare("DELETE FROM main.memory_index_meta WHERE key = ?")
|
||||
.run(params.metaKey);
|
||||
params.targetDb
|
||||
.prepare(
|
||||
`INSERT INTO main.memory_index_meta (key, value)
|
||||
SELECT key, value FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_meta WHERE key = ?`,
|
||||
)
|
||||
.run(params.metaKey);
|
||||
|
||||
params.targetDb.exec(`
|
||||
DELETE FROM main.memory_index_sources;
|
||||
INSERT INTO main.memory_index_sources (id, path, source, hash, mtime, size)
|
||||
SELECT id, path, source, hash, mtime, size
|
||||
FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_sources;
|
||||
|
||||
DELETE FROM main.memory_index_chunks;
|
||||
INSERT INTO main.memory_index_chunks (
|
||||
id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
|
||||
)
|
||||
SELECT
|
||||
id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
|
||||
FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_chunks;
|
||||
|
||||
DELETE FROM main.${MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE};
|
||||
INSERT INTO main.${MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE} (
|
||||
chunk_id, importance, triggers, project_key
|
||||
)
|
||||
SELECT chunk_id, importance, triggers, project_key
|
||||
FROM ${MEMORY_REINDEX_SCHEMA}.${MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE};
|
||||
|
||||
DELETE FROM main.memory_index_chunk_provenance;
|
||||
INSERT INTO main.memory_index_chunk_provenance (
|
||||
chunk_id, origin_class, session_kind, observed_at, supersedes_key
|
||||
)
|
||||
SELECT chunk_id, origin_class, session_kind, observed_at, supersedes_key
|
||||
FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_chunk_provenance;
|
||||
`);
|
||||
|
||||
replaceVirtualTable({
|
||||
db: params.targetDb,
|
||||
tableName: "memory_index_chunks_fts",
|
||||
columns: "text, id, path, source, model, start_line, end_line",
|
||||
});
|
||||
replaceMemoryPathFtsTable(params.targetDb);
|
||||
if (publishesPathFts) {
|
||||
ensureMemoryPathFtsTriggers(params.targetDb);
|
||||
}
|
||||
replaceVirtualTable({
|
||||
db: params.targetDb,
|
||||
tableName: "memory_index_chunks_vec",
|
||||
columns: "id, embedding",
|
||||
// A vector-disabled connection may not have sqlite-vec loaded and cannot
|
||||
// drop an old virtual table. Missing vector metadata forces a strict
|
||||
// rebuild before that table can be queried again.
|
||||
ignoreDropErrorWhenSourceMissing: true,
|
||||
});
|
||||
if (params.vectorIndexComplete) {
|
||||
markMemoryVectorIndexClean(params.targetDb);
|
||||
}
|
||||
},
|
||||
{ withCommit: params.withCommit },
|
||||
);
|
||||
} finally {
|
||||
params.targetDb.exec(`DETACH DATABASE ${MEMORY_REINDEX_SCHEMA}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove one closed shadow memory database and its journal-mode sidecars. */
|
||||
export async function removeMemoryDatabaseFiles(dbPath: string): Promise<void> {
|
||||
for (const suffix of MEMORY_DATABASE_FILE_SUFFIXES) {
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ import { readSessionResetRecallCutoffMetadata } from "../session-reset-recall-me
|
|||
import type { EmbeddingProvider } from "./embeddings.js";
|
||||
import type { IndexedMemoryChunk } from "./manager-chunk-writer.js";
|
||||
import { prepareMemoryIndexInWorker } from "./manager-cpu-worker-runtime.js";
|
||||
import { readMemoryDatabaseRevision } from "./manager-db.js";
|
||||
import { readMemoryDatabaseRevision } from "./manager-db-kernel.js";
|
||||
import {
|
||||
clearMemoryEmbeddingCacheIdentities,
|
||||
collectMemoryCachedEmbeddings,
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import path from "node:path";
|
||||
import { serialize } from "node:v8";
|
||||
import * as sqliteCapabilities from "openclaw/plugin-sdk/memory-core-host-engine-knn";
|
||||
import { ensureMemoryIndexSchema } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import * as sqliteRuntime from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import * as sqliteWorkerRuntime from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import { useAutoCleanupTempDirTracker } from "openclaw/plugin-sdk/test-env";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { MemoryIndexDatabase } from "./manager-database-context.js";
|
||||
import { readMemoryDatabaseRevision } from "./manager-db.js";
|
||||
import { readMemoryDatabaseRevision } from "./manager-db-kernel.js";
|
||||
import { memoryPublicationBatches } from "./manager-publication-transfer.js";
|
||||
import { openExistingSqliteWorkerBackend } from "./manager-publication.worker.js";
|
||||
import { readMemoryShadowIdentity } from "./manager-shadow-task.js";
|
||||
|
|
@ -100,14 +100,16 @@ describe("bounded memory publication transfer", () => {
|
|||
|
||||
it("opens keyword publication when SQLite extension loading is unavailable", () => {
|
||||
const owner = createOwner();
|
||||
vi.spyOn(sqliteCapabilities, "supportsNodeSqliteExtensionLoading").mockReturnValue(false);
|
||||
const open = sqliteRuntime.openNodeSqliteDatabase;
|
||||
vi.spyOn(sqliteRuntime, "openNodeSqliteDatabase").mockImplementation((location, options) => {
|
||||
if (options?.allowExtension) {
|
||||
throw new Error("SQLite extension loading is unavailable");
|
||||
}
|
||||
return open(location, options);
|
||||
});
|
||||
vi.spyOn(sqliteWorkerRuntime, "supportsNodeSqliteExtensionLoading").mockReturnValue(false);
|
||||
const open = sqliteWorkerRuntime.openNodeSqliteDatabase;
|
||||
vi.spyOn(sqliteWorkerRuntime, "openNodeSqliteDatabase").mockImplementation(
|
||||
(location, options) => {
|
||||
if (options?.allowExtension) {
|
||||
throw new Error("SQLite extension loading is unavailable");
|
||||
}
|
||||
return open(location, options);
|
||||
},
|
||||
);
|
||||
const backend = createBackend(owner);
|
||||
const { chunks, embeddings: _embeddings, ...header } = replacement();
|
||||
backend.execute({
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { supportsNodeSqliteExtensionLoading } from "openclaw/plugin-sdk/memory-core-host-engine-knn";
|
||||
import {
|
||||
assertTransactionUsable,
|
||||
openNodeSqliteDatabase,
|
||||
resolveExistingSqliteFileUri,
|
||||
requestSqliteWorkerOperationAdmission,
|
||||
runSqliteImmediateTransactionSync,
|
||||
supportsNodeSqliteExtensionLoading,
|
||||
type SqliteWorkerBackend,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
} from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import { hasMemorySessionTombstone } from "../memory-session-tombstones.js";
|
||||
import { publishMemoryDatabaseTables, readMemoryDatabaseRevision } from "./manager-db.js";
|
||||
import { publishMemoryDatabaseTables, readMemoryDatabaseRevision } from "./manager-db-kernel.js";
|
||||
import type {
|
||||
MemoryPublicationConnection,
|
||||
MemoryPublicationOperations,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
// Memory Core owns detached search-time index maintenance lifecycle.
|
||||
import { toErrorObject } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { MemoryIndexRevisionConflictError } from "./manager-db.js";
|
||||
import { MemoryIndexRevisionConflictError } from "./manager-db-kernel.js";
|
||||
|
||||
type MemorySearchMaintenanceManager<DirtyGeneration> = {
|
||||
adoptReindexRetryState(generation: DirtyGeneration): void;
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import { forgetMemoryEntries } from "../memory-forget.js";
|
|||
import type { EmbeddingProvider } from "./embeddings.js";
|
||||
import { memoryCpuProcessEntrypoints } from "./manager-cpu-entrypoints.js";
|
||||
import * as memoryCpuWorkerRuntime from "./manager-cpu-worker-runtime.js";
|
||||
import { MemoryIndexRevisionConflictError } from "./manager-db.js";
|
||||
import { MemoryIndexRevisionConflictError } from "./manager-db-kernel.js";
|
||||
import { createManagerIndexFixture } from "./manager-index.test-support.js";
|
||||
|
||||
const { closeAllMemorySearchManagers, getMemorySearchManager } = await import("./index.js");
|
||||
|
|
|
|||
|
|
@ -1,16 +1,15 @@
|
|||
import type { DatabaseSync, StatementSync } from "node:sqlite";
|
||||
import { hashText, type MemorySource } from "openclaw/plugin-sdk/memory-core-host-engine-indexing";
|
||||
import {
|
||||
hashText,
|
||||
MEMORY_INDEX_FTS_TABLE,
|
||||
MEMORY_INDEX_VECTOR_TABLE,
|
||||
type MemorySource,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-schema";
|
||||
import {
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
getNodeSqliteKysely,
|
||||
runSqliteImmediateTransactionSync,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
} from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import { createMemoryChunkWriter, type IndexedMemoryChunk } from "./manager-chunk-writer.js";
|
||||
import {
|
||||
markMemoryVectorRebuildRequired,
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import {
|
|||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { runSqliteImmediateTransaction } from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import { withMemoryWorkspaceLock } from "../memory-workspace-lock.js";
|
||||
import { MemoryIndexRevisionConflictError } from "./manager-db.js";
|
||||
import { MemoryIndexRevisionConflictError } from "./manager-db-kernel.js";
|
||||
import type { MemoryIndexEntry } from "./manager-index-preparation.js";
|
||||
import { MemoryManagerSessionSyncOps } from "./manager-session-sync-ops.js";
|
||||
import {
|
||||
|
|
|
|||
|
|
@ -21,12 +21,8 @@ import {
|
|||
type EmbeddingProviderRuntime,
|
||||
} from "./embeddings.js";
|
||||
import { MemoryIndexDatabase } from "./manager-database-context.js";
|
||||
import {
|
||||
cleanupAgedMemoryReindexTempFiles,
|
||||
memoryDatabaseTableExists,
|
||||
readMemoryDatabaseRevision,
|
||||
removeMemoryDatabaseFiles,
|
||||
} from "./manager-db.js";
|
||||
import { memoryDatabaseTableExists, readMemoryDatabaseRevision } from "./manager-db-kernel.js";
|
||||
import { cleanupAgedMemoryReindexTempFiles, removeMemoryDatabaseFiles } from "./manager-db.js";
|
||||
import { isMemoryEmbeddingOperationError } from "./manager-embedding-errors.js";
|
||||
import { withMemoryIndexPublishGeneration } from "./manager-index-generation-lease.js";
|
||||
import {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
// Memory Core plugin module owns persisted vector completeness state.
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import {
|
||||
MEMORY_INDEX_META_TABLE,
|
||||
type MemoryVectorIndexState,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { MEMORY_INDEX_META_TABLE } from "openclaw/plugin-sdk/memory-core-host-engine-schema";
|
||||
import type { MemoryVectorIndexState } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
const VECTOR_REBUILD_META_KEY = "memory_vector_rebuild_v1";
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import * as sqliteRuntime from "openclaw/plugin-sdk/sqlite-runtime";
|
|||
import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { recordMemorySessionTombstones } from "../memory-entry-origins.js";
|
||||
import { MemoryIndexRevisionConflictError } from "./manager-db.js";
|
||||
import { MemoryIndexRevisionConflictError } from "./manager-db-kernel.js";
|
||||
import { createManagerIndexFixture } from "./manager-index.test-support.js";
|
||||
import { closeAllMemoryIndexManagers } from "./manager-runtime.js";
|
||||
import { MemoryIndexManager } from "./manager.js";
|
||||
|
|
|
|||
|
|
@ -19,16 +19,13 @@ import {
|
|||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { normalizeAgentId } from "openclaw/plugin-sdk/routing";
|
||||
import { createPluginRuntimeStore } from "openclaw/plugin-sdk/runtime-store";
|
||||
import {
|
||||
borrowOpenClawAgentDatabase,
|
||||
withOpenClawAgentDatabaseWrite,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import { withOpenClawAgentDatabaseWrite } from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import { runInMemoryBackgroundContext } from "./background-context.js";
|
||||
import type { MemoryCoreAcquireLocalService } from "./embedding-local-service.js";
|
||||
import type { EmbeddingProvider, EmbeddingProviderRequest } from "./embeddings.js";
|
||||
import { getMemoryManagerLifecycle } from "./lifecycle.js";
|
||||
import { MemoryIndexDatabase } from "./manager-database-context.js";
|
||||
import { memoryDatabaseTableExists, openMemoryDatabaseReadOnlyAtPath } from "./manager-db.js";
|
||||
import { memoryDatabaseTableExists } from "./manager-db-kernel.js";
|
||||
import {
|
||||
resolveEffectiveMemorySearchSettings,
|
||||
resolveMemoryEmbeddingProviderRequirement,
|
||||
|
|
@ -249,24 +246,17 @@ export class MemoryIndexManager extends MemorySearchOrchestration implements Mem
|
|||
for (const memorySource of effectiveSettings.sources) {
|
||||
this.sources.add(memorySource);
|
||||
}
|
||||
const vectorEnabled = effectiveSettings.store.vector.enabled;
|
||||
const readOnly = this.purpose === "status";
|
||||
if (source && (!source.publishedDatabase.db.isOpen || this.purpose !== "maintenance")) {
|
||||
throw new Error("Memory maintenance source connection is unavailable");
|
||||
}
|
||||
const connection = readOnly
|
||||
? openMemoryDatabaseReadOnlyAtPath(dbPath, vectorEnabled, this.agentId)
|
||||
: borrowOpenClawAgentDatabase(params.databaseOptions);
|
||||
if (source && connection.db !== source.publishedDatabase.db) {
|
||||
connection.release();
|
||||
throw new Error("Memory maintenance source connection changed");
|
||||
}
|
||||
this.publishedDatabase = new MemoryIndexDatabase(
|
||||
connection.db,
|
||||
connection.release,
|
||||
this.publishedDatabase = MemoryIndexDatabase.openPublished({
|
||||
agentId: this.agentId,
|
||||
writeOptions: params.databaseOptions,
|
||||
readOnly,
|
||||
params.databaseOptions,
|
||||
);
|
||||
allowExtension: effectiveSettings.store.vector.enabled,
|
||||
maintenanceSource: source?.publishedDatabase,
|
||||
});
|
||||
try {
|
||||
this.providerKey = this.computeProviderKey();
|
||||
this.cache = {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { openOpenClawAgentDatabase } from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { readMemoryDatabaseRevision } from "./memory/manager-db.js";
|
||||
import { readMemoryDatabaseRevision } from "./memory/manager-db-kernel.js";
|
||||
import { createManagerIndexFixture } from "./memory/manager-index.test-support.js";
|
||||
import { MEMORY_INDEX_PROVENANCE_VERSION } from "./memory/manager-reindex-state.js";
|
||||
import { createMemorySearchTool, testing } from "./tools.js";
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { createDeferred } from "openclaw/plugin-sdk/extension-shared";
|
|||
import { MEMORY_CHUNKING_VERSION } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { openOpenClawAgentDatabase } from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { readMemoryDatabaseRevision } from "./memory/manager-db.js";
|
||||
import { readMemoryDatabaseRevision } from "./memory/manager-db-kernel.js";
|
||||
import { createManagerIndexFixture } from "./memory/manager-index.test-support.js";
|
||||
import { MEMORY_INDEX_PROVENANCE_VERSION } from "./memory/manager-reindex-state.js";
|
||||
import { createMemorySearchTool, testing } from "./tools.js";
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import { openOpenClawAgentDatabase } from "openclaw/plugin-sdk/sqlite-runtime";
|
|||
import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { EmbeddingProvider } from "./memory/embeddings.js";
|
||||
import { readMemoryDatabaseRevision } from "./memory/manager-db.js";
|
||||
import { readMemoryDatabaseRevision } from "./memory/manager-db-kernel.js";
|
||||
import * as generationLease from "./memory/manager-index-generation-lease.js";
|
||||
import {
|
||||
createManagerIndexFixture,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import {
|
|||
openNodeSqliteDatabase,
|
||||
tableExists,
|
||||
type SqliteWorkerBackend,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
} from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import type { QaExecutionIdentityStorageOperations } from "./execution-identity-storage-inspection.js";
|
||||
|
||||
type QaExecutionIdentityDatabase = {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import {
|
|||
openNodeSqliteDatabase,
|
||||
runSqliteImmediateTransactionSync,
|
||||
type SqliteWorkerCommand,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
} from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import { MAX_REPORT_BYTES } from "./limits.js";
|
||||
import { DAY_MS } from "./periods.js";
|
||||
import type {
|
||||
|
|
|
|||
|
|
@ -933,6 +933,9 @@
|
|||
],
|
||||
"openclaw/plugin-sdk/memory-core-host-engine-indexing": [
|
||||
"../packages/plugin-sdk/dist/src/plugin-sdk/memory-core-host-engine-indexing.d.ts"
|
||||
],
|
||||
"openclaw/plugin-sdk/sqlite-worker-runtime": [
|
||||
"../packages/plugin-sdk/dist/src/plugin-sdk/sqlite-worker-runtime.d.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ const workerModuleUrl = resolveRuntimeWorkerUrl(workboardSqliteBackendEntrypoint
|
|||
|
||||
const sqliteStatements = vi.hoisted(() => ({ count: 0 }));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/sqlite-runtime", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("openclaw/plugin-sdk/sqlite-runtime")>();
|
||||
vi.mock("openclaw/plugin-sdk/sqlite-worker-runtime", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("openclaw/plugin-sdk/sqlite-worker-runtime")>();
|
||||
return {
|
||||
...actual,
|
||||
openNodeSqliteDatabase: (...args: Parameters<typeof actual.openNodeSqliteDatabase>) => {
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import {
|
|||
iterateSqliteQuerySync,
|
||||
runSqliteImmediateTransactionSync,
|
||||
sqliteStringSet,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
} from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type {
|
||||
PersistedWorkboardAttachment,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ const { close, configureSqliteConnectionPragmas } = vi.hoisted(() => ({
|
|||
configureSqliteConnectionPragmas: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/sqlite-runtime", () => ({
|
||||
vi.mock("openclaw/plugin-sdk/sqlite-worker-runtime", () => ({
|
||||
openNodeSqliteDatabase: vi.fn(() => ({ close })),
|
||||
}));
|
||||
vi.mock("openclaw/plugin-sdk/plugin-state-runtime", () => ({
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import {
|
|||
getNodeSqliteKysely,
|
||||
iterateSqliteQuerySync,
|
||||
sqliteStringSet,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
} from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
export type Row = Record<string, unknown>;
|
||||
|
||||
export function jsonValue(value: unknown): string | null {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import {
|
|||
configureSqliteConnectionPragmas,
|
||||
migrateSqliteSchemaToStrict,
|
||||
} from "openclaw/plugin-sdk/plugin-state-runtime";
|
||||
import { openNodeSqliteDatabase } from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import { openNodeSqliteDatabase } from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
const SCHEMA_VERSION = 3;
|
||||
const WORKBOARD_SQLITE_BUSY_TIMEOUT_MS = 5000;
|
||||
const WORKBOARD_SQLITE_DIR_MODE = 0o700;
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import type { WorkboardCard } from "@openclaw/workboard-contract";
|
|||
import {
|
||||
compileSqliteQueryBindings,
|
||||
getNodeSqliteKysely,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
} from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import {
|
||||
jsonValue,
|
||||
type CARD_CHILD_TABLES,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
import type { SqliteWorkerBackend, SqliteWorkerCommand } from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import type {
|
||||
SqliteWorkerBackend,
|
||||
SqliteWorkerCommand,
|
||||
} from "openclaw/plugin-sdk/sqlite-worker-runtime";
|
||||
import type {
|
||||
WorkboardSqliteOperations,
|
||||
WorkboardSqliteWorkerOperations,
|
||||
|
|
|
|||
|
|
@ -917,6 +917,9 @@
|
|||
],
|
||||
"openclaw/plugin-sdk/memory-core-host-engine-indexing": [
|
||||
"../../packages/plugin-sdk/dist/src/plugin-sdk/memory-core-host-engine-indexing.d.ts"
|
||||
],
|
||||
"openclaw/plugin-sdk/sqlite-worker-runtime": [
|
||||
"../../packages/plugin-sdk/dist/src/plugin-sdk/sqlite-worker-runtime.d.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -423,7 +423,8 @@
|
|||
"!dist/plugin-sdk/provider-auth-aliases.d.ts",
|
||||
"!dist/plugin-sdk/provider-model-metadata.d.ts",
|
||||
"!dist/plugin-sdk/memory-core-host-engine-knn.d.ts",
|
||||
"!dist/plugin-sdk/memory-core-host-engine-indexing.d.ts"
|
||||
"!dist/plugin-sdk/memory-core-host-engine-indexing.d.ts",
|
||||
"!dist/plugin-sdk/sqlite-worker-runtime.d.ts"
|
||||
],
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
|
|
@ -1198,6 +1199,9 @@
|
|||
"./plugin-sdk/sqlite-runtime": {
|
||||
"default": "./dist/plugin-sdk/sqlite-runtime.js"
|
||||
},
|
||||
"./plugin-sdk/sqlite-worker-runtime": {
|
||||
"default": "./dist/plugin-sdk/sqlite-worker-runtime.js"
|
||||
},
|
||||
"./plugin-sdk/session-transcript-hit": {
|
||||
"default": "./dist/plugin-sdk/session-transcript-hit.js"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -238,6 +238,7 @@
|
|||
"session-transcript-runtime",
|
||||
"sqlite-runtime",
|
||||
"sqlite-runtime-testing",
|
||||
"sqlite-worker-runtime",
|
||||
"session-transcript-hit",
|
||||
"session-visibility",
|
||||
"ssrf-dispatcher",
|
||||
|
|
|
|||
|
|
@ -181,6 +181,7 @@
|
|||
"speech-provider",
|
||||
"sqlite-runtime",
|
||||
"sqlite-runtime-testing",
|
||||
"sqlite-worker-runtime",
|
||||
"ssrf-dispatcher",
|
||||
"ssrf-runtime-internal",
|
||||
"string-normalization-runtime",
|
||||
|
|
|
|||
|
|
@ -1,10 +1,16 @@
|
|||
// Focused memory host schema helpers for doctor and migration control-plane paths.
|
||||
// Memory schema operations shared by host maintenance and native publication workers.
|
||||
export {
|
||||
dropMemoryPathFtsTriggers,
|
||||
ensureMemoryChunkProvenance,
|
||||
ensureMemoryIndexSchema,
|
||||
ensureMemoryPathFtsTriggers,
|
||||
ensureMemoryRecallMetadataSchema,
|
||||
MEMORY_EMBEDDING_CACHE_TABLE,
|
||||
MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE,
|
||||
MEMORY_INDEX_CHUNKS_TABLE,
|
||||
MEMORY_INDEX_FTS_TABLE,
|
||||
MEMORY_INDEX_META_TABLE,
|
||||
MEMORY_INDEX_PATHS_FTS_TABLE,
|
||||
MEMORY_INDEX_SOURCES_TABLE,
|
||||
MEMORY_INDEX_VECTOR_TABLE,
|
||||
} from "../../packages/memory-host-sdk/src/host/memory-schema.js";
|
||||
|
|
|
|||
28
src/plugin-sdk/sqlite-worker-runtime.ts
Normal file
28
src/plugin-sdk/sqlite-worker-runtime.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
// SQLite backends use native primitives without loading host database lifecycle owners.
|
||||
export type { Generated, Selectable } from "kysely";
|
||||
export {
|
||||
compileSqliteQueryBindings,
|
||||
enableNodeSqliteKyselyStatementCache,
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
getNodeSqliteKysely,
|
||||
iterateSqliteQuerySync,
|
||||
prepareSqliteQuerySync,
|
||||
sqliteStringSet,
|
||||
} from "../infra/kysely-sync.js";
|
||||
export {
|
||||
openNodeSqliteDatabase,
|
||||
resolveExistingSqliteFileUri,
|
||||
supportsNodeSqliteExtensionLoading,
|
||||
} from "../infra/node-sqlite.js";
|
||||
export {
|
||||
assertTransactionUsable,
|
||||
runSqliteImmediateTransactionSync,
|
||||
} from "../infra/sqlite-transaction.js";
|
||||
export type {
|
||||
SqliteWorkerBackend,
|
||||
SqliteWorkerCommand,
|
||||
SqliteWorkerOperations,
|
||||
} from "../infra/sqlite-worker-contract.js";
|
||||
export { requestSqliteWorkerOperationAdmission } from "../infra/sqlite-worker-operation-admission.js";
|
||||
export { tableExists } from "../state/openclaw-state-db-schema-helpers.js";
|
||||
Loading…
Add table
Add a link
Reference in a new issue