mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
perf(memory): add QMD search diagnostics and runtime cache (#96655)
This commit is contained in:
Bek
GitHub
parent
1089253ca9
commit
9636bea901
15 changed files with 1927 additions and 24 deletions
|
|
@ -1,2 +1,2 @@
|
|||
e9fb501204b6c4c0e08c09174311f85ae8a129bf35e18edea0fa217ee4203ad8 plugin-sdk-api-baseline.json
|
||||
50db19cf60c5465b22d342ccdabe465ae08d0c475e6403b86360f775bfc3513a plugin-sdk-api-baseline.jsonl
|
||||
760812c17f7e48d7ceafeebbbe348dad13916ccb9ecaf41b3abc9a09b1e690c1 plugin-sdk-api-baseline.json
|
||||
4d9b76016b2f845e101949a3d2ac92437f49783906d1c263d65f3534bb333de5 plugin-sdk-api-baseline.jsonl
|
||||
|
|
|
|||
|
|
@ -199,11 +199,17 @@ vi.mock("openclaw/plugin-sdk/file-lock", async () => {
|
|||
import { spawn as mockedSpawn } from "node:child_process";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import {
|
||||
type MemorySearchRuntimeDebug,
|
||||
requireNodeSqlite,
|
||||
resolveMemoryBackendConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { formatSessionTranscriptMemoryHitKey } from "openclaw/plugin-sdk/session-transcript-hit";
|
||||
import {
|
||||
configureMemoryCoreDreamingState,
|
||||
configureMemoryCoreDreamingStateForTests,
|
||||
resetMemoryCoreDreamingStateForTests,
|
||||
} from "../dreaming-state.js";
|
||||
import { resolveQmdSessionArtifactIdentity } from "../qmd-session-artifacts.js";
|
||||
import { QmdMemoryManager, resolveQmdMcporterSearchProcessTimeoutMs } from "./qmd-manager.js";
|
||||
|
||||
|
|
@ -270,6 +276,14 @@ describe("QmdMemoryManager", () => {
|
|||
return mock.mock.calls.map((call: unknown[]) => String(call[0]));
|
||||
}
|
||||
|
||||
function qmdCommandCalls(): string[][] {
|
||||
return spawnMock.mock.calls.map((call: unknown[]) => call[1] as string[]);
|
||||
}
|
||||
|
||||
function countQmdCommand(predicate: (args: string[]) => boolean): number {
|
||||
return qmdCommandCalls().filter(predicate).length;
|
||||
}
|
||||
|
||||
function expectMockMessageContains(mock: Mock, text: string): void {
|
||||
expect(mockMessages(mock).join("\n")).toContain(text);
|
||||
}
|
||||
|
|
@ -290,6 +304,387 @@ describe("QmdMemoryManager", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it("reuses persisted collection validation across transient cli managers", async () => {
|
||||
await configureMemoryCoreDreamingStateForTests();
|
||||
const first = await createManager({ mode: "cli" });
|
||||
await first.manager.close();
|
||||
expect(countQmdCommand((args) => args[0] === "collection" && args[1] === "list")).toBe(1);
|
||||
|
||||
spawnMock.mockClear();
|
||||
const second = await createManager({ mode: "cli" });
|
||||
await second.manager.close();
|
||||
|
||||
expect(countQmdCommand((args) => args[0] === "collection" && args[1] === "list")).toBe(0);
|
||||
expect(countQmdCommand((args) => args[0] === "collection" && args[1] === "show")).toBe(0);
|
||||
expect(countQmdCommand((args) => args[0] === "collection" && args[1] === "add")).toBe(0);
|
||||
});
|
||||
|
||||
it("does not cache incomplete collection validation", async () => {
|
||||
await configureMemoryCoreDreamingStateForTests();
|
||||
spawnMock.mockImplementation((_cmd: string, args: string[]) => {
|
||||
if (args[0] === "collection" && args[1] === "add") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(child, "stderr", "permission denied", 1);
|
||||
return child;
|
||||
}
|
||||
return createMockChild();
|
||||
});
|
||||
|
||||
const first = await createManager({ mode: "cli" });
|
||||
await first.manager.close();
|
||||
|
||||
spawnMock.mockClear();
|
||||
spawnMock.mockImplementation(() => createMockChild());
|
||||
const second = await createManager({ mode: "cli" });
|
||||
await second.manager.close();
|
||||
|
||||
expect(countQmdCommand((args) => args[0] === "collection" && args[1] === "list")).toBe(1);
|
||||
expect(countQmdCommand((args) => args[0] === "collection" && args[1] === "add")).toBe(1);
|
||||
});
|
||||
|
||||
it("runs collection validation when the runtime cache store is unavailable", async () => {
|
||||
configureMemoryCoreDreamingState(() => {
|
||||
throw new Error("state store unavailable");
|
||||
});
|
||||
try {
|
||||
const manager = await createManager({ mode: "cli" });
|
||||
await manager.manager.close();
|
||||
} finally {
|
||||
await configureMemoryCoreDreamingStateForTests();
|
||||
}
|
||||
|
||||
expect(countQmdCommand((args) => args[0] === "collection" && args[1] === "list")).toBe(1);
|
||||
expect(countQmdCommand((args) => args[0] === "collection" && args[1] === "add")).toBe(1);
|
||||
});
|
||||
|
||||
it("reports collection validation debug only once per validation run", async () => {
|
||||
await configureMemoryCoreDreamingStateForTests();
|
||||
spawnMock.mockImplementation((_cmd: string, args: string[]) => {
|
||||
if (args[0] === "query" || args[0] === "search" || args[0] === "vsearch") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(child, "stdout", "[]");
|
||||
return child;
|
||||
}
|
||||
return createMockChild();
|
||||
});
|
||||
const { manager } = await createManager({ mode: "cli" });
|
||||
const firstDebug: MemorySearchRuntimeDebug[] = [];
|
||||
const secondDebug: MemorySearchRuntimeDebug[] = [];
|
||||
|
||||
await manager.search("fact", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
onDebug: (entry) => {
|
||||
firstDebug.push(entry);
|
||||
},
|
||||
});
|
||||
await manager.search("fact again", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
onDebug: (entry) => {
|
||||
secondDebug.push(entry);
|
||||
},
|
||||
});
|
||||
|
||||
expect(firstDebug.at(-1)?.qmd?.collectionValidation?.cacheState).toBe("write");
|
||||
expect(secondDebug.at(-1)?.qmd?.collectionValidation).toBeUndefined();
|
||||
});
|
||||
|
||||
it("misses collection validation cache when managed collection config changes", async () => {
|
||||
await configureMemoryCoreDreamingStateForTests();
|
||||
const first = await createManager({ mode: "cli" });
|
||||
await first.manager.close();
|
||||
|
||||
const otherWorkspaceDir = path.join(tmpRoot, "other-workspace");
|
||||
await fs.mkdir(otherWorkspaceDir, { recursive: true });
|
||||
const changedCfg = {
|
||||
...cfg,
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
...cfg.memory?.qmd,
|
||||
paths: [{ path: otherWorkspaceDir, pattern: "**/*.md", name: "workspace" }],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
spawnMock.mockClear();
|
||||
const second = await createManager({ mode: "cli", cfg: changedCfg });
|
||||
await second.manager.close();
|
||||
|
||||
expect(countQmdCommand((args) => args[0] === "collection" && args[1] === "list")).toBe(1);
|
||||
});
|
||||
|
||||
it("bypasses validation cache for missing-collection search repair", async () => {
|
||||
await configureMemoryCoreDreamingStateForTests();
|
||||
const { manager } = await createManager();
|
||||
spawnMock.mockClear();
|
||||
let searchAttempts = 0;
|
||||
spawnMock.mockImplementation((_cmd: string, args: string[]) => {
|
||||
if (args[0] === "query" || args[0] === "search" || args[0] === "vsearch") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
searchAttempts += 1;
|
||||
if (searchAttempts === 1) {
|
||||
emitAndClose(child, "stderr", "collection workspace-main not found", 1);
|
||||
} else {
|
||||
emitAndClose(child, "stdout", "[]");
|
||||
}
|
||||
return child;
|
||||
}
|
||||
return createMockChild();
|
||||
});
|
||||
const debug: MemorySearchRuntimeDebug[] = [];
|
||||
|
||||
await manager.search("fact", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
onDebug: (entry) => {
|
||||
debug.push(entry);
|
||||
},
|
||||
});
|
||||
|
||||
expect(searchAttempts).toBe(2);
|
||||
expect(countQmdCommand((args) => args[0] === "collection" && args[1] === "list")).toBe(1);
|
||||
expect(debug.at(-1)?.qmd?.collectionValidation?.cacheState).toBe("bypass-force");
|
||||
});
|
||||
|
||||
it("reuses persisted qmd multi-collection support probe across managers", async () => {
|
||||
await configureMemoryCoreDreamingStateForTests();
|
||||
cfg = {
|
||||
...cfg,
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: false,
|
||||
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||
sessions: { enabled: true },
|
||||
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
spawnMock.mockImplementation((_cmd: string, args: string[]) => {
|
||||
if (args[0] === "--help") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(child, "stdout", "Usage: qmd search -c one or more collections");
|
||||
return child;
|
||||
}
|
||||
if (args[0] === "search") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(child, "stdout", "[]");
|
||||
return child;
|
||||
}
|
||||
return createMockChild();
|
||||
});
|
||||
|
||||
const first = await createManager({ mode: "cli" });
|
||||
await first.manager.search("fact", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
});
|
||||
await first.manager.close();
|
||||
expect(countQmdCommand((args) => args[0] === "--help")).toBe(1);
|
||||
|
||||
spawnMock.mockClear();
|
||||
const second = await createManager({ mode: "cli" });
|
||||
const debug: MemorySearchRuntimeDebug[] = [];
|
||||
await second.manager.search("fact", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
onDebug: (entry) => {
|
||||
debug.push(entry);
|
||||
},
|
||||
});
|
||||
await second.manager.close();
|
||||
|
||||
expect(countQmdCommand((args) => args[0] === "--help")).toBe(0);
|
||||
expect(debug.at(-1)?.qmd?.multiCollectionProbe?.cacheState).toBe("hit");
|
||||
expect(debug.at(-1)?.qmd?.searchPlan?.groupCount).toBe(2);
|
||||
});
|
||||
|
||||
it("reports multi-collection probe debug only when the probe runs", async () => {
|
||||
await configureMemoryCoreDreamingStateForTests();
|
||||
cfg = {
|
||||
...cfg,
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: false,
|
||||
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||
sessions: { enabled: true },
|
||||
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
spawnMock.mockImplementation((_cmd: string, args: string[]) => {
|
||||
if (args[0] === "--help") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(child, "stdout", "Usage: qmd search -c one or more collections");
|
||||
return child;
|
||||
}
|
||||
if (args[0] === "search") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(child, "stdout", "[]");
|
||||
return child;
|
||||
}
|
||||
return createMockChild();
|
||||
});
|
||||
const { manager } = await createManager({ mode: "cli" });
|
||||
const firstDebug: MemorySearchRuntimeDebug[] = [];
|
||||
const secondDebug: MemorySearchRuntimeDebug[] = [];
|
||||
|
||||
await manager.search("fact", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
onDebug: (entry) => {
|
||||
firstDebug.push(entry);
|
||||
},
|
||||
});
|
||||
await manager.search("fact again", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
onDebug: (entry) => {
|
||||
secondDebug.push(entry);
|
||||
},
|
||||
});
|
||||
|
||||
expect(firstDebug.at(-1)?.qmd?.multiCollectionProbe?.cacheState).toBe("write");
|
||||
expect(secondDebug.at(-1)?.qmd?.multiCollectionProbe).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps concurrent search debug isolated on a shared qmd manager", async () => {
|
||||
await configureMemoryCoreDreamingStateForTests();
|
||||
cfg = {
|
||||
...cfg,
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: false,
|
||||
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||
sessions: { enabled: true },
|
||||
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
let firstSearchChild: MockChild | undefined;
|
||||
let searchCalls = 0;
|
||||
spawnMock.mockImplementation((_cmd: string, args: string[]) => {
|
||||
if (args[0] === "search") {
|
||||
searchCalls += 1;
|
||||
const child = createMockChild({ autoClose: false });
|
||||
if (searchCalls === 1) {
|
||||
firstSearchChild = child;
|
||||
return child;
|
||||
}
|
||||
emitAndClose(child, "stdout", "[]");
|
||||
return child;
|
||||
}
|
||||
if (args[0] === "--version") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(child, "stdout", "qmd 1.0.0");
|
||||
return child;
|
||||
}
|
||||
return createMockChild();
|
||||
});
|
||||
const { manager } = await createManager({ mode: "full" });
|
||||
const firstDebug: MemorySearchRuntimeDebug[] = [];
|
||||
const secondDebug: MemorySearchRuntimeDebug[] = [];
|
||||
|
||||
const firstSearch = manager.search("memory fact", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
sources: ["memory"],
|
||||
onDebug: (entry) => {
|
||||
firstDebug.push(entry);
|
||||
},
|
||||
});
|
||||
await waitUntil(() => searchCalls === 1);
|
||||
const secondSearch = manager.search("session fact", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
sources: ["sessions"],
|
||||
onDebug: (entry) => {
|
||||
secondDebug.push(entry);
|
||||
},
|
||||
});
|
||||
await waitUntil(() => searchCalls === 2);
|
||||
emitAndClose(requireValue(firstSearchChild, "first search child missing"), "stdout", "[]");
|
||||
|
||||
await Promise.all([firstSearch, secondSearch]);
|
||||
|
||||
expect(firstDebug.at(-1)?.qmd?.searchPlan?.sources).toEqual(["memory"]);
|
||||
expect(secondDebug.at(-1)?.qmd?.searchPlan?.sources).toEqual(["sessions"]);
|
||||
});
|
||||
|
||||
it("rewrites stale multi-collection probe cache when combined filters are rejected", async () => {
|
||||
await configureMemoryCoreDreamingStateForTests();
|
||||
const otherWorkspaceDir = path.join(tmpRoot, "other-workspace");
|
||||
await fs.mkdir(otherWorkspaceDir, { recursive: true });
|
||||
cfg = {
|
||||
...cfg,
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: false,
|
||||
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||
paths: [
|
||||
{ path: workspaceDir, pattern: "**/*.md", name: "workspace" },
|
||||
{ path: otherWorkspaceDir, pattern: "**/*.md", name: "other" },
|
||||
],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const isCombinedSearch = (args: string[]) =>
|
||||
(args[0] === "search" || args[0] === "query") &&
|
||||
args.filter((token) => token === "-c").length > 1;
|
||||
spawnMock.mockImplementation((_cmd: string, args: string[]) => {
|
||||
if (args[0] === "--version") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(child, "stdout", "qmd 1.0.0");
|
||||
return child;
|
||||
}
|
||||
if (args[0] === "--help") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(child, "stdout", "Usage: qmd search -c one or more collections");
|
||||
return child;
|
||||
}
|
||||
if (isCombinedSearch(args)) {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(child, "stderr", "unknown flag: -c", 1);
|
||||
return child;
|
||||
}
|
||||
if (args[0] === "search" || args[0] === "query" || args[0] === "vsearch") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(child, "stdout", "[]");
|
||||
return child;
|
||||
}
|
||||
return createMockChild();
|
||||
});
|
||||
|
||||
const first = await createManager({ mode: "cli" });
|
||||
const firstDebug: MemorySearchRuntimeDebug[] = [];
|
||||
await first.manager.search("fact", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
onDebug: (entry) => {
|
||||
firstDebug.push(entry);
|
||||
},
|
||||
});
|
||||
await first.manager.close();
|
||||
|
||||
expect(firstDebug.at(-1)?.qmd?.multiCollectionProbe).toMatchObject({
|
||||
cacheState: "write",
|
||||
supported: false,
|
||||
});
|
||||
|
||||
spawnMock.mockClear();
|
||||
const second = await createManager({ mode: "cli" });
|
||||
const secondDebug: MemorySearchRuntimeDebug[] = [];
|
||||
await second.manager.search("fact", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
onDebug: (entry) => {
|
||||
secondDebug.push(entry);
|
||||
},
|
||||
});
|
||||
await second.manager.close();
|
||||
|
||||
expect(countQmdCommand((args) => args[0] === "--help")).toBe(0);
|
||||
expect(countQmdCommand(isCombinedSearch)).toBe(0);
|
||||
expect(secondDebug.at(-1)?.qmd?.multiCollectionProbe).toMatchObject({
|
||||
cacheState: "hit",
|
||||
supported: false,
|
||||
});
|
||||
});
|
||||
|
||||
async function expectPathMissing(targetPath: string): Promise<void> {
|
||||
try {
|
||||
await fs.lstat(targetPath);
|
||||
|
|
@ -419,6 +814,7 @@ describe("QmdMemoryManager", () => {
|
|||
delete (globalThis as Record<PropertyKey, unknown>)[MCPORTER_STATE_KEY];
|
||||
delete (globalThis as Record<PropertyKey, unknown>)[QMD_EMBED_QUEUE_KEY];
|
||||
delete (globalThis as Record<PropertyKey, unknown>)[MEMORY_EMBEDDING_PROVIDERS_KEY];
|
||||
resetMemoryCoreDreamingStateForTests();
|
||||
});
|
||||
|
||||
it("debounces back-to-back sync calls", async () => {
|
||||
|
|
|
|||
|
|
@ -74,6 +74,16 @@ import {
|
|||
type QmdSessionArtifactMapping,
|
||||
} from "../qmd-session-artifacts.js";
|
||||
import { resolveQmdCollectionPatternFlags, type QmdCollectionPatternFlag } from "./qmd-compat.js";
|
||||
import {
|
||||
clearQmdMultiCollectionProbeCache,
|
||||
readQmdCollectionValidationCache,
|
||||
readQmdMultiCollectionProbeCache,
|
||||
writeQmdCollectionValidationCache,
|
||||
writeQmdMultiCollectionProbeCache,
|
||||
type QmdRuntimeCollectionValidationCacheContext,
|
||||
type QmdRuntimeManagedCollection,
|
||||
type QmdRuntimeMultiCollectionProbeCacheContext,
|
||||
} from "./qmd-runtime-cache.js";
|
||||
import {
|
||||
countChokidarWatchedEntries,
|
||||
type MemoryWatchPressureWarningState,
|
||||
|
|
@ -324,6 +334,19 @@ type ManagedCollection = {
|
|||
kind: "memory" | "custom" | "sessions";
|
||||
};
|
||||
|
||||
type QmdCollectionValidationDebug = NonNullable<
|
||||
NonNullable<MemorySearchRuntimeDebug["qmd"]>["collectionValidation"]
|
||||
>;
|
||||
type QmdMultiCollectionProbeDebug = NonNullable<
|
||||
NonNullable<MemorySearchRuntimeDebug["qmd"]>["multiCollectionProbe"]
|
||||
>;
|
||||
type QmdSearchPlanDebug = NonNullable<NonNullable<MemorySearchRuntimeDebug["qmd"]>["searchPlan"]>;
|
||||
type QmdSearchRuntimeDebugContext = {
|
||||
collectionValidation?: QmdCollectionValidationDebug;
|
||||
multiCollectionProbe?: QmdMultiCollectionProbeDebug;
|
||||
searchPlan?: QmdSearchPlanDebug;
|
||||
};
|
||||
|
||||
type QmdManagerMode = "full" | "status" | "cli";
|
||||
type QmdManagerRuntimeConfig = {
|
||||
workspaceDir: string;
|
||||
|
|
@ -441,6 +464,7 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
private mode: QmdManagerMode = "full";
|
||||
private readonly closeSignal: Promise<void>;
|
||||
private resolveCloseSignal!: () => void;
|
||||
private qmdRuntimeIdentityPromise: Promise<string> | null = null;
|
||||
private db: SqliteDatabase | null = null;
|
||||
private lastUpdateAt: number | null = null;
|
||||
private lastEmbedAt: number | null = null;
|
||||
|
|
@ -453,6 +477,7 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
private readonly sessionWarm = new Set<string>();
|
||||
private collectionPatternFlag: QmdCollectionPatternFlag | null = "--mask";
|
||||
private multiCollectionFilterSupported: boolean | null = null;
|
||||
private pendingCollectionValidationDebug: QmdCollectionValidationDebug | undefined;
|
||||
|
||||
private constructor(params: {
|
||||
agentId: string;
|
||||
|
|
@ -612,11 +637,171 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
}
|
||||
}
|
||||
|
||||
private async ensureCollections(): Promise<void> {
|
||||
private qmdRuntimeCacheSources(): string[] {
|
||||
return [...this.sources].toSorted();
|
||||
}
|
||||
|
||||
private qmdRuntimeCacheCollections(): QmdRuntimeManagedCollection[] {
|
||||
return this.qmd.collections.map((collection) => ({
|
||||
name: collection.name,
|
||||
kind: collection.kind,
|
||||
path: collection.path,
|
||||
pattern: collection.pattern,
|
||||
}));
|
||||
}
|
||||
|
||||
private buildQmdRuntimeEnvironmentHash(): string {
|
||||
const relevantEnv = Object.fromEntries(
|
||||
Object.keys(this.env)
|
||||
.filter(
|
||||
(key) =>
|
||||
key === "PATH" ||
|
||||
key === "HOME" ||
|
||||
key === "LOCALAPPDATA" ||
|
||||
key === "XDG_CONFIG_HOME" ||
|
||||
key === "XDG_CACHE_HOME" ||
|
||||
key === "QMD_CONFIG_DIR" ||
|
||||
key.startsWith("QMD_"),
|
||||
)
|
||||
.toSorted()
|
||||
.map((key) => [key, this.env[key] ?? ""]),
|
||||
);
|
||||
return crypto.createHash("sha256").update(JSON.stringify(relevantEnv)).digest("hex");
|
||||
}
|
||||
|
||||
private async buildQmdCollectionValidationCacheContext(): Promise<QmdRuntimeCollectionValidationCacheContext> {
|
||||
return {
|
||||
workspaceDir: this.workspaceDir,
|
||||
agentId: this.agentId,
|
||||
qmdCommand: this.qmd.command,
|
||||
qmdVersion: await this.resolveQmdRuntimeIdentity(),
|
||||
qmdEnvironmentHash: this.buildQmdRuntimeEnvironmentHash(),
|
||||
qmdIndexPath: this.indexPath,
|
||||
searchMode: this.qmd.searchMode,
|
||||
collections: this.qmdRuntimeCacheCollections(),
|
||||
sources: this.qmdRuntimeCacheSources(),
|
||||
};
|
||||
}
|
||||
|
||||
private async buildQmdMultiCollectionProbeCacheContext(): Promise<QmdRuntimeMultiCollectionProbeCacheContext> {
|
||||
return {
|
||||
workspaceDir: this.workspaceDir,
|
||||
agentId: this.agentId,
|
||||
qmdCommand: this.qmd.command,
|
||||
qmdVersion: await this.resolveQmdRuntimeIdentity(),
|
||||
qmdEnvironmentHash: this.buildQmdRuntimeEnvironmentHash(),
|
||||
qmdIndexPath: this.indexPath,
|
||||
searchMode: this.qmd.searchMode,
|
||||
sources: this.qmdRuntimeCacheSources(),
|
||||
};
|
||||
}
|
||||
|
||||
private resolveQmdRuntimeIdentity(): Promise<string> {
|
||||
this.qmdRuntimeIdentityPromise ??= this.readQmdRuntimeIdentity();
|
||||
return this.qmdRuntimeIdentityPromise;
|
||||
}
|
||||
|
||||
private async readQmdRuntimeIdentity(): Promise<string> {
|
||||
const commandIdentity = `command:${this.qmd.command}`;
|
||||
try {
|
||||
const result = await this.runQmd(["--version"], {
|
||||
timeoutMs: Math.min(this.qmd.limits.timeoutMs, 2_000),
|
||||
});
|
||||
const versionText = `${result.stdout}\n${result.stderr}`.trim();
|
||||
return versionText ? `${commandIdentity};version:${versionText}` : commandIdentity;
|
||||
} catch {
|
||||
return commandIdentity;
|
||||
}
|
||||
}
|
||||
|
||||
private recordSearchPlanDebug(params: {
|
||||
debugContext: QmdSearchRuntimeDebugContext;
|
||||
command: "query" | "search" | "vsearch";
|
||||
collectionNames: string[];
|
||||
collectionGroups: string[][];
|
||||
}): void {
|
||||
const sources = uniqueValues(
|
||||
params.collectionNames
|
||||
.map((collectionName) => this.collectionRoots.get(collectionName)?.kind)
|
||||
.filter((source): source is MemorySource => Boolean(source)),
|
||||
);
|
||||
params.debugContext.searchPlan = {
|
||||
command: params.command,
|
||||
collectionCount: params.collectionNames.length,
|
||||
groupCount: params.collectionGroups.length,
|
||||
sources,
|
||||
};
|
||||
}
|
||||
|
||||
private beginQmdSearchRuntimeDebug(): QmdSearchRuntimeDebugContext {
|
||||
const debugContext: QmdSearchRuntimeDebugContext = {};
|
||||
if (this.pendingCollectionValidationDebug) {
|
||||
debugContext.collectionValidation = this.pendingCollectionValidationDebug;
|
||||
this.pendingCollectionValidationDebug = undefined;
|
||||
}
|
||||
return debugContext;
|
||||
}
|
||||
|
||||
private consumeQmdRuntimeDebug(
|
||||
debugContext: QmdSearchRuntimeDebugContext,
|
||||
): MemorySearchRuntimeDebug["qmd"] | undefined {
|
||||
const debug: NonNullable<MemorySearchRuntimeDebug["qmd"]> = {};
|
||||
if (debugContext.collectionValidation) {
|
||||
debug.collectionValidation = debugContext.collectionValidation;
|
||||
}
|
||||
if (debugContext.multiCollectionProbe) {
|
||||
debug.multiCollectionProbe = debugContext.multiCollectionProbe;
|
||||
}
|
||||
if (debugContext.searchPlan) {
|
||||
debug.searchPlan = debugContext.searchPlan;
|
||||
}
|
||||
return Object.keys(debug).length > 0 ? debug : undefined;
|
||||
}
|
||||
|
||||
private async ensureCollectionPathsBestEffort(): Promise<void> {
|
||||
for (const collection of this.qmd.collections) {
|
||||
try {
|
||||
await this.ensureCollectionPath(collection);
|
||||
} catch (err) {
|
||||
log.warn(
|
||||
`qmd collection path prepare failed for ${collection.name}: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureCollections(options?: {
|
||||
force?: boolean;
|
||||
debugContext?: QmdSearchRuntimeDebugContext;
|
||||
}): Promise<void> {
|
||||
const startedAt = Date.now();
|
||||
const cacheContext = await this.buildQmdCollectionValidationCacheContext();
|
||||
if (!options?.force) {
|
||||
const cached = await readQmdCollectionValidationCache(cacheContext);
|
||||
if (cached.state === "hit") {
|
||||
await this.ensureCollectionPathsBestEffort();
|
||||
const debug: QmdCollectionValidationDebug = {
|
||||
cacheState: "hit",
|
||||
elapsedMs: Math.max(0, Date.now() - startedAt),
|
||||
collectionCount: cached.value.validation.collectionCount,
|
||||
listCalls: 0,
|
||||
showCalls: 0,
|
||||
};
|
||||
if (options?.debugContext) {
|
||||
options.debugContext.collectionValidation = debug;
|
||||
} else {
|
||||
this.pendingCollectionValidationDebug = debug;
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const stats = { listCalls: 0, showCalls: 0 };
|
||||
let validationComplete = true;
|
||||
// QMD collections are persisted inside the index database and must be created
|
||||
// via the CLI. Prefer listing existing collections when supported, otherwise
|
||||
// fall back to best-effort idempotent `qmd collection add`.
|
||||
const existing = await this.listCollectionsBestEffort();
|
||||
const existing = await this.listCollectionsBestEffort(stats);
|
||||
|
||||
await this.migrateLegacyUnscopedCollections(existing);
|
||||
|
||||
|
|
@ -631,6 +816,7 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
if (!this.isCollectionMissingError(message)) {
|
||||
validationComplete = false;
|
||||
log.warn(`qmd collection remove failed for ${collection.name}: ${message}`);
|
||||
}
|
||||
}
|
||||
|
|
@ -661,13 +847,36 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
pattern: collection.pattern,
|
||||
});
|
||||
} else {
|
||||
validationComplete = false;
|
||||
log.warn(`qmd collection add skipped for ${collection.name}: ${message}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
validationComplete = false;
|
||||
log.warn(`qmd collection add failed for ${collection.name}: ${message}`);
|
||||
}
|
||||
}
|
||||
const wroteCache = validationComplete
|
||||
? await writeQmdCollectionValidationCache(cacheContext)
|
||||
: false;
|
||||
const debug: QmdCollectionValidationDebug = {
|
||||
cacheState: validationComplete
|
||||
? options?.force
|
||||
? "bypass-force"
|
||||
: wroteCache
|
||||
? "write"
|
||||
: "error"
|
||||
: "error",
|
||||
elapsedMs: Math.max(0, Date.now() - startedAt),
|
||||
collectionCount: this.qmd.collections.length,
|
||||
listCalls: stats.listCalls,
|
||||
showCalls: stats.showCalls,
|
||||
};
|
||||
if (options?.debugContext) {
|
||||
options.debugContext.collectionValidation = debug;
|
||||
} else {
|
||||
this.pendingCollectionValidationDebug = debug;
|
||||
}
|
||||
}
|
||||
|
||||
private async tryRebindSameNameCollection(params: {
|
||||
|
|
@ -713,9 +922,15 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
);
|
||||
}
|
||||
|
||||
private async listCollectionsBestEffort(): Promise<Map<string, ListedCollection>> {
|
||||
private async listCollectionsBestEffort(stats?: {
|
||||
listCalls: number;
|
||||
showCalls: number;
|
||||
}): Promise<Map<string, ListedCollection>> {
|
||||
const existing = new Map<string, ListedCollection>();
|
||||
try {
|
||||
if (stats) {
|
||||
stats.listCalls += 1;
|
||||
}
|
||||
const result = await this.runQmd(["collection", "list", "--json"], {
|
||||
timeoutMs: this.qmd.update.commandTimeoutMs,
|
||||
});
|
||||
|
|
@ -737,6 +952,9 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
continue;
|
||||
}
|
||||
try {
|
||||
if (stats) {
|
||||
stats.showCalls += 1;
|
||||
}
|
||||
const showResult = await this.runQmd(["collection", "show", collection.name], {
|
||||
timeoutMs: this.qmd.update.commandTimeoutMs,
|
||||
});
|
||||
|
|
@ -956,14 +1174,17 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
);
|
||||
}
|
||||
|
||||
private async tryRepairMissingCollectionSearch(err: unknown): Promise<boolean> {
|
||||
private async tryRepairMissingCollectionSearch(
|
||||
err: unknown,
|
||||
debugContext: QmdSearchRuntimeDebugContext,
|
||||
): Promise<boolean> {
|
||||
if (!this.isMissingCollectionSearchError(err)) {
|
||||
return false;
|
||||
}
|
||||
log.warn(
|
||||
"qmd search failed because a managed collection is missing; repairing collections and retrying once",
|
||||
);
|
||||
await this.ensureCollections();
|
||||
await this.ensureCollections({ force: true, debugContext });
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -1318,6 +1539,7 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
if (searchSignal?.aborted) {
|
||||
throw asAbortError(searchSignal);
|
||||
}
|
||||
const debugContext = this.beginQmdSearchRuntimeDebug();
|
||||
const trimmed = query.trim();
|
||||
if (!trimmed) {
|
||||
return [];
|
||||
|
|
@ -1344,6 +1566,7 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
const runSearchAttempt = async (
|
||||
allowMissingCollectionRepair: boolean,
|
||||
): Promise<QmdQueryResult[]> => {
|
||||
let attemptedCombinedCollectionFilter = false;
|
||||
try {
|
||||
if (mcporterEnabled) {
|
||||
const minScore = opts?.minScore ?? 0;
|
||||
|
|
@ -1402,7 +1625,15 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
const collectionGroups = await this.resolveCollectionSearchGroups(
|
||||
collectionNames,
|
||||
searchSignal,
|
||||
debugContext,
|
||||
);
|
||||
this.recordSearchPlanDebug({
|
||||
debugContext,
|
||||
command: qmdSearchCommand,
|
||||
collectionNames,
|
||||
collectionGroups,
|
||||
});
|
||||
attemptedCombinedCollectionFilter = collectionGroups.some((group) => group.length > 1);
|
||||
if (collectionGroups.length > 1) {
|
||||
return await this.runQueryAcrossCollectionGroups(
|
||||
trimmed,
|
||||
|
|
@ -1424,6 +1655,9 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
qmdSearchCommand !== "query" &&
|
||||
this.isUnsupportedQmdOptionError(err)
|
||||
) {
|
||||
if (attemptedCombinedCollectionFilter) {
|
||||
await this.markQmdMultiCollectionFiltersUnsupported(debugContext);
|
||||
}
|
||||
effectiveSearchMode = "query";
|
||||
searchFallbackReason = "unsupported-search-flags";
|
||||
log.warn(
|
||||
|
|
@ -1433,7 +1667,14 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
const collectionGroups = await this.resolveCollectionSearchGroups(
|
||||
collectionNames,
|
||||
searchSignal,
|
||||
debugContext,
|
||||
);
|
||||
this.recordSearchPlanDebug({
|
||||
debugContext,
|
||||
command: "query",
|
||||
collectionNames,
|
||||
collectionGroups,
|
||||
});
|
||||
if (collectionGroups.length > 1) {
|
||||
return await this.runQueryAcrossCollectionGroups(
|
||||
trimmed,
|
||||
|
|
@ -1463,7 +1704,7 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
try {
|
||||
parsed = await runSearchAttempt(true);
|
||||
} catch (err) {
|
||||
if (!(await this.tryRepairMissingCollectionSearch(err))) {
|
||||
if (!(await this.tryRepairMissingCollectionSearch(err, debugContext))) {
|
||||
throw err instanceof Error ? err : new Error(String(err));
|
||||
}
|
||||
parsed = await runSearchAttempt(false);
|
||||
|
|
@ -1512,6 +1753,7 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
configuredMode: qmdSearchCommand,
|
||||
effectiveMode: effectiveSearchMode,
|
||||
fallback: searchFallbackReason,
|
||||
qmd: this.consumeQmdRuntimeDebug(debugContext),
|
||||
});
|
||||
let ranked = results;
|
||||
if (opts?.sources?.length) {
|
||||
|
|
@ -3370,23 +3612,41 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
private async resolveCollectionSearchGroups(
|
||||
collectionNames: string[],
|
||||
signal?: AbortSignal,
|
||||
debugContext?: QmdSearchRuntimeDebugContext,
|
||||
): Promise<string[][]> {
|
||||
if (collectionNames.length <= 1) {
|
||||
return [collectionNames];
|
||||
}
|
||||
if (!(await this.supportsQmdMultiCollectionFilters(signal))) {
|
||||
if (!(await this.supportsQmdMultiCollectionFilters(signal, debugContext))) {
|
||||
return collectionNames.map((collectionName) => [collectionName]);
|
||||
}
|
||||
return this.groupCollectionNamesBySource(collectionNames);
|
||||
}
|
||||
|
||||
private async supportsQmdMultiCollectionFilters(signal?: AbortSignal): Promise<boolean> {
|
||||
private async supportsQmdMultiCollectionFilters(
|
||||
signal?: AbortSignal,
|
||||
debugContext?: QmdSearchRuntimeDebugContext,
|
||||
): Promise<boolean> {
|
||||
if (signal?.aborted) {
|
||||
throw asAbortError(signal);
|
||||
}
|
||||
if (this.multiCollectionFilterSupported !== null) {
|
||||
return this.multiCollectionFilterSupported;
|
||||
}
|
||||
const startedAt = Date.now();
|
||||
const cacheContext = await this.buildQmdMultiCollectionProbeCacheContext();
|
||||
const cached = await readQmdMultiCollectionProbeCache(cacheContext);
|
||||
if (cached.state === "hit") {
|
||||
this.multiCollectionFilterSupported = cached.value.multiCollectionProbe.supported;
|
||||
if (debugContext) {
|
||||
debugContext.multiCollectionProbe = {
|
||||
cacheState: "hit",
|
||||
elapsedMs: Math.max(0, Date.now() - startedAt),
|
||||
supported: this.multiCollectionFilterSupported,
|
||||
};
|
||||
}
|
||||
return this.multiCollectionFilterSupported;
|
||||
}
|
||||
try {
|
||||
const result = await this.runQmd(["--help"], {
|
||||
timeoutMs: Math.min(this.qmd.limits.timeoutMs, 5_000),
|
||||
|
|
@ -3395,17 +3655,50 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
const helpText = `${result.stdout}\n${result.stderr}`;
|
||||
this.multiCollectionFilterSupported =
|
||||
/\b(?:one or more collections|collection\(s\)|multiple -c flags)\b/i.test(helpText);
|
||||
const wroteCache = await writeQmdMultiCollectionProbeCache(
|
||||
cacheContext,
|
||||
this.multiCollectionFilterSupported,
|
||||
);
|
||||
if (debugContext) {
|
||||
debugContext.multiCollectionProbe = {
|
||||
cacheState: wroteCache ? "write" : "error",
|
||||
elapsedMs: Math.max(0, Date.now() - startedAt),
|
||||
supported: this.multiCollectionFilterSupported,
|
||||
};
|
||||
}
|
||||
} catch (err) {
|
||||
// Cancellation says nothing about QMD capabilities; leave the probe uncached.
|
||||
if (signal?.aborted) {
|
||||
throw asAbortError(signal);
|
||||
}
|
||||
this.multiCollectionFilterSupported = false;
|
||||
if (debugContext) {
|
||||
debugContext.multiCollectionProbe = {
|
||||
cacheState: "error",
|
||||
elapsedMs: Math.max(0, Date.now() - startedAt),
|
||||
supported: false,
|
||||
};
|
||||
}
|
||||
log.debug(`qmd multi-collection filter probe failed: ${String(err)}`);
|
||||
}
|
||||
return this.multiCollectionFilterSupported;
|
||||
}
|
||||
|
||||
private async markQmdMultiCollectionFiltersUnsupported(
|
||||
debugContext: QmdSearchRuntimeDebugContext,
|
||||
): Promise<void> {
|
||||
const startedAt = Date.now();
|
||||
const cacheContext = await this.buildQmdMultiCollectionProbeCacheContext();
|
||||
this.multiCollectionFilterSupported = false;
|
||||
await clearQmdMultiCollectionProbeCache(cacheContext);
|
||||
const wroteCache = await writeQmdMultiCollectionProbeCache(cacheContext, false);
|
||||
debugContext.multiCollectionProbe = {
|
||||
cacheState: wroteCache ? "write" : "error",
|
||||
elapsedMs: Math.max(0, Date.now() - startedAt),
|
||||
supported: false,
|
||||
};
|
||||
}
|
||||
|
||||
private async runQueryAcrossCollectionGroups(
|
||||
query: string,
|
||||
limit: number,
|
||||
|
|
|
|||
323
extensions/memory-core/src/memory/qmd-runtime-cache.test.ts
Normal file
323
extensions/memory-core/src/memory/qmd-runtime-cache.test.ts
Normal file
|
|
@ -0,0 +1,323 @@
|
|||
import path from "node:path";
|
||||
import { withTempDir } from "openclaw/plugin-sdk/test-env";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
configureMemoryCoreDreamingState,
|
||||
configureMemoryCoreDreamingStateForTests,
|
||||
openMemoryCoreStateStore,
|
||||
memoryCoreWorkspaceEntryKey,
|
||||
resetMemoryCoreDreamingStateForTests,
|
||||
} from "../dreaming-state.js";
|
||||
import {
|
||||
QMD_RUNTIME_CACHE_COLLECTION_VALIDATION_NAMESPACE,
|
||||
QMD_RUNTIME_CACHE_COLLECTION_VALIDATION_TTL_MS,
|
||||
QMD_RUNTIME_CACHE_MULTI_COLLECTION_PROBE_NAMESPACE,
|
||||
QMD_RUNTIME_CACHE_MULTI_COLLECTION_PROBE_TTL_MS,
|
||||
buildQmdMultiCollectionProbeCacheContextHash,
|
||||
clearQmdCollectionValidationCache,
|
||||
clearQmdMultiCollectionProbeCache,
|
||||
readQmdCollectionValidationCache,
|
||||
readQmdMultiCollectionProbeCache,
|
||||
type QmdRuntimeCollectionValidationCacheContext,
|
||||
type QmdRuntimeManagedCollection,
|
||||
type QmdRuntimeMultiCollectionProbeCacheContext,
|
||||
writeQmdCollectionValidationCache,
|
||||
writeQmdMultiCollectionProbeCache,
|
||||
} from "./qmd-runtime-cache.js";
|
||||
|
||||
beforeAll(async () => {
|
||||
await configureMemoryCoreDreamingStateForTests();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
resetMemoryCoreDreamingStateForTests();
|
||||
});
|
||||
|
||||
async function clearStore(namespace: string): Promise<void> {
|
||||
try {
|
||||
await openMemoryCoreStateStore({
|
||||
namespace,
|
||||
maxEntries: 1_000,
|
||||
}).clear();
|
||||
} catch {
|
||||
// fail open
|
||||
}
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await clearStore(QMD_RUNTIME_CACHE_COLLECTION_VALIDATION_NAMESPACE);
|
||||
await clearStore(QMD_RUNTIME_CACHE_MULTI_COLLECTION_PROBE_NAMESPACE);
|
||||
});
|
||||
|
||||
async function withWorkspace<T>(run: (workspaceDir: string) => Promise<T>): Promise<T> {
|
||||
return await withTempDir("qmd-runtime-cache-", run);
|
||||
}
|
||||
|
||||
function managedCollections(): QmdRuntimeManagedCollection[] {
|
||||
return [
|
||||
{
|
||||
name: "project-notes",
|
||||
kind: "memory",
|
||||
path: "/repo/project-notes",
|
||||
pattern: "*.md",
|
||||
},
|
||||
{
|
||||
name: "sessions",
|
||||
kind: "sessions",
|
||||
path: "/repo/sessions",
|
||||
pattern: "*",
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function collectionValidationContext(
|
||||
workspaceDir: string,
|
||||
): QmdRuntimeCollectionValidationCacheContext {
|
||||
return {
|
||||
workspaceDir,
|
||||
agentId: "agent-a",
|
||||
qmdCommand: "qmd",
|
||||
qmdIndexPath: path.join(workspaceDir, ".openclaw", "index.sqlite"),
|
||||
searchMode: "search",
|
||||
collections: managedCollections(),
|
||||
sources: ["memory", "sessions"],
|
||||
};
|
||||
}
|
||||
|
||||
function multiCollectionProbeContext(
|
||||
workspaceDir: string,
|
||||
): QmdRuntimeMultiCollectionProbeCacheContext {
|
||||
return {
|
||||
workspaceDir,
|
||||
agentId: "agent-a",
|
||||
qmdCommand: "qmd",
|
||||
qmdIndexPath: path.join(workspaceDir, ".openclaw", "index.sqlite"),
|
||||
searchMode: "search",
|
||||
sources: ["memory", "sessions"],
|
||||
};
|
||||
}
|
||||
|
||||
describe("qmd-runtime-cache", () => {
|
||||
it("writes and reads collection validation cache entries", async () => {
|
||||
await withWorkspace(async (workspaceDir) => {
|
||||
const context = collectionValidationContext(workspaceDir);
|
||||
const writeStartedAtMs = 1_000;
|
||||
|
||||
const writeOk = await writeQmdCollectionValidationCache(context, writeStartedAtMs);
|
||||
expect(writeOk).toBe(true);
|
||||
|
||||
const read = await readQmdCollectionValidationCache(
|
||||
{ ...context, sources: ["sessions", "memory"] },
|
||||
writeStartedAtMs + 1,
|
||||
);
|
||||
expect(read).toMatchObject({
|
||||
state: "hit",
|
||||
value: {
|
||||
validation: {
|
||||
ok: true,
|
||||
collectionCount: context.collections.length,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("writes and reads multi-collection probe cache entries", async () => {
|
||||
await withWorkspace(async (workspaceDir) => {
|
||||
const context = multiCollectionProbeContext(workspaceDir);
|
||||
const writeStartedAtMs = 2_000;
|
||||
|
||||
const writeOk = await writeQmdMultiCollectionProbeCache(context, true, writeStartedAtMs);
|
||||
expect(writeOk).toBe(true);
|
||||
|
||||
const read = await readQmdMultiCollectionProbeCache(context, writeStartedAtMs + 1);
|
||||
expect(read).toMatchObject({
|
||||
state: "hit",
|
||||
value: {
|
||||
multiCollectionProbe: {
|
||||
supported: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("scopes cache entries by workspace", async () => {
|
||||
await withWorkspace(async (firstWorkspace) => {
|
||||
await withWorkspace(async (secondWorkspace) => {
|
||||
const context = collectionValidationContext(firstWorkspace);
|
||||
|
||||
expect(await writeQmdCollectionValidationCache(context, 3_000)).toBe(true);
|
||||
|
||||
const sameLogicalDifferentWorkspace: QmdRuntimeCollectionValidationCacheContext = {
|
||||
...context,
|
||||
workspaceDir: secondWorkspace,
|
||||
qmdIndexPath: path.join(secondWorkspace, ".openclaw", "index.sqlite"),
|
||||
};
|
||||
|
||||
const miss = await readQmdCollectionValidationCache(sameLogicalDifferentWorkspace, 3_001);
|
||||
expect(miss).toStrictEqual({ state: "miss" });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("misses collection validation cache when managed collection paths change", async () => {
|
||||
await withWorkspace(async (workspaceDir) => {
|
||||
const context = collectionValidationContext(workspaceDir);
|
||||
|
||||
expect(await writeQmdCollectionValidationCache(context, 3_500)).toBe(true);
|
||||
|
||||
const changedContext: QmdRuntimeCollectionValidationCacheContext = {
|
||||
...context,
|
||||
collections: context.collections.map((collection) =>
|
||||
collection.name === "project-notes"
|
||||
? {
|
||||
name: collection.name,
|
||||
kind: collection.kind,
|
||||
path: `${collection.path}-moved`,
|
||||
pattern: collection.pattern,
|
||||
}
|
||||
: collection,
|
||||
),
|
||||
};
|
||||
|
||||
expect(await readQmdCollectionValidationCache(changedContext, 3_501)).toStrictEqual({
|
||||
state: "miss",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("misses validation and probe caches when qmd runtime environment changes", async () => {
|
||||
await withWorkspace(async (workspaceDir) => {
|
||||
const validationContext = {
|
||||
...collectionValidationContext(workspaceDir),
|
||||
qmdEnvironmentHash: "env-a",
|
||||
};
|
||||
const probeContext = {
|
||||
...multiCollectionProbeContext(workspaceDir),
|
||||
qmdEnvironmentHash: "env-a",
|
||||
};
|
||||
|
||||
expect(await writeQmdCollectionValidationCache(validationContext, 3_600)).toBe(true);
|
||||
expect(await writeQmdMultiCollectionProbeCache(probeContext, true, 3_600)).toBe(true);
|
||||
|
||||
expect(
|
||||
await readQmdCollectionValidationCache(
|
||||
{ ...validationContext, qmdEnvironmentHash: "env-b" },
|
||||
3_601,
|
||||
),
|
||||
).toStrictEqual({ state: "miss" });
|
||||
expect(
|
||||
await readQmdMultiCollectionProbeCache(
|
||||
{ ...probeContext, qmdEnvironmentHash: "env-b" },
|
||||
3_601,
|
||||
),
|
||||
).toStrictEqual({ state: "miss" });
|
||||
});
|
||||
});
|
||||
|
||||
it("treats cache misses for malformed values and expired entries", async () => {
|
||||
await withWorkspace(async (workspaceDir) => {
|
||||
const context = multiCollectionProbeContext(workspaceDir);
|
||||
const nowMs = 4_000;
|
||||
await writeQmdMultiCollectionProbeCache(context, false, nowMs);
|
||||
|
||||
const key = memoryCoreWorkspaceEntryKey(
|
||||
workspaceDir,
|
||||
`qmd-runtime-cache.multi-collection-probe:${buildQmdMultiCollectionProbeCacheContextHash(context)}`,
|
||||
);
|
||||
const store = openMemoryCoreStateStore({
|
||||
namespace: QMD_RUNTIME_CACHE_MULTI_COLLECTION_PROBE_NAMESPACE,
|
||||
maxEntries: 1_000,
|
||||
});
|
||||
|
||||
await store.register(key, {
|
||||
version: 1,
|
||||
createdAtMs: "bad",
|
||||
expiresAtMs: 0,
|
||||
keyHash: "bad",
|
||||
multiCollectionProbe: { supported: true },
|
||||
});
|
||||
|
||||
const malformed = await readQmdMultiCollectionProbeCache(context, nowMs + 1);
|
||||
expect(malformed).toStrictEqual({ state: "miss" });
|
||||
|
||||
const expired = await readQmdMultiCollectionProbeCache(
|
||||
context,
|
||||
nowMs + QMD_RUNTIME_CACHE_MULTI_COLLECTION_PROBE_TTL_MS + 1,
|
||||
);
|
||||
expect(expired).toStrictEqual({ state: "miss" });
|
||||
});
|
||||
});
|
||||
|
||||
it("uses separate namespaces for validation and probe entries", async () => {
|
||||
await withWorkspace(async (workspaceDir) => {
|
||||
const validationContext = collectionValidationContext(workspaceDir);
|
||||
const probeContext = multiCollectionProbeContext(workspaceDir);
|
||||
|
||||
expect(await writeQmdCollectionValidationCache(validationContext, 5_000)).toBe(true);
|
||||
expect(await writeQmdMultiCollectionProbeCache(probeContext, true, 5_000)).toBe(true);
|
||||
|
||||
const validationStore = openMemoryCoreStateStore({
|
||||
namespace: QMD_RUNTIME_CACHE_COLLECTION_VALIDATION_NAMESPACE,
|
||||
maxEntries: 1_000,
|
||||
});
|
||||
const probeStore = openMemoryCoreStateStore({
|
||||
namespace: QMD_RUNTIME_CACHE_MULTI_COLLECTION_PROBE_NAMESPACE,
|
||||
maxEntries: 1_000,
|
||||
});
|
||||
|
||||
expect((await validationStore.entries()).length).toBeGreaterThan(0);
|
||||
expect((await probeStore.entries()).length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
it("fails open when state store is unavailable", async () => {
|
||||
await withWorkspace(async (workspaceDir) => {
|
||||
const validationContext = collectionValidationContext(workspaceDir);
|
||||
const probeContext = multiCollectionProbeContext(workspaceDir);
|
||||
|
||||
configureMemoryCoreDreamingState(() => {
|
||||
throw new Error("state store unavailable");
|
||||
});
|
||||
|
||||
try {
|
||||
expect(await readQmdCollectionValidationCache(validationContext)).toStrictEqual({
|
||||
state: "miss",
|
||||
});
|
||||
expect(await writeQmdCollectionValidationCache(validationContext)).toBe(false);
|
||||
expect(await readQmdMultiCollectionProbeCache(probeContext)).toStrictEqual({
|
||||
state: "miss",
|
||||
});
|
||||
expect(await writeQmdMultiCollectionProbeCache(probeContext, true)).toBe(false);
|
||||
} finally {
|
||||
await configureMemoryCoreDreamingStateForTests();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("exposes bounded TTL windows", () => {
|
||||
expect(QMD_RUNTIME_CACHE_COLLECTION_VALIDATION_TTL_MS).toBe(5 * 60_000);
|
||||
expect(QMD_RUNTIME_CACHE_MULTI_COLLECTION_PROBE_TTL_MS).toBe(10 * 60_000);
|
||||
});
|
||||
|
||||
it("can clear cache keys explicitly", async () => {
|
||||
await withWorkspace(async (workspaceDir) => {
|
||||
const validationContext = collectionValidationContext(workspaceDir);
|
||||
const probeContext = multiCollectionProbeContext(workspaceDir);
|
||||
|
||||
expect(await writeQmdCollectionValidationCache(validationContext)).toBe(true);
|
||||
expect(await writeQmdMultiCollectionProbeCache(probeContext, true)).toBe(true);
|
||||
|
||||
await clearQmdCollectionValidationCache(validationContext);
|
||||
await clearQmdMultiCollectionProbeCache(probeContext);
|
||||
|
||||
expect(await readQmdCollectionValidationCache(validationContext)).toStrictEqual({
|
||||
state: "miss",
|
||||
});
|
||||
expect(await readQmdMultiCollectionProbeCache(probeContext)).toStrictEqual({
|
||||
state: "miss",
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
435
extensions/memory-core/src/memory/qmd-runtime-cache.ts
Normal file
435
extensions/memory-core/src/memory/qmd-runtime-cache.ts
Normal file
|
|
@ -0,0 +1,435 @@
|
|||
// Memory Core QMD runtime cache helpers.
|
||||
import { createHash } from "node:crypto";
|
||||
import type { PluginStateKeyedStore } from "openclaw/plugin-sdk/plugin-state-runtime";
|
||||
import { memoryCoreWorkspaceEntryKey, openMemoryCoreStateStore } from "../dreaming-state.js";
|
||||
|
||||
export const QMD_RUNTIME_CACHE_COLLECTION_VALIDATION_NAMESPACE =
|
||||
"qmd-runtime-cache.collection-validation";
|
||||
export const QMD_RUNTIME_CACHE_MULTI_COLLECTION_PROBE_NAMESPACE =
|
||||
"qmd-runtime-cache.multi-collection-probe";
|
||||
export const QMD_RUNTIME_CACHE_COLLECTION_VALIDATION_MAX_ENTRIES = 1_000;
|
||||
export const QMD_RUNTIME_CACHE_MULTI_COLLECTION_PROBE_MAX_ENTRIES = 1_000;
|
||||
export const QMD_RUNTIME_CACHE_COLLECTION_VALIDATION_TTL_MS = 5 * 60_000;
|
||||
export const QMD_RUNTIME_CACHE_MULTI_COLLECTION_PROBE_TTL_MS = 10 * 60_000;
|
||||
|
||||
const QMD_RUNTIME_CACHE_ENTRY_VERSION = 1;
|
||||
|
||||
export type QmdRuntimeManagedCollection = {
|
||||
name: string;
|
||||
kind: "memory" | "custom" | "sessions";
|
||||
path: string;
|
||||
pattern: string;
|
||||
};
|
||||
|
||||
type QmdRuntimeCacheContextBase = {
|
||||
workspaceDir: string;
|
||||
agentId: string;
|
||||
qmdCommand: string;
|
||||
qmdVersion?: string;
|
||||
qmdEnvironmentHash?: string;
|
||||
qmdIndexPath: string;
|
||||
searchMode: string;
|
||||
};
|
||||
|
||||
export type QmdRuntimeCollectionValidationCacheContext = QmdRuntimeCacheContextBase & {
|
||||
collections: readonly QmdRuntimeManagedCollection[];
|
||||
sources: readonly string[];
|
||||
};
|
||||
|
||||
export type QmdRuntimeMultiCollectionProbeCacheContext = QmdRuntimeCacheContextBase & {
|
||||
sources: readonly string[];
|
||||
};
|
||||
|
||||
export type QmdRuntimeCacheCollectionValidationEntry = {
|
||||
version: 1;
|
||||
createdAtMs: number;
|
||||
expiresAtMs: number;
|
||||
keyHash: string;
|
||||
validation: {
|
||||
ok: true;
|
||||
collectionConfigHash: string;
|
||||
collectionCount: number;
|
||||
};
|
||||
};
|
||||
|
||||
export type QmdRuntimeCacheMultiCollectionProbeEntry = {
|
||||
version: 1;
|
||||
createdAtMs: number;
|
||||
expiresAtMs: number;
|
||||
keyHash: string;
|
||||
multiCollectionProbe: {
|
||||
supported: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
export type QmdRuntimeCacheResult<T> =
|
||||
| {
|
||||
state: "hit";
|
||||
value: T;
|
||||
}
|
||||
| { state: "miss" };
|
||||
|
||||
function normalizeText(value: string): string {
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
function normalizeCollection(collection: QmdRuntimeManagedCollection) {
|
||||
return {
|
||||
name: normalizeText(collection.name),
|
||||
kind: collection.kind,
|
||||
pathHash: normalizePathIdentity(collection.path),
|
||||
pattern: normalizeText(collection.pattern),
|
||||
};
|
||||
}
|
||||
|
||||
function hashText(value: string): string {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function normalizePathIdentity(value: string): string {
|
||||
const normalized =
|
||||
process.platform === "win32" ? normalizeText(value).toLowerCase() : normalizeText(value);
|
||||
return hashText(normalized);
|
||||
}
|
||||
|
||||
function sortedUnique(values: readonly string[]): string[] {
|
||||
return [...new Set(values.map((value) => normalizeText(value)).filter(Boolean))].toSorted();
|
||||
}
|
||||
|
||||
function buildCollectionConfigHash(collections: readonly QmdRuntimeManagedCollection[]): string {
|
||||
const normalized = collections
|
||||
.map((collection) => ({
|
||||
...normalizeCollection(collection),
|
||||
}))
|
||||
.toSorted(
|
||||
(left, right) =>
|
||||
left.name.localeCompare(right.name) ||
|
||||
left.kind.localeCompare(right.kind) ||
|
||||
left.pathHash.localeCompare(right.pathHash) ||
|
||||
left.pattern.localeCompare(right.pattern),
|
||||
)
|
||||
.map((entry) => `${entry.name}|${entry.kind}|${entry.pathHash}|${entry.pattern}`)
|
||||
.join(";");
|
||||
return hashText(normalized);
|
||||
}
|
||||
|
||||
function buildCollectionValidationCacheContextInput(
|
||||
params: QmdRuntimeCollectionValidationCacheContext,
|
||||
): string {
|
||||
return JSON.stringify({
|
||||
agentId: normalizeText(params.agentId),
|
||||
commandHash: hashText(normalizeText(params.qmdCommand)),
|
||||
environmentHash: normalizeText(params.qmdEnvironmentHash ?? ""),
|
||||
indexPathHash: normalizePathIdentity(params.qmdIndexPath),
|
||||
qmdVersion: normalizeText(params.qmdVersion ?? ""),
|
||||
searchMode: params.searchMode,
|
||||
sourceSet: sortedUnique(params.sources),
|
||||
collectionConfigHash: buildCollectionConfigHash(params.collections),
|
||||
});
|
||||
}
|
||||
|
||||
function buildMultiCollectionProbeCacheContextInput(
|
||||
params: QmdRuntimeMultiCollectionProbeCacheContext,
|
||||
): string {
|
||||
return JSON.stringify({
|
||||
agentId: normalizeText(params.agentId),
|
||||
commandHash: hashText(normalizeText(params.qmdCommand)),
|
||||
environmentHash: normalizeText(params.qmdEnvironmentHash ?? ""),
|
||||
indexPathHash: normalizePathIdentity(params.qmdIndexPath),
|
||||
qmdVersion: normalizeText(params.qmdVersion ?? ""),
|
||||
searchMode: params.searchMode,
|
||||
sourceSet: sortedUnique(params.sources),
|
||||
});
|
||||
}
|
||||
|
||||
function buildCollectionValidationCacheHash(
|
||||
params: QmdRuntimeCollectionValidationCacheContext,
|
||||
): string {
|
||||
return hashText(buildCollectionValidationCacheContextInput(params));
|
||||
}
|
||||
|
||||
function buildMultiCollectionProbeCacheHash(
|
||||
params: QmdRuntimeMultiCollectionProbeCacheContext,
|
||||
): string {
|
||||
return hashText(buildMultiCollectionProbeCacheContextInput(params));
|
||||
}
|
||||
|
||||
export function buildQmdCollectionValidationCacheContextHash(
|
||||
params: QmdRuntimeCollectionValidationCacheContext,
|
||||
): string {
|
||||
return buildCollectionValidationCacheHash(params);
|
||||
}
|
||||
|
||||
export function buildQmdMultiCollectionProbeCacheContextHash(
|
||||
params: QmdRuntimeMultiCollectionProbeCacheContext,
|
||||
): string {
|
||||
return buildMultiCollectionProbeCacheHash(params);
|
||||
}
|
||||
|
||||
function collectionValidationStore(): PluginStateKeyedStore<QmdRuntimeCacheCollectionValidationEntry> {
|
||||
return openMemoryCoreStateStore<QmdRuntimeCacheCollectionValidationEntry>({
|
||||
namespace: QMD_RUNTIME_CACHE_COLLECTION_VALIDATION_NAMESPACE,
|
||||
maxEntries: QMD_RUNTIME_CACHE_COLLECTION_VALIDATION_MAX_ENTRIES,
|
||||
});
|
||||
}
|
||||
|
||||
function multiCollectionProbeStore(): PluginStateKeyedStore<QmdRuntimeCacheMultiCollectionProbeEntry> {
|
||||
return openMemoryCoreStateStore<QmdRuntimeCacheMultiCollectionProbeEntry>({
|
||||
namespace: QMD_RUNTIME_CACHE_MULTI_COLLECTION_PROBE_NAMESPACE,
|
||||
maxEntries: QMD_RUNTIME_CACHE_MULTI_COLLECTION_PROBE_MAX_ENTRIES,
|
||||
});
|
||||
}
|
||||
|
||||
function collectionValidationEntryKey(params: QmdRuntimeCollectionValidationCacheContext): string {
|
||||
return memoryCoreWorkspaceEntryKey(
|
||||
params.workspaceDir,
|
||||
`qmd-runtime-cache.collection-validation:${buildCollectionValidationCacheHash(params)}`,
|
||||
);
|
||||
}
|
||||
|
||||
function multiCollectionProbeEntryKey(params: QmdRuntimeMultiCollectionProbeCacheContext): string {
|
||||
return memoryCoreWorkspaceEntryKey(
|
||||
params.workspaceDir,
|
||||
`qmd-runtime-cache.multi-collection-probe:${buildMultiCollectionProbeCacheHash(params)}`,
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeCollectionValidationEntry(
|
||||
value: unknown,
|
||||
nowMs: number,
|
||||
expectedKeyHash: string,
|
||||
): QmdRuntimeCacheCollectionValidationEntry | undefined {
|
||||
if (typeof value !== "object" || value === null) {
|
||||
return undefined;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
if (record.version !== QMD_RUNTIME_CACHE_ENTRY_VERSION) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const createdAtMs =
|
||||
typeof record.createdAtMs === "number"
|
||||
? Math.max(0, Math.floor(record.createdAtMs))
|
||||
: Number.NaN;
|
||||
const expiresAtMs =
|
||||
typeof record.expiresAtMs === "number"
|
||||
? Math.max(0, Math.floor(record.expiresAtMs))
|
||||
: Number.NaN;
|
||||
if (
|
||||
!Number.isFinite(createdAtMs) ||
|
||||
!Number.isFinite(expiresAtMs) ||
|
||||
!Number.isFinite(nowMs) ||
|
||||
nowMs >= expiresAtMs
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const keyHash = normalizeText(typeof record.keyHash === "string" ? record.keyHash : "");
|
||||
if (keyHash !== expectedKeyHash) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const validation = record.validation;
|
||||
if (typeof validation !== "object" || validation === null) {
|
||||
return undefined;
|
||||
}
|
||||
const validationRecord = validation as Record<string, unknown>;
|
||||
if (validationRecord.ok !== true) {
|
||||
return undefined;
|
||||
}
|
||||
if (typeof validationRecord.collectionConfigHash !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
if (typeof validationRecord.collectionCount !== "number") {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
version: QMD_RUNTIME_CACHE_ENTRY_VERSION,
|
||||
createdAtMs,
|
||||
expiresAtMs,
|
||||
keyHash,
|
||||
validation: {
|
||||
ok: true,
|
||||
collectionConfigHash: normalizeText(validationRecord.collectionConfigHash),
|
||||
collectionCount: Math.max(0, Math.floor(validationRecord.collectionCount)),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeMultiCollectionProbeEntry(
|
||||
value: unknown,
|
||||
nowMs: number,
|
||||
expectedKeyHash: string,
|
||||
): QmdRuntimeCacheMultiCollectionProbeEntry | undefined {
|
||||
if (typeof value !== "object" || value === null) {
|
||||
return undefined;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
if (record.version !== QMD_RUNTIME_CACHE_ENTRY_VERSION) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const createdAtMs =
|
||||
typeof record.createdAtMs === "number"
|
||||
? Math.max(0, Math.floor(record.createdAtMs))
|
||||
: Number.NaN;
|
||||
const expiresAtMs =
|
||||
typeof record.expiresAtMs === "number"
|
||||
? Math.max(0, Math.floor(record.expiresAtMs))
|
||||
: Number.NaN;
|
||||
if (
|
||||
!Number.isFinite(createdAtMs) ||
|
||||
!Number.isFinite(expiresAtMs) ||
|
||||
!Number.isFinite(nowMs) ||
|
||||
nowMs >= expiresAtMs
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const keyHash = normalizeText(typeof record.keyHash === "string" ? record.keyHash : "");
|
||||
if (keyHash !== expectedKeyHash) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const probe = record.multiCollectionProbe;
|
||||
if (typeof probe !== "object" || probe === null) {
|
||||
return undefined;
|
||||
}
|
||||
const probeRecord = probe as Record<string, unknown>;
|
||||
if (typeof probeRecord.supported !== "boolean") {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
version: QMD_RUNTIME_CACHE_ENTRY_VERSION,
|
||||
createdAtMs,
|
||||
expiresAtMs,
|
||||
keyHash,
|
||||
multiCollectionProbe: {
|
||||
supported: probeRecord.supported,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function readQmdCollectionValidationCache(
|
||||
params: QmdRuntimeCollectionValidationCacheContext,
|
||||
nowMs = Date.now(),
|
||||
): Promise<QmdRuntimeCacheResult<QmdRuntimeCacheCollectionValidationEntry>> {
|
||||
try {
|
||||
const store = collectionValidationStore();
|
||||
const key = collectionValidationEntryKey(params);
|
||||
const expectedKeyHash = buildCollectionValidationCacheHash(params);
|
||||
const raw = await store.lookup(key);
|
||||
if (!raw) {
|
||||
return { state: "miss" };
|
||||
}
|
||||
const validated = normalizeCollectionValidationEntry(raw, nowMs, expectedKeyHash);
|
||||
return validated ? { state: "hit", value: validated } : { state: "miss" };
|
||||
} catch {
|
||||
return { state: "miss" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeQmdCollectionValidationCache(
|
||||
params: QmdRuntimeCollectionValidationCacheContext,
|
||||
nowMs = Date.now(),
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const key = collectionValidationEntryKey(params);
|
||||
const keyHash = buildCollectionValidationCacheHash(params);
|
||||
const collectionConfigHash = buildCollectionConfigHash(params.collections);
|
||||
const createdAtMs = Math.max(0, Math.floor(nowMs));
|
||||
const ttlMs = QMD_RUNTIME_CACHE_COLLECTION_VALIDATION_TTL_MS;
|
||||
const store = collectionValidationStore();
|
||||
await store.register(
|
||||
key,
|
||||
{
|
||||
version: QMD_RUNTIME_CACHE_ENTRY_VERSION,
|
||||
createdAtMs,
|
||||
expiresAtMs: createdAtMs + ttlMs,
|
||||
keyHash,
|
||||
validation: {
|
||||
ok: true,
|
||||
collectionConfigHash,
|
||||
collectionCount: params.collections.length,
|
||||
},
|
||||
},
|
||||
{ ttlMs },
|
||||
);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function clearQmdCollectionValidationCache(
|
||||
params: QmdRuntimeCollectionValidationCacheContext,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const store = collectionValidationStore();
|
||||
await store.delete(collectionValidationEntryKey(params));
|
||||
} catch {
|
||||
// fail open
|
||||
}
|
||||
}
|
||||
|
||||
export async function readQmdMultiCollectionProbeCache(
|
||||
params: QmdRuntimeMultiCollectionProbeCacheContext,
|
||||
nowMs = Date.now(),
|
||||
): Promise<QmdRuntimeCacheResult<QmdRuntimeCacheMultiCollectionProbeEntry>> {
|
||||
try {
|
||||
const store = multiCollectionProbeStore();
|
||||
const key = multiCollectionProbeEntryKey(params);
|
||||
const expectedKeyHash = buildMultiCollectionProbeCacheHash(params);
|
||||
const raw = await store.lookup(key);
|
||||
if (!raw) {
|
||||
return { state: "miss" };
|
||||
}
|
||||
const validated = normalizeMultiCollectionProbeEntry(raw, nowMs, expectedKeyHash);
|
||||
return validated ? { state: "hit", value: validated } : { state: "miss" };
|
||||
} catch {
|
||||
return { state: "miss" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeQmdMultiCollectionProbeCache(
|
||||
params: QmdRuntimeMultiCollectionProbeCacheContext,
|
||||
supported: boolean,
|
||||
nowMs = Date.now(),
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const key = multiCollectionProbeEntryKey(params);
|
||||
const keyHash = buildMultiCollectionProbeCacheHash(params);
|
||||
const createdAtMs = Math.max(0, Math.floor(nowMs));
|
||||
const ttlMs = QMD_RUNTIME_CACHE_MULTI_COLLECTION_PROBE_TTL_MS;
|
||||
const store = multiCollectionProbeStore();
|
||||
await store.register(
|
||||
key,
|
||||
{
|
||||
version: QMD_RUNTIME_CACHE_ENTRY_VERSION,
|
||||
createdAtMs,
|
||||
expiresAtMs: createdAtMs + ttlMs,
|
||||
keyHash,
|
||||
multiCollectionProbe: {
|
||||
supported,
|
||||
},
|
||||
},
|
||||
{ ttlMs },
|
||||
);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function clearQmdMultiCollectionProbeCache(
|
||||
params: QmdRuntimeMultiCollectionProbeCacheContext,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const store = multiCollectionProbeStore();
|
||||
await store.delete(multiCollectionProbeEntryKey(params));
|
||||
} catch {
|
||||
// fail open
|
||||
}
|
||||
}
|
||||
|
|
@ -326,6 +326,10 @@ describe("getMemorySearchManager caching", () => {
|
|||
|
||||
expect(first.manager).toBe(second.manager);
|
||||
expect(createQmdManagerMock.mock.calls).toHaveLength(1);
|
||||
expect(first.debug?.managerCacheState).toBe("cached-full-miss");
|
||||
expect(second.debug?.managerCacheState).toBe("cached-full-hit");
|
||||
expect(first.debug?.qmdIdentityHash).toMatch(/^[0-9a-f]{64}$/);
|
||||
expect(second.debug?.qmdIdentityHash).toBe(first.debug?.qmdIdentityHash);
|
||||
});
|
||||
|
||||
it("keeps the cached QMD manager active when the caller cancels a search", async () => {
|
||||
|
|
@ -806,6 +810,10 @@ describe("getMemorySearchManager caching", () => {
|
|||
const fullManager = requireManager(full);
|
||||
const cliManager = requireManager(cli);
|
||||
|
||||
expect(cli.debug?.managerCacheState).toBe("transient-cli");
|
||||
expect(full.debug?.managerCacheState).toBe("cached-full-miss");
|
||||
expect(full.debug?.qmdIdentityHash).toMatch(/^[0-9a-f]{64}$/);
|
||||
expect(cli.debug?.qmdIdentityHash).toBe(full.debug?.qmdIdentityHash);
|
||||
expect(cliManager).toBe(cliPrimary);
|
||||
expect(cliManager).not.toBe(fullManager);
|
||||
const fullCreateParams = qmdCreateParams();
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { createHash } from "node:crypto";
|
||||
// Memory Core plugin module implements search manager behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
|
|
@ -48,6 +49,24 @@ type QmdManagerOpenFailure = {
|
|||
retryAfterMs: number;
|
||||
};
|
||||
|
||||
type MemorySearchManagerCacheState =
|
||||
| "cached-full-hit"
|
||||
| "cached-full-miss"
|
||||
| "transient-cli"
|
||||
| "transient-status"
|
||||
| "pending-create-wait"
|
||||
| "fallback-builtin"
|
||||
| "recent-failure-cooldown";
|
||||
|
||||
export type MemorySearchManagerDebug = {
|
||||
backend?: "builtin" | "qmd";
|
||||
purpose?: MemorySearchManagerPurpose;
|
||||
managerMs?: number;
|
||||
managerCacheState?: MemorySearchManagerCacheState;
|
||||
qmdIdentityHash?: string;
|
||||
failureCode?: "qmd-unavailable";
|
||||
};
|
||||
|
||||
type MemorySearchManagerCacheStore = {
|
||||
qmdManagerCache: Map<string, CachedQmdManagerEntry>;
|
||||
pendingQmdManagerCreates: Map<string, PendingQmdManagerCreate>;
|
||||
|
|
@ -109,6 +128,7 @@ function loadQmdManagerModule() {
|
|||
export type MemorySearchManagerResult = {
|
||||
manager: Maybe<MemorySearchManager>;
|
||||
error?: string;
|
||||
debug?: MemorySearchManagerDebug;
|
||||
};
|
||||
|
||||
export type MemorySearchManagerPurpose = "default" | "status" | "cli";
|
||||
|
|
@ -149,11 +169,42 @@ function clearQmdManagerOpenFailure(scopeKey: string, identityKey: string): void
|
|||
}
|
||||
}
|
||||
|
||||
function hashQmdManagerIdentity(identityKey: string): string {
|
||||
return createHash("sha256").update(identityKey).digest("hex");
|
||||
}
|
||||
|
||||
function applyManagerDebug(
|
||||
result: MemorySearchManagerResult,
|
||||
debug: MemorySearchManagerDebug,
|
||||
): MemorySearchManagerResult {
|
||||
if (result.debug && Object.keys(result.debug).length > 0 && Object.keys(debug).length === 0) {
|
||||
return result;
|
||||
}
|
||||
return {
|
||||
...result,
|
||||
debug: {
|
||||
...result.debug,
|
||||
...debug,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function getMemorySearchManager(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
purpose?: MemorySearchManagerPurpose;
|
||||
}): Promise<MemorySearchManagerResult> {
|
||||
const acquireStartedAt = Date.now();
|
||||
const purpose = params.purpose ?? "default";
|
||||
const finish = (
|
||||
result: MemorySearchManagerResult,
|
||||
debug: MemorySearchManagerDebug,
|
||||
): MemorySearchManagerResult =>
|
||||
applyManagerDebug(result, {
|
||||
purpose,
|
||||
managerMs: Math.max(0, Date.now() - acquireStartedAt),
|
||||
...debug,
|
||||
});
|
||||
const resolved = resolveMemoryBackendConfig(params);
|
||||
if (resolved.backend === "qmd" && resolved.qmd) {
|
||||
const qmdResolved = resolved.qmd;
|
||||
|
|
@ -163,6 +214,7 @@ export async function getMemorySearchManager(params: {
|
|||
const transient = params.purpose === "status" || params.purpose === "cli";
|
||||
const scopeKey = buildQmdManagerScopeKey(normalizedAgentId);
|
||||
const identityKey = buildQmdManagerIdentityKey(normalizedAgentId, qmdResolved, runtimeConfig);
|
||||
const debugIdentityHash = hashQmdManagerIdentity(identityKey);
|
||||
|
||||
const createPrimaryQmdManager = async (
|
||||
mode: "full" | "status" | "cli",
|
||||
|
|
@ -254,10 +306,24 @@ export async function getMemorySearchManager(params: {
|
|||
// Status callers often close the manager they receive. Wrap the live
|
||||
// full manager with a no-op close so health/status probes do not tear
|
||||
// down the active QMD manager for the process.
|
||||
return { manager: new BorrowedMemoryManager(cached.manager) };
|
||||
return finish(
|
||||
{ manager: new BorrowedMemoryManager(cached.manager) },
|
||||
{
|
||||
backend: "qmd",
|
||||
managerCacheState: "cached-full-hit",
|
||||
qmdIdentityHash: debugIdentityHash,
|
||||
},
|
||||
);
|
||||
}
|
||||
if (params.purpose !== "cli") {
|
||||
return { manager: cached.manager };
|
||||
return finish(
|
||||
{ manager: cached.manager },
|
||||
{
|
||||
backend: "qmd",
|
||||
managerCacheState: "cached-full-hit",
|
||||
qmdIdentityHash: debugIdentityHash,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -266,20 +332,44 @@ export async function getMemorySearchManager(params: {
|
|||
params.purpose === "cli" ? "cli" : "status",
|
||||
);
|
||||
return manager
|
||||
? { manager }
|
||||
: await getBuiltinMemorySearchManagerAfterQmdFailure(params, failureReason);
|
||||
? finish(
|
||||
{ manager },
|
||||
{
|
||||
backend: "qmd",
|
||||
managerCacheState: params.purpose === "cli" ? "transient-cli" : "transient-status",
|
||||
qmdIdentityHash: debugIdentityHash,
|
||||
},
|
||||
)
|
||||
: finish(await getBuiltinMemorySearchManagerAfterQmdFailure(params, failureReason), {
|
||||
backend: "qmd",
|
||||
managerCacheState: "fallback-builtin",
|
||||
qmdIdentityHash: debugIdentityHash,
|
||||
failureCode: "qmd-unavailable",
|
||||
});
|
||||
}
|
||||
|
||||
const recentFailure = getActiveQmdManagerOpenFailure(scopeKey, identityKey);
|
||||
if (recentFailure) {
|
||||
log.debug?.(`qmd memory unavailable; using builtin during cooldown: ${recentFailure.reason}`);
|
||||
return await getBuiltinMemorySearchManagerAfterQmdFailure(params, recentFailure.reason);
|
||||
return finish(
|
||||
await getBuiltinMemorySearchManagerAfterQmdFailure(params, recentFailure.reason),
|
||||
{
|
||||
backend: "qmd",
|
||||
managerCacheState: "recent-failure-cooldown",
|
||||
qmdIdentityHash: debugIdentityHash,
|
||||
failureCode: "qmd-unavailable",
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const pending = PENDING_QMD_MANAGER_CREATES.get(scopeKey);
|
||||
if (pending) {
|
||||
await pending.promise;
|
||||
return await getMemorySearchManager(params);
|
||||
return finish(await getMemorySearchManager(params), {
|
||||
backend: "qmd",
|
||||
managerCacheState: "pending-create-wait",
|
||||
qmdIdentityHash: debugIdentityHash,
|
||||
});
|
||||
}
|
||||
|
||||
let pendingFailureReason: string | undefined;
|
||||
|
|
@ -309,11 +399,25 @@ export async function getMemorySearchManager(params: {
|
|||
PENDING_QMD_MANAGER_CREATES.set(scopeKey, pendingCreate);
|
||||
const manager = await pendingCreate.promise;
|
||||
return manager
|
||||
? { manager }
|
||||
: await getBuiltinMemorySearchManagerAfterQmdFailure(params, pendingFailureReason);
|
||||
? finish(
|
||||
{ manager },
|
||||
{
|
||||
backend: "qmd",
|
||||
managerCacheState: "cached-full-miss",
|
||||
qmdIdentityHash: debugIdentityHash,
|
||||
},
|
||||
)
|
||||
: finish(await getBuiltinMemorySearchManagerAfterQmdFailure(params, pendingFailureReason), {
|
||||
backend: "qmd",
|
||||
managerCacheState: "fallback-builtin",
|
||||
qmdIdentityHash: debugIdentityHash,
|
||||
failureCode: "qmd-unavailable",
|
||||
});
|
||||
}
|
||||
|
||||
return await getBuiltinMemorySearchManager(params);
|
||||
return finish(await getBuiltinMemorySearchManager(params), {
|
||||
backend: "builtin",
|
||||
});
|
||||
}
|
||||
|
||||
async function getBuiltinMemorySearchManagerAfterQmdFailure(
|
||||
|
|
|
|||
44
extensions/memory-core/src/runtime-provider.test.ts
Normal file
44
extensions/memory-core/src/runtime-provider.test.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
// Memory Core provider tests cover plugin runtime integration.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-runtime-core";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const managerDebug = {
|
||||
backend: "qmd" as const,
|
||||
purpose: "default" as const,
|
||||
managerMs: 7,
|
||||
managerCacheState: "cached-full-hit" as const,
|
||||
qmdIdentityHash: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
};
|
||||
|
||||
const getMemorySearchManagerMock = vi.hoisted(() =>
|
||||
vi.fn(async () => ({
|
||||
manager: null,
|
||||
debug: managerDebug,
|
||||
error: undefined,
|
||||
})),
|
||||
);
|
||||
|
||||
vi.mock("./memory/index.js", () => ({
|
||||
closeAllMemorySearchManagers: vi.fn(async () => {}),
|
||||
closeMemorySearchManager: vi.fn(async () => {}),
|
||||
getMemorySearchManager: getMemorySearchManagerMock,
|
||||
}));
|
||||
|
||||
import { memoryRuntime } from "./runtime-provider.js";
|
||||
|
||||
describe("memoryRuntime", () => {
|
||||
it("preserves manager debug metadata", async () => {
|
||||
const cfg = {} as OpenClawConfig;
|
||||
|
||||
const result = await memoryRuntime.getMemorySearchManager({
|
||||
cfg,
|
||||
agentId: "main",
|
||||
});
|
||||
|
||||
expect(result.debug).toEqual(managerDebug);
|
||||
expect(getMemorySearchManagerMock).toHaveBeenCalledWith({
|
||||
cfg,
|
||||
agentId: "main",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -9,9 +9,10 @@ import {
|
|||
|
||||
export const memoryRuntime: MemoryPluginRuntime = {
|
||||
async getMemorySearchManager(params) {
|
||||
const { manager, error } = await getMemorySearchManager(params);
|
||||
const { manager, debug, error } = await getMemorySearchManager(params);
|
||||
return {
|
||||
manager,
|
||||
debug,
|
||||
error,
|
||||
};
|
||||
},
|
||||
|
|
|
|||
|
|
@ -67,18 +67,28 @@ export async function getMemoryManagerContextWithPurpose(params: {
|
|||
}): Promise<
|
||||
| {
|
||||
manager: NonNullable<MemorySearchManagerResult["manager"]>;
|
||||
debug?: NonNullable<MemorySearchManagerResult["debug"]>;
|
||||
}
|
||||
| {
|
||||
error: string | undefined;
|
||||
}
|
||||
> {
|
||||
const { getMemorySearchManager } = await loadMemoryToolRuntime();
|
||||
const { manager, error } = await getMemorySearchManager({
|
||||
const startedAt = Date.now();
|
||||
const { manager, debug, error } = await getMemorySearchManager({
|
||||
cfg: params.cfg,
|
||||
agentId: params.agentId,
|
||||
purpose: params.purpose,
|
||||
});
|
||||
return manager ? { manager } : { error };
|
||||
return manager
|
||||
? {
|
||||
manager,
|
||||
debug: {
|
||||
...debug,
|
||||
managerMs: debug?.managerMs ?? Math.max(0, Date.now() - startedAt),
|
||||
},
|
||||
}
|
||||
: { error };
|
||||
}
|
||||
|
||||
export function createMemoryTool(params: {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
// Memory Core tests cover tools plugin behavior.
|
||||
import type { MemorySearchRuntimeDebug } from "openclaw/plugin-sdk/memory-core-host-runtime-files";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
getMemoryCloseMockCalls,
|
||||
|
|
@ -381,6 +382,95 @@ describe("memory_search unavailable payloads", () => {
|
|||
expect(searchCalls).toBe(2);
|
||||
});
|
||||
|
||||
it("merges qmd runtime debug across zero-hit retry attempts", async () => {
|
||||
setMemoryBackend("qmd");
|
||||
let searchCalls = 0;
|
||||
setMemorySearchImpl(async (opts) => {
|
||||
searchCalls += 1;
|
||||
if (searchCalls === 1) {
|
||||
opts?.onDebug?.({
|
||||
backend: "qmd",
|
||||
configuredMode: "search",
|
||||
effectiveMode: "search",
|
||||
qmd: {
|
||||
collectionValidation: {
|
||||
cacheState: "hit",
|
||||
elapsedMs: 2,
|
||||
collectionCount: 2,
|
||||
listCalls: 0,
|
||||
showCalls: 0,
|
||||
},
|
||||
multiCollectionProbe: {
|
||||
cacheState: "hit",
|
||||
elapsedMs: 1,
|
||||
supported: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
return [];
|
||||
}
|
||||
opts?.onDebug?.({
|
||||
backend: "qmd",
|
||||
configuredMode: "search",
|
||||
effectiveMode: "query",
|
||||
fallback: "unsupported-search-flags",
|
||||
qmd: {
|
||||
searchPlan: {
|
||||
command: "query",
|
||||
collectionCount: 2,
|
||||
groupCount: 2,
|
||||
sources: ["memory", "sessions"],
|
||||
},
|
||||
},
|
||||
});
|
||||
return [
|
||||
{
|
||||
path: "MEMORY.md",
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
score: 0.9,
|
||||
snippet: "Thread-hidden codename: ORBIT-22.",
|
||||
source: "memory" as const,
|
||||
},
|
||||
];
|
||||
});
|
||||
|
||||
const tool = createMemorySearchToolOrThrow({
|
||||
config: {
|
||||
agents: { list: [{ id: "main", default: true }] },
|
||||
memory: { backend: "qmd", citations: "off" },
|
||||
},
|
||||
});
|
||||
const result = await tool.execute("zero-hit-debug-retry", {
|
||||
query: "hidden thread codename",
|
||||
});
|
||||
const details = result.details as {
|
||||
debug?: {
|
||||
effectiveMode?: string;
|
||||
fallback?: string;
|
||||
qmd?: MemorySearchRuntimeDebug["qmd"];
|
||||
};
|
||||
};
|
||||
|
||||
expect(searchCalls).toBe(2);
|
||||
expect(details.debug?.effectiveMode).toBe("query");
|
||||
expect(details.debug?.fallback).toBe("unsupported-search-flags");
|
||||
expect(details.debug?.qmd?.collectionValidation).toMatchObject({
|
||||
cacheState: "hit",
|
||||
collectionCount: 2,
|
||||
});
|
||||
expect(details.debug?.qmd?.multiCollectionProbe).toMatchObject({
|
||||
cacheState: "hit",
|
||||
supported: true,
|
||||
});
|
||||
expect(details.debug?.qmd?.searchPlan).toEqual({
|
||||
command: "query",
|
||||
collectionCount: 2,
|
||||
groupCount: 2,
|
||||
sources: ["memory", "sessions"],
|
||||
});
|
||||
});
|
||||
|
||||
it("returns unavailable metadata when the index identity is paused", async () => {
|
||||
let searchCalls = 0;
|
||||
setMemorySearchImpl(async () => {
|
||||
|
|
@ -422,6 +512,14 @@ describe("memory_search unavailable payloads", () => {
|
|||
configuredMode: opts.qmdSearchModeOverride ?? "query",
|
||||
effectiveMode: "query",
|
||||
fallback: "unsupported-search-flags",
|
||||
qmd: {
|
||||
searchPlan: {
|
||||
command: "query",
|
||||
collectionCount: 2,
|
||||
groupCount: 2,
|
||||
sources: ["memory", "sessions"],
|
||||
},
|
||||
},
|
||||
});
|
||||
return [
|
||||
{
|
||||
|
|
@ -470,6 +568,18 @@ describe("memory_search unavailable payloads", () => {
|
|||
fallback?: unknown;
|
||||
hits?: unknown;
|
||||
searchMs?: number;
|
||||
toolMs?: number;
|
||||
managerMs?: number;
|
||||
outsideSearchMs?: number;
|
||||
managerCacheState?: unknown;
|
||||
qmd?: {
|
||||
searchPlan?: {
|
||||
command?: unknown;
|
||||
collectionCount?: unknown;
|
||||
groupCount?: unknown;
|
||||
sources?: unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
expect(details.mode).toBe("query");
|
||||
|
|
@ -479,6 +589,94 @@ describe("memory_search unavailable payloads", () => {
|
|||
expect(details.debug?.fallback).toBe("unsupported-search-flags");
|
||||
expect(details.debug?.hits).toBe(1);
|
||||
expect(details.debug?.searchMs).toBeGreaterThanOrEqual(0);
|
||||
expect(details.debug?.toolMs).toBeGreaterThanOrEqual(details.debug?.searchMs ?? 0);
|
||||
expect(details.debug?.outsideSearchMs).toBeGreaterThanOrEqual(0);
|
||||
expect(details.debug?.managerMs).toBeGreaterThanOrEqual(0);
|
||||
expect(details.debug?.managerCacheState).toBeUndefined();
|
||||
expect(details.debug?.qmd?.searchPlan).toEqual({
|
||||
command: "query",
|
||||
collectionCount: 2,
|
||||
groupCount: 2,
|
||||
sources: ["memory", "sessions"],
|
||||
});
|
||||
});
|
||||
|
||||
it("includes manager acquisition timing and cache-state debug payload", async () => {
|
||||
setMemorySearchManagerImpl(
|
||||
async () =>
|
||||
({
|
||||
manager: {
|
||||
search: vi.fn(async () => {
|
||||
return [
|
||||
{
|
||||
path: "MEMORY.md",
|
||||
startLine: 1,
|
||||
endLine: 2,
|
||||
score: 0.9,
|
||||
snippet: "ramen",
|
||||
source: "memory",
|
||||
},
|
||||
];
|
||||
}),
|
||||
readFile: vi.fn(),
|
||||
status: vi.fn(() => ({
|
||||
backend: "qmd",
|
||||
provider: "qmd",
|
||||
model: "qmd",
|
||||
requestedProvider: "qmd",
|
||||
files: 0,
|
||||
chunks: 0,
|
||||
dirty: false,
|
||||
workspaceDir: "/tmp/workspace",
|
||||
dbPath: "/tmp/workspace/index.sqlite",
|
||||
sources: ["memory"],
|
||||
sourceCounts: [{ source: "memory", files: 0, chunks: 0 }],
|
||||
})),
|
||||
sync: vi.fn(async () => {}),
|
||||
probeEmbeddingAvailability: vi.fn(async () => ({ ok: true })),
|
||||
probeVectorAvailability: vi.fn(async () => true),
|
||||
},
|
||||
debug: {
|
||||
managerMs: 17,
|
||||
managerCacheState: "cached-full-hit",
|
||||
},
|
||||
}) as any,
|
||||
);
|
||||
setMemorySearchImpl(async () => [
|
||||
{
|
||||
path: "MEMORY.md",
|
||||
startLine: 1,
|
||||
endLine: 2,
|
||||
score: 0.9,
|
||||
snippet: "ramen",
|
||||
source: "memory",
|
||||
},
|
||||
]);
|
||||
|
||||
const tool = createMemorySearchToolOrThrow({
|
||||
config: {
|
||||
agents: { list: [{ id: "main", default: true }] },
|
||||
memory: { backend: "qmd" },
|
||||
},
|
||||
});
|
||||
const result = await tool.execute("manager-debug", { query: "favorite food" });
|
||||
const details = result.details as {
|
||||
debug?: {
|
||||
backend?: string;
|
||||
managerMs?: number;
|
||||
toolMs?: number;
|
||||
outsideSearchMs?: number;
|
||||
managerCacheState?: string;
|
||||
hits?: number;
|
||||
searchMs?: number;
|
||||
};
|
||||
};
|
||||
|
||||
expect(details.debug?.backend).toBe("qmd");
|
||||
expect(details.debug?.managerMs).toBe(17);
|
||||
expect(details.debug?.toolMs).toBeGreaterThanOrEqual(details.debug?.searchMs ?? 0);
|
||||
expect(details.debug?.outsideSearchMs).toBeGreaterThanOrEqual(0);
|
||||
expect(details.debug?.managerCacheState).toBe("cached-full-hit");
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -44,12 +44,35 @@ type MemorySearchToolResult =
|
|||
| MemoryCorpusSearchResult;
|
||||
type MemoryManagerContext = Awaited<ReturnType<typeof getMemoryManagerContextWithPurpose>>;
|
||||
type ActiveMemoryManagerContext = Extract<MemoryManagerContext, { manager: unknown }>;
|
||||
type QmdRuntimeDebug = NonNullable<MemorySearchRuntimeDebug["qmd"]>;
|
||||
|
||||
const MEMORY_SEARCH_TOOL_TIMEOUT_MS = 15_000;
|
||||
const MEMORY_SEARCH_TOOL_COOLDOWN_MS = 60_000;
|
||||
|
||||
const memorySearchToolCooldowns = new Map<string, { until: number; error: string }>();
|
||||
|
||||
function mergeQmdRuntimeDebug(
|
||||
entries: readonly MemorySearchRuntimeDebug[],
|
||||
): MemorySearchRuntimeDebug["qmd"] | undefined {
|
||||
const merged: QmdRuntimeDebug = {};
|
||||
for (const entry of entries) {
|
||||
const qmd = entry.qmd;
|
||||
if (!qmd) {
|
||||
continue;
|
||||
}
|
||||
if (!merged.collectionValidation && qmd.collectionValidation) {
|
||||
merged.collectionValidation = qmd.collectionValidation;
|
||||
}
|
||||
if (qmd.multiCollectionProbe) {
|
||||
merged.multiCollectionProbe = qmd.multiCollectionProbe;
|
||||
}
|
||||
if (qmd.searchPlan) {
|
||||
merged.searchPlan = qmd.searchPlan;
|
||||
}
|
||||
}
|
||||
return Object.keys(merged).length > 0 ? merged : undefined;
|
||||
}
|
||||
|
||||
function resolveMemorySearchToolCooldownKey(options: {
|
||||
agentId?: string;
|
||||
agentSessionKey?: string;
|
||||
|
|
@ -415,6 +438,7 @@ export function createMemorySearchTool(options: {
|
|||
const outcome = await runMemorySearchToolWithDeadline({
|
||||
timeoutMs: MEMORY_SEARCH_TOOL_TIMEOUT_MS,
|
||||
run: async (deadlineSignal) => {
|
||||
const toolStartedAt = Date.now();
|
||||
const { resolveMemoryBackendConfig } = await loadMemoryToolRuntime();
|
||||
const shouldQuerySupplements = requestedCorpus === "wiki" || requestedCorpus === "all";
|
||||
const shouldQueryMemory = requestedCorpus !== "wiki" && !cooldown;
|
||||
|
|
@ -471,13 +495,20 @@ export function createMemorySearchTool(options: {
|
|||
let fallback: unknown;
|
||||
let searchMode: string | undefined;
|
||||
let pausedIndexIdentityReason: string | undefined;
|
||||
let managerMs: number | undefined;
|
||||
let managerCacheState: string | undefined;
|
||||
let searchDebug:
|
||||
| {
|
||||
backend: string;
|
||||
configuredMode?: string;
|
||||
effectiveMode?: string;
|
||||
fallback?: string;
|
||||
toolMs?: number;
|
||||
managerMs?: number;
|
||||
outsideSearchMs?: number;
|
||||
searchMs: number;
|
||||
managerCacheState?: string;
|
||||
qmd?: MemorySearchRuntimeDebug["qmd"];
|
||||
hits: number;
|
||||
}
|
||||
| undefined;
|
||||
|
|
@ -506,6 +537,8 @@ export function createMemorySearchTool(options: {
|
|||
},
|
||||
...(searchSources ? { sources: searchSources } : {}),
|
||||
};
|
||||
managerMs = memory.debug?.managerMs;
|
||||
managerCacheState = memory.debug?.managerCacheState;
|
||||
try {
|
||||
rawResults = await activeMemory.manager.search(query, searchOptions);
|
||||
} catch (error) {
|
||||
|
|
@ -522,6 +555,8 @@ export function createMemorySearchTool(options: {
|
|||
if ("error" in refreshed) {
|
||||
throw error;
|
||||
}
|
||||
managerMs = refreshed.debug?.managerMs;
|
||||
managerCacheState = refreshed.debug?.managerCacheState;
|
||||
activeMemory = refreshed;
|
||||
rawResults = await activeMemory.manager.search(query, searchOptions);
|
||||
}
|
||||
|
|
@ -580,7 +615,9 @@ export function createMemorySearchTool(options: {
|
|||
model = status.model;
|
||||
fallback = status.fallback;
|
||||
const latestDebug = runtimeDebug.at(-1);
|
||||
const qmdDebug = mergeQmdRuntimeDebug(runtimeDebug);
|
||||
searchMode = latestDebug?.effectiveMode;
|
||||
const searchMs = Math.max(0, Date.now() - searchStartedAt);
|
||||
searchDebug = {
|
||||
backend: status.backend,
|
||||
configuredMode: latestDebug?.configuredMode,
|
||||
|
|
@ -589,7 +626,10 @@ export function createMemorySearchTool(options: {
|
|||
? (latestDebug?.effectiveMode ?? latestDebug?.configuredMode)
|
||||
: "n/a",
|
||||
fallback: latestDebug?.fallback,
|
||||
searchMs: Math.max(0, Date.now() - searchStartedAt),
|
||||
managerMs,
|
||||
searchMs,
|
||||
managerCacheState,
|
||||
qmd: qmdDebug,
|
||||
hits: rawResults.length,
|
||||
};
|
||||
});
|
||||
|
|
@ -620,6 +660,14 @@ export function createMemorySearchTool(options: {
|
|||
maxResults: effectiveMax,
|
||||
balanceCorpora: requestedCorpus === "all",
|
||||
});
|
||||
if (searchDebug) {
|
||||
const finalToolMs = Math.max(0, Date.now() - toolStartedAt);
|
||||
searchDebug = {
|
||||
...searchDebug,
|
||||
toolMs: finalToolMs,
|
||||
outsideSearchMs: Math.max(0, finalToolMs - searchDebug.searchMs),
|
||||
};
|
||||
}
|
||||
return jsonResult({
|
||||
results,
|
||||
provider,
|
||||
|
|
|
|||
|
|
@ -55,11 +55,39 @@ export type MemorySyncParams = {
|
|||
};
|
||||
|
||||
/** Runtime backend/mode diagnostics for memory search. */
|
||||
export type MemorySearchRuntimeQmdCollectionValidationDebug = {
|
||||
cacheState?: "hit" | "miss" | "write" | "bypass-force" | "error";
|
||||
elapsedMs: number;
|
||||
collectionCount: number;
|
||||
listCalls?: number;
|
||||
showCalls?: number;
|
||||
};
|
||||
|
||||
export type MemorySearchRuntimeQmdMultiCollectionProbeDebug = {
|
||||
cacheState?: "hit" | "miss" | "write" | "error";
|
||||
elapsedMs: number;
|
||||
supported: boolean;
|
||||
};
|
||||
|
||||
export type MemorySearchRuntimeQmdSearchPlanDebug = {
|
||||
command?: "query" | "search" | "vsearch";
|
||||
collectionCount?: number;
|
||||
groupCount?: number;
|
||||
sources?: MemorySource[];
|
||||
};
|
||||
|
||||
export type MemorySearchRuntimeQmdDebug = {
|
||||
collectionValidation?: MemorySearchRuntimeQmdCollectionValidationDebug;
|
||||
multiCollectionProbe?: MemorySearchRuntimeQmdMultiCollectionProbeDebug;
|
||||
searchPlan?: MemorySearchRuntimeQmdSearchPlanDebug;
|
||||
};
|
||||
|
||||
export type MemorySearchRuntimeDebug = {
|
||||
backend: "builtin" | "qmd";
|
||||
configuredMode?: string;
|
||||
effectiveMode?: string;
|
||||
fallback?: string;
|
||||
qmd?: MemorySearchRuntimeQmdDebug;
|
||||
};
|
||||
|
||||
/** Result of reading a memory file, optionally paginated/truncated. */
|
||||
|
|
|
|||
|
|
@ -131,7 +131,7 @@ type FacadeModule = {
|
|||
getMemorySearchManager: (params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
purpose?: "default" | "status";
|
||||
purpose?: "default" | "status" | "cli";
|
||||
}) => Promise<{
|
||||
manager: MemorySearchManager | null;
|
||||
error?: string;
|
||||
|
|
|
|||
|
|
@ -102,6 +102,21 @@ export type MemoryPluginRuntime = {
|
|||
purpose?: "default" | "status" | "cli";
|
||||
}): Promise<{
|
||||
manager: RegisteredMemorySearchManager | null;
|
||||
debug?: {
|
||||
backend?: "builtin" | "qmd";
|
||||
purpose?: "default" | "status" | "cli";
|
||||
managerMs?: number;
|
||||
managerCacheState?:
|
||||
| "cached-full-hit"
|
||||
| "cached-full-miss"
|
||||
| "transient-cli"
|
||||
| "transient-status"
|
||||
| "pending-create-wait"
|
||||
| "fallback-builtin"
|
||||
| "recent-failure-cooldown";
|
||||
qmdIdentityHash?: string;
|
||||
failureCode?: "qmd-unavailable";
|
||||
};
|
||||
error?: string;
|
||||
}>;
|
||||
resolveMemoryBackendConfig(params: {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue