diff --git a/docs/concepts/dreaming.md b/docs/concepts/dreaming.md index 5fbf26509134..71c2bb49ae03 100644 --- a/docs/concepts/dreaming.md +++ b/docs/concepts/dreaming.md @@ -92,7 +92,7 @@ source reference. The model returns operation decisions, not replacement memory prose. The memory writer applies those decisions to the existing file using each candidate's -bounded, sourced entry. An accepted rewrite must: +bounded, sourced entry. An accepted rewrite or append compaction must: - preserve prior entries within `phases.deep.maxPriorEntryLossFraction` - include every promoted candidate's `Source: path#Lx-Ly` reference @@ -276,7 +276,7 @@ All settings live under `plugins.entries.memory-core.config.dreaming`. Enable or disable the dreaming sweep. - Reject a consolidation rewrite when it removes more than this fraction of prior entries. + Reject a consolidation rewrite or append compaction when it removes more than this fraction of prior entries. Append compaction only removes whole machine-generated promotion sections. Cron cadence for the full dreaming sweep. diff --git a/docs/concepts/memory-architecture.md b/docs/concepts/memory-architecture.md index efbc8138c0a0..94de37374d24 100644 --- a/docs/concepts/memory-architecture.md +++ b/docs/concepts/memory-architecture.md @@ -185,7 +185,12 @@ touching long-term memory. The consolidation output is accepted only if it passes structural validation, stays within the bootstrap file budget, and does not lose more than a bounded fraction of existing entries. A rejected rewrite falls back -to the previous append-only behavior for that sweep. +to append-only behavior for that sweep. Promotion uses the smallest configured +per-file bootstrap limit among agents sharing the workspace, capped by the +writer's own limit. If an append still cannot fit after older generated +sections are removed, the writer preserves `MEMORY.md` unchanged and leaves +the candidates eligible for a later sweep instead of committing an oversized +file. **Write safety.** Replacing `MEMORY.md` uses optimistic concurrency: the content hash captured when consolidation input was built is re-checked diff --git a/docs/reference/memory-config.md b/docs/reference/memory-config.md index 89f230802bf7..911a723e2de1 100644 --- a/docs/reference/memory-config.md +++ b/docs/reference/memory-config.md @@ -737,7 +737,7 @@ For conceptual behavior and slash commands, see [Dreaming](/concepts/dreaming). | `frequency` | `string` | `0 3 * * *` | Optional cron cadence for the full dreaming sweep | | `model` | `string` | default model | Optional Dream Diary subagent model override | | `phases.deep.maxPromotedSnippetTokens` | `number` | `160` | Maximum estimated tokens kept from each short-term recall snippet promoted into `MEMORY.md`; provenance metadata remains visible | -| `phases.deep.maxPriorEntryLossFraction` | `number` | `0.25` | Reject a consolidation rewrite that removes more than this fraction of prior entries | +| `phases.deep.maxPriorEntryLossFraction` | `number` | `0.25` | Reject consolidation or append compaction that removes more than this fraction of prior entries | ### Example diff --git a/extensions/memory-core/src/cli-index-search.runtime.ts b/extensions/memory-core/src/cli-index-search.runtime.ts index 1d97f44837f6..aa79d6eec367 100644 --- a/extensions/memory-core/src/cli-index-search.runtime.ts +++ b/extensions/memory-core/src/cli-index-search.runtime.ts @@ -2,7 +2,7 @@ import path from "node:path"; import { resolveMemorySearchStaleness } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; import { resolveMemoryDreamingConfig, - resolveMemoryDreamingWorkspaces, + resolveMemoryDreamingWorkspace, resolveMemoryDeepDreamingConfig, } from "openclaw/plugin-sdk/memory-core-host-status"; import { @@ -32,6 +32,7 @@ import type { MemoryPromoteExplainOptions, MemorySearchCommandOptions, } from "./cli.types.js"; +import { resolveMemoryPromotionFileMaxChars } from "./memory-budget.js"; import { forgetMemoryEntries } from "./memory-forget.js"; import { captureMemoryRebuildNotice } from "./memory-rebuild-notice.js"; import { formatMemoryVectorDegradedWriteReason } from "./memory/manager-vector-warning.js"; @@ -415,11 +416,12 @@ export async function runMemoryPromote( let applyResult: Awaited> | undefined; if (opts.apply) { try { + const workspaceAgentIds = resolveMemoryDreamingWorkspace(cfg, workspaceDir)?.agentIds ?? [ + agentId, + ]; applyResult = await applyShortTermPromotions({ agentId, - workspaceAgentIds: resolveMemoryDreamingWorkspaces(cfg).find( - (workspace) => path.resolve(workspace.workspaceDir) === path.resolve(workspaceDir), - )?.agentIds, + workspaceAgentIds, workspaceDir, candidates, limit: opts.limit, @@ -428,6 +430,11 @@ export async function runMemoryPromote( minUniqueQueries: opts.minUniqueQueries ?? dreaming.minUniqueQueries, maxAgeDays: dreaming.maxAgeDays, maxPromotedSnippetTokens: dreaming.maxPromotedSnippetTokens, + maxPriorEntryLossFraction: dreaming.maxPriorEntryLossFraction, + memoryFileMaxChars: resolveMemoryPromotionFileMaxChars({ + cfg, + agentIds: workspaceAgentIds, + }), timezone: dreaming.timezone, }); } catch (err) { diff --git a/extensions/memory-core/src/cli.test.ts b/extensions/memory-core/src/cli.test.ts index e583b6efb532..dfef59d31c28 100644 --- a/extensions/memory-core/src/cli.test.ts +++ b/extensions/memory-core/src/cli.test.ts @@ -3477,6 +3477,144 @@ describe("memory cli", () => { }); }); + it("honors the configured prior-entry loss limit during CLI promotion", async () => { + await withTempWorkspace(async (workspaceDir) => { + const promotionSection = (date: string, index: number) => + [ + `## Promoted From Short-Term Memory (${date})`, + ``, + `- ${"x".repeat(350)}`, + "", + ].join("\n"); + await fs.writeFile( + path.join(workspaceDir, "MEMORY.md"), + [0, 1, 2, 3] + .map((index) => promotionSection(`2026-04-${String(index + 1).padStart(2, "0")}`, index)) + .join("\n"), + "utf-8", + ); + await writeDailyMemoryNote(workspaceDir, "2026-04-10", ["Retain the release checklist."]); + await recordShortTermRecalls({ + workspaceDir, + query: "release checklist", + results: [ + { + path: "memory/2026-04-10.md", + startLine: 1, + endLine: 1, + score: 0.91, + snippet: "Retain the release checklist.", + source: "memory", + }, + ], + }); + getRuntimeConfig.mockReturnValue({ + agents: { + list: [{ id: "main", default: true, workspace: workspaceDir, bootstrapMaxChars: 1_400 }], + }, + plugins: { + entries: { + "memory-core": { + config: { dreaming: { phases: { deep: { maxPriorEntryLossFraction: 1 } } } }, + }, + }, + }, + }); + const close = vi.fn(async () => {}); + mockManager({ status: () => makeMemoryStatus({ workspaceDir }), close }); + + await runMemoryCli([ + "promote", + "--apply", + "--min-score", + "0", + "--min-recall-count", + "0", + "--min-unique-queries", + "0", + ]); + + const memory = await fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8"); + expect(memory).toContain("Retain the release checklist."); + expect(memory.length).toBeLessThanOrEqual(1_400); + expect(close).toHaveBeenCalled(); + }); + }); + + it.runIf(process.platform !== "win32")( + "uses the smallest bootstrap cap across CLI workspace symlink aliases", + async () => { + await withTempWorkspace(async (workspaceDir) => { + const workspaceAliasDir = `${workspaceDir}-alias`; + await fs.symlink(workspaceDir, workspaceAliasDir, "dir"); + const existingMemory = `# Long-Term Memory\n\n${"x".repeat(9_100)}\n`; + await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), existingMemory, "utf-8"); + await writeDailyMemoryNote(workspaceDir, "2026-04-01", ["Shared workspace fact."]); + await recordShortTermRecalls({ + workspaceDir: workspaceAliasDir, + query: "shared workspace", + results: [ + { + path: "memory/2026-04-01.md", + startLine: 1, + endLine: 1, + score: 0.91, + snippet: "Shared workspace fact.", + source: "memory", + }, + ], + }); + getRuntimeConfig.mockReturnValue({ + agents: { + list: [ + { + id: "alpha", + default: true, + workspace: workspaceDir, + bootstrapMaxChars: 9_000, + }, + { id: "beta", workspace: workspaceAliasDir, bootstrapMaxChars: 12_000 }, + ], + }, + }); + const close = vi.fn(async () => {}); + mockManager({ + status: () => makeMemoryStatus({ workspaceDir: workspaceAliasDir }), + close, + }); + + const writeJson = spyRuntimeJson(defaultRuntime); + await runMemoryCli([ + "promote", + "--agent", + "beta", + "--apply", + "--json", + "--min-score", + "0", + "--min-recall-count", + "0", + "--min-unique-queries", + "0", + ]); + + const payload = firstWrittenJsonArg<{ + candidates: unknown[]; + apply: { appliedCandidates: unknown[]; rejectedCandidates: Array<{ reason: string }> }; + }>(writeJson); + expect(payload?.candidates).toHaveLength(1); + expect(payload?.apply.appliedCandidates).toEqual([]); + expect(payload?.apply.rejectedCandidates).toEqual([ + expect.objectContaining({ reason: expect.stringContaining("budget") }), + ]); + expect(await fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8")).toBe( + existingMemory, + ); + expect(close).toHaveBeenCalled(); + }); + }, + ); + it("names apply-time rejections without ranking blocked origins", async () => { await withTempWorkspace(async (workspaceDir) => { const relativePath = "memory/2026-04-02.md"; diff --git a/extensions/memory-core/src/dreaming.ts b/extensions/memory-core/src/dreaming.ts index 42f27ee93765..345c575a0843 100644 --- a/extensions/memory-core/src/dreaming.ts +++ b/extensions/memory-core/src/dreaming.ts @@ -24,6 +24,7 @@ import { peekSystemEventEntries } from "openclaw/plugin-sdk/system-event-runtime import { appendFailedDreamingEvent } from "./dreaming-events.js"; import type { NarrativePhaseData } from "./dreaming-narrative.js"; import { formatErrorMessage, includesSystemEventToken } from "./dreaming-shared.js"; +import { resolveMemoryPromotionFileMaxChars } from "./memory-budget.js"; import type { PromotionRejectionCategory } from "./short-term-promotion-types.js"; const RUNTIME_CRON_RECONCILE_INTERVAL_MS = 60_000; @@ -638,6 +639,10 @@ async function runShortTermDreamingPromotionIfTriggered(params: { maxAgeDays: params.config.maxAgeDays, maxPromotedSnippetTokens: params.config.maxPromotedSnippetTokens, maxPriorEntryLossFraction: params.config.maxPriorEntryLossFraction, + memoryFileMaxChars: resolveMemoryPromotionFileMaxChars({ + cfg: params.cfg, + agentIds, + }), consolidation: { ...(params.subagent ? { subagent: params.subagent } : {}), ...(params.config.execution?.model ? { model: params.config.execution.model } : {}), @@ -675,7 +680,10 @@ async function runShortTermDreamingPromotionIfTriggered(params: { `memory-core: dreaming applied details [workspace=${workspaceDir}] ${appliedSummary}`, ); } - const deepHasContent = candidates.length > 0 || applied.applied > 0; + const hasReportableRejections = applied.rejectedCandidates.some( + ({ category }) => category !== "memory budget", + ); + const deepHasContent = repair.changed || applied.applied > 0 || hasReportableRejections; await writeDeepDreamingReport({ workspaceDir, bodyLines: reportLines, @@ -685,14 +693,12 @@ async function runShortTermDreamingPromotionIfTriggered(params: { storage: params.config.storage ?? { mode: "separate", separateReports: false }, }); // Generate dream diary narrative from promoted memories. - if (candidates.length > 0 || applied.applied > 0) { + if (applied.applied > 0) { const data: NarrativePhaseData = { phase: "deep", - snippets: candidates.map((c) => c.snippet).filter(Boolean), + snippets: applied.appliedCandidates.map((c) => c.snippet).filter(Boolean), promotions: applied.appliedCandidates.map((c) => c.snippet).filter(Boolean), - sourceEntryKeys: [ - ...new Set([...candidates, ...applied.appliedCandidates].map((c) => c.key)), - ], + sourceEntryKeys: [...new Set(applied.appliedCandidates.map((c) => c.key))], }; if (!params.subagent) { await appendFallbackNarrativeEntry({ diff --git a/extensions/memory-core/src/memory-budget-append.ts b/extensions/memory-core/src/memory-budget-append.ts new file mode 100644 index 000000000000..dab888edba5d --- /dev/null +++ b/extensions/memory-core/src/memory-budget-append.ts @@ -0,0 +1,18 @@ +import { compactMemoryForBudget, type CompactMemoryParams } from "./memory-budget.js"; + +/** Compose the complete post-promotion file from the bounded compaction result. */ +export function buildBudgetedMemoryAppend(params: CompactMemoryParams): { + content: string; + droppedDates: string[]; +} { + const compaction = compactMemoryForBudget(params); + const header = compaction.compacted.trim().length > 0 ? "" : "# Long-Term Memory\n\n"; + const base = + compaction.compacted.length === 0 || compaction.compacted.endsWith("\n") + ? compaction.compacted + : `${compaction.compacted}\n`; + return { + content: `${header}${base}${params.newSection}`, + droppedDates: compaction.droppedDates, + }; +} diff --git a/extensions/memory-core/src/memory-budget.test.ts b/extensions/memory-core/src/memory-budget.test.ts index bc6f7c5591f1..de6051defdcd 100644 --- a/extensions/memory-core/src/memory-budget.test.ts +++ b/extensions/memory-core/src/memory-budget.test.ts @@ -1,9 +1,40 @@ // Memory Core tests cover memory budget plugin behavior. +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { describe, expect, it } from "vitest"; -import { compactMemoryForBudget, DEFAULT_MEMORY_FILE_MAX_CHARS } from "./memory-budget.js"; +import { + compactMemoryForBudget, + DEFAULT_MEMORY_FILE_MAX_CHARS, + resolveMemoryPromotionFileMaxChars, +} from "./memory-budget.js"; const PROMOTION_MARKER_LINE = ""; +describe("promotion file budget resolution", () => { + const cfg = { + agents: { + defaults: { bootstrapMaxChars: 9_500 }, + list: [ + { id: "alpha", bootstrapMaxChars: 12_000 }, + { id: "beta", bootstrapMaxChars: 9_000 }, + ], + }, + } as OpenClawConfig; + + it("uses the smallest bootstrap cap among agents sharing the workspace", () => { + expect(resolveMemoryPromotionFileMaxChars({ cfg, agentIds: ["alpha", "beta"] })).toBe(9_000); + }); + + it("retains the promotion writer ceiling when the agent cap is larger", () => { + expect(resolveMemoryPromotionFileMaxChars({ cfg, agentIds: ["alpha"] })).toBe( + DEFAULT_MEMORY_FILE_MAX_CHARS, + ); + }); + + it("falls back to the configured default for an unlisted workspace owner", () => { + expect(resolveMemoryPromotionFileMaxChars({ cfg, agentIds: ["gamma"] })).toBe(9_500); + }); +}); + function promotionSection(date: string, sizeChars: number): string { const heading = `## Promoted From Short-Term Memory (${date})\n`; const marker = `${PROMOTION_MARKER_LINE}\n`; @@ -114,6 +145,24 @@ describe("compactMemoryForBudget — bounded MEMORY.md compaction (regression fo expect(result.compacted).not.toContain("Promoted From Short-Term Memory"); }); + it("stops before compaction would exceed the prior-entry loss limit", () => { + const existing = [ + promotionSection("2026-04-10", 500), + promotionSection("2026-04-15", 500), + promotionSection("2026-04-20", 500), + promotionSection("2026-04-25", 500), + ].join("\n"); + const result = compactMemoryForBudget({ + existingMemory: existing, + newSection: `\n${promotionSection("2026-04-29", 500)}`, + budgetChars: 1_400, + maxPriorEntryLossFraction: 0.25, + }); + + expect(result.droppedDates).toEqual(["2026-04-10"]); + expect(result.compacted).toContain("(2026-04-15)"); + }); + it("returns existing unchanged when the file has no promotion sections (cannot compact)", () => { const existing = "# Long-Term Memory\n\nLots of user content here.\n".repeat(50); const newSection = `\n${promotionSection("2026-04-29", 200)}`; diff --git a/extensions/memory-core/src/memory-budget.ts b/extensions/memory-core/src/memory-budget.ts index 5cb7d0682b20..9eba65f87d30 100644 --- a/extensions/memory-core/src/memory-budget.ts +++ b/extensions/memory-core/src/memory-budget.ts @@ -14,6 +14,9 @@ * is preserved unconditionally. */ +import { resolveAgentConfig } from "openclaw/plugin-sdk/agent-scope-runtime"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; + const PROMOTION_SECTION_HEADING_RE = /^## Promoted From Short-Term Memory \(([^)]+)\)\s*$/; const PROMOTION_SUBSECTION_HEADING_RE = /^### (?:Global|Project: .+?)\s*$/; @@ -32,6 +35,35 @@ const SETEXT_HEADING_UNDERLINE_RE = /^ {0,3}(?:=+|-+)[ \t]*$/; */ export const DEFAULT_MEMORY_FILE_MAX_CHARS = 10_000; +/** + * Keep promotion output within every consuming agent's per-file bootstrap + * budget. An unconfigured bootstrap limit stays above the promotion writer's + * own ceiling, so only explicit lower limits need to reduce the budget here. + */ +export function resolveMemoryPromotionFileMaxChars(params: { + cfg?: OpenClawConfig; + agentIds: readonly string[]; +}): number { + const defaultBootstrapLimit = params.cfg?.agents?.defaults?.bootstrapMaxChars; + const agentIds: Array = + params.agentIds.length > 0 ? [...new Set(params.agentIds)] : [undefined]; + let limit = DEFAULT_MEMORY_FILE_MAX_CHARS; + + for (const agentId of agentIds) { + const configuredLimit = agentId + ? (resolveAgentConfig(params.cfg ?? {}, agentId)?.bootstrapMaxChars ?? defaultBootstrapLimit) + : defaultBootstrapLimit; + if ( + typeof configuredLimit === "number" && + Number.isFinite(configuredLimit) && + configuredLimit > 0 + ) { + limit = Math.min(limit, Math.floor(configuredLimit)); + } + } + return limit; +} + /** * Reserve for writer-side overhead that the helper does not see directly: * the `# Long-Term Memory\n\n` header re-emitted when compaction empties @@ -44,7 +76,7 @@ const WRITE_OVERHEAD_RESERVE = 21; type MemoryBlock = | { kind: "preserved"; text: string } - | { kind: "promotion"; date: string; text: string }; + | { kind: "promotion"; date: string; text: string; entryCount: number }; function isGeneratedPromotionBlock(lines: string[]): boolean { let sawEntry = false; @@ -125,7 +157,12 @@ function parseMemoryBlocks(content: string): MemoryBlock[] { } const text = currentLines.join("\n"); if (currentKind === "promotion" && currentDate && isGeneratedPromotionBlock(currentLines)) { - blocks.push({ kind: "promotion", date: currentDate, text }); + blocks.push({ + kind: "promotion", + date: currentDate, + text, + entryCount: currentLines.filter((line) => PROMOTION_ENTRY_MARKER_RE.test(line)).length, + }); } else { blocks.push({ kind: "preserved", text }); } @@ -167,10 +204,12 @@ function joinBlocks(blocks: MemoryBlock[]): string { return blocks.map((block) => block.text).join("\n"); } -type CompactMemoryParams = { +export type CompactMemoryParams = { existingMemory: string; newSection: string; budgetChars: number; + /** Maximum fraction of existing generated entries that this write may remove. */ + maxPriorEntryLossFraction?: number; }; type CompactMemoryResult = { @@ -190,10 +229,9 @@ type CompactMemoryResult = { * - Promotion sections are dropped in ascending date order (oldest first). * - If `existingMemory + newSection` already fits the budget, the existing * memory is returned unchanged. - * - If the budget cannot be satisfied even by dropping every promotion - * section, the function drops them all and returns; the caller writes - * the new section anyway. This is the "log and continue" failure mode — - * refusing the new write would silently swallow the freshest material. + * - Compaction stops before exceeding `maxPriorEntryLossFraction` when set. + * - If the budget cannot be satisfied within that loss bound, the caller owns + * the final fit check and may defer the new section without rewriting memory. */ export function compactMemoryForBudget(params: CompactMemoryParams): CompactMemoryResult { const { existingMemory, newSection, budgetChars } = params; @@ -212,9 +250,19 @@ export function compactMemoryForBudget(params: CompactMemoryParams): CompactMemo const blocks = parseMemoryBlocks(existingMemory); const promotionEntries = blocks .map((block, index) => - block.kind === "promotion" ? { index, date: block.date, length: block.text.length } : null, + block.kind === "promotion" + ? { + index, + date: block.date, + length: block.text.length, + entryCount: block.entryCount, + } + : null, + ) + .filter( + (entry): entry is { index: number; date: string; length: number; entryCount: number } => + entry !== null, ) - .filter((entry): entry is { index: number; date: string; length: number } => entry !== null) .toSorted((a, b) => a.date.localeCompare(b.date)); if (promotionEntries.length === 0) { @@ -223,6 +271,9 @@ export function compactMemoryForBudget(params: CompactMemoryParams): CompactMemo const droppedIndices = new Set(); const droppedDates: string[] = []; + const totalEntryCount = promotionEntries.reduce((total, entry) => total + entry.entryCount, 0); + const maxLossFraction = Math.max(0, Math.min(1, params.maxPriorEntryLossFraction ?? 1)); + let droppedEntryCount = 0; let projectedExistingSize = existingMemory.length; // Block boundaries cost one newline each in joinBlocks; subtract a // newline along with the block text so the projection stays honest. @@ -232,8 +283,15 @@ export function compactMemoryForBudget(params: CompactMemoryParams): CompactMemo if (projectedExistingSize + newSection.length <= effectiveBudget) { break; } + if ( + totalEntryCount > 0 && + (droppedEntryCount + entry.entryCount) / totalEntryCount > maxLossFraction + ) { + break; + } droppedIndices.add(entry.index); droppedDates.push(entry.date); + droppedEntryCount += entry.entryCount; projectedExistingSize = Math.max(0, projectedExistingSize - entry.length - blockSeparatorCost); } diff --git a/extensions/memory-core/src/memory-forget-consolidation.test.ts b/extensions/memory-core/src/memory-forget-consolidation.test.ts index 658272fb01bd..1b3abb47db2d 100644 --- a/extensions/memory-core/src/memory-forget-consolidation.test.ts +++ b/extensions/memory-core/src/memory-forget-consolidation.test.ts @@ -375,6 +375,7 @@ describe("memory forget", () => { candidates, nowMs, memoryFileMaxChars: 450, + maxPriorEntryLossFraction: 1, ...thresholds, }); expect(promoted.appended).toBe(1); diff --git a/extensions/memory-core/src/short-term-promotion-apply.ts b/extensions/memory-core/src/short-term-promotion-apply.ts index 1b4209d030d5..57ff9b9a6f3f 100644 --- a/extensions/memory-core/src/short-term-promotion-apply.ts +++ b/extensions/memory-core/src/short-term-promotion-apply.ts @@ -21,7 +21,8 @@ import { isPromotionOriginBlocked, } from "./dreaming-consolidation-candidates.js"; import { applyMemoryConsolidationPlan, consolidateMemory } from "./dreaming-consolidation.js"; -import { compactMemoryForBudget, DEFAULT_MEMORY_FILE_MAX_CHARS } from "./memory-budget.js"; +import { buildBudgetedMemoryAppend } from "./memory-budget-append.js"; +import { DEFAULT_MEMORY_FILE_MAX_CHARS } from "./memory-budget.js"; import { pruneMemoryEntryOrigins, reserveMemoryEntryOrigins } from "./memory-entry-origins.js"; import { withMemoryWorkspaceLock } from "./memory-workspace-lock.js"; import { @@ -137,13 +138,6 @@ function formatPromotedSnippetForMemory(rawSnippet: string, maxTokens: number): return truncatePromotedSnippet(normalized || "(no snippet captured)", maxTokens); } -function withTrailingNewline(content: string): string { - if (!content) { - return ""; - } - return content.endsWith("\n") ? content : `${content}\n`; -} - function consolidationCandidateFingerprint(candidate: PromotionCandidate): string { return JSON.stringify({ key: candidate.key, @@ -416,6 +410,10 @@ export async function applyShortTermPromotions( typeof options.memoryFileMaxChars === "number" && Number.isFinite(options.memoryFileMaxChars) ? Math.max(0, Math.floor(options.memoryFileMaxChars)) : DEFAULT_MEMORY_FILE_MAX_CHARS; + const maxPriorEntryLossFraction = Math.max( + 0, + Math.min(1, options.maxPriorEntryLossFraction ?? 0.25), + ); const consolidationPlan = options.agentId && options.consolidation?.subagent && toAppend.length > 0 ? await consolidateMemory({ @@ -424,10 +422,7 @@ export async function applyShortTermPromotions( existingMemory, candidates: toAppend, ...(options.consolidation.model ? { model: options.consolidation.model } : {}), - maxPriorEntryLossFraction: Math.max( - 0, - Math.min(1, options.maxPriorEntryLossFraction ?? 0.25), - ), + maxPriorEntryLossFraction, memoryFileMaxChars: budgetChars, ...(typeof options.maxPromotedSnippetTokens === "number" ? { maxPromotedSnippetTokens: options.maxPromotedSnippetTokens } @@ -522,10 +517,7 @@ export async function applyShortTermPromotions( nowMs, ...(options.timezone ? { timezone: options.timezone } : {}), memoryFileMaxChars: budgetChars, - maxPriorEntryLossFraction: Math.max( - 0, - Math.min(1, options.maxPriorEntryLossFraction ?? 0.25), - ), + maxPriorEntryLossFraction, }); } } @@ -604,37 +596,44 @@ export async function applyShortTermPromotions( if (toAppend.length > 0) { // Model absence or rejected output preserves the shipped append-only // promotion contract, so a deep sweep never loses eligible memories. - const section = buildPromotionSection( - toAppend, - nowMs, - options.timezone, - options.maxPromotedSnippetTokens, - ); - const compaction = compactMemoryForBudget({ + const appendPlan = buildBudgetedMemoryAppend({ existingMemory, - newSection: section, + newSection: buildPromotionSection( + toAppend, + nowMs, + options.timezone, + options.maxPromotedSnippetTokens, + ), budgetChars, + maxPriorEntryLossFraction, }); - const droppedDates = compaction.droppedDates; - const baseMemory = compaction.compacted; - const header = baseMemory.trim().length > 0 ? "" : "# Long-Term Memory\n\n"; - const content = `${header}${withTrailingNewline(baseMemory)}${section}`; - // Append fallback keeps the historical read-modify-replace contract. Policy accepts - // its external-editor race because OpenClaw writers remain serialized by this sweep lock. - await commitMemoryContent({ - filePath: memoryWritePath, - tempPrefix: `${path.basename(memoryPath)}.promotion`, - expectedHash: hashMemoryContent(existingMemory), - expectedContent: existingMemory, - allowInPlaceFallback: true, - content, - }); - committedMemoryContent = content; - for (const candidate of toAppend) { - successfulCandidates.set(candidate.key, candidate); + const { content, droppedDates } = appendPlan; + if (budgetChars > 0 && content.length > budgetChars) { + const reason = `MEMORY.md budget exceeded (${content.length} > ${budgetChars} chars)`; + for (const candidate of toAppend) { + reject(candidate.key, "memory budget", reason); + } + options.consolidation?.logger.info( + `memory-core: deferred ${toAppend.length} promotion candidate(s) because ${reason}.`, + ); + } else { + // Append fallback keeps the historical read-modify-replace contract. Policy accepts + // its external-editor race because OpenClaw writers remain serialized by this sweep lock. + await commitMemoryContent({ + filePath: memoryWritePath, + tempPrefix: `${path.basename(memoryPath)}.promotion`, + expectedHash: hashMemoryContent(existingMemory), + expectedContent: existingMemory, + allowInPlaceFallback: true, + content, + }); + committedMemoryContent = content; + for (const candidate of toAppend) { + successfulCandidates.set(candidate.key, candidate); + } + compactedDates = droppedDates; + appendedCandidates = toAppend.length; } - compactedDates = droppedDates; - appendedCandidates = toAppend.length; } } if (rewriteSkippedReason) { diff --git a/extensions/memory-core/src/short-term-promotion-types.ts b/extensions/memory-core/src/short-term-promotion-types.ts index 56da31d59cb5..0a907ad36724 100644 --- a/extensions/memory-core/src/short-term-promotion-types.ts +++ b/extensions/memory-core/src/short-term-promotion-types.ts @@ -153,9 +153,9 @@ export type ApplyShortTermPromotionsOptions = { /** * Maximum size of MEMORY.md on disk after a promotion write, in * characters. When the post-write size would exceed this budget, the - * oldest auto-promotion sections are compacted out before write so the - * file stays bounded and bootstrap injection keeps reaching new - * sessions. Pass `0` to disable compaction. Defaults to + * oldest auto-promotion sections may be compacted out, within + * `maxPriorEntryLossFraction`, so the file stays bounded and bootstrap + * injection keeps reaching new sessions. Pass `0` to disable compaction. Defaults to * `DEFAULT_MEMORY_FILE_MAX_CHARS`. See #73691. */ memoryFileMaxChars?: number; @@ -166,6 +166,7 @@ export type ApplyShortTermPromotionsOptions = { * metadata. */ maxPromotedSnippetTokens?: number; + /** Maximum fraction of prior entries a promotion write may remove. */ maxPriorEntryLossFraction?: number; consolidation?: { subagent?: import("./dreaming-narrative.js").DreamingCompletion; @@ -190,6 +191,7 @@ export type PromotionRejectionCategory = | "selection limit" | "source rehydration" | "source changed" + | "memory budget" | "candidate changed"; export type ApplyShortTermPromotionsResult = { diff --git a/extensions/memory-core/src/short-term-promotion.test.ts b/extensions/memory-core/src/short-term-promotion.test.ts index 1c84bfe49f34..773c340d272d 100644 --- a/extensions/memory-core/src/short-term-promotion.test.ts +++ b/extensions/memory-core/src/short-term-promotion.test.ts @@ -1869,6 +1869,61 @@ describe("short-term promotion", () => { await expectEnoent(fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8")); }); + it("excludes a candidate downgraded during consolidation from the fallback write", async (workspaceDir) => { + const relativePath = "memory/2026-04-01.md"; + const firstSnippet = "Gateway binds loopback on the admin host."; + const secondSnippet = "Backups use the encrypted archive bucket."; + await writeDailyMemoryNote(workspaceDir, "2026-04-01", [firstSnippet, secondSnippet]); + await recordMemoryRecalls( + workspaceDir, + "operations policy", + [firstSnippet, secondSnippet].map((snippet, index) => ({ + path: relativePath, + startLine: index + 1, + endLine: index + 1, + score: 0.92, + snippet, + source: "memory" as const, + provenance: { + originClass: "agent" as const, + sessionKind: "interactive" as const, + observedAt: Date.parse("2026-04-01T12:00:00.000Z"), + }, + })), + ); + const ranked = await rankAllCandidates(workspaceDir); + expect(ranked).toHaveLength(2); + + const complete = vi.fn(async () => { + const store = await testing.readRecallStore(workspaceDir, new Date().toISOString()); + const downgraded = expectDefined( + Object.values(store.entries).find((entry) => entry.startLine === 1), + "downgraded recall entry", + ); + downgraded.provenance = { + originClass: "untrusted", + sessionKind: "interactive", + observedAt: Date.parse("2026-04-01T12:05:00.000Z"), + }; + await testing.writeRawRecallStore(workspaceDir, store); + return { text: "invalid consolidation response" }; + }); + const applied = await applyAllCandidates(workspaceDir, ranked, { + agentId: "main", + consolidation: { + subagent: { complete }, + logger: { info: vi.fn(), warn: vi.fn() }, + }, + }); + + expect(complete).toHaveBeenCalledOnce(); + expect(applied.appliedCandidates.map((candidate) => candidate.startLine)).toEqual([2]); + expect(applied.rejectedCandidates.map(({ candidate }) => candidate.startLine)).toEqual([1]); + const memory = await fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8"); + expect(memory).not.toContain(firstSnippet); + expect(memory).toContain(secondSnippet); + }); + it("does not double-prefix promoted snippets that are already markdown bullets", async (workspaceDir) => { await writeDailyMemoryNote(workspaceDir, "2026-04-01", [ "alpha", @@ -2926,6 +2981,7 @@ describe("short-term promotion", () => { return await applyAllCandidates(workspaceDir, await rankAllCandidates(workspaceDir), { nowMs, memoryFileMaxChars: 1_400, + maxPriorEntryLossFraction: 1, }); } @@ -3058,6 +3114,94 @@ describe("short-term promotion", () => { expect((await fs.stat(memoryPath)).mode & 0o7777).toBe(0o640); } }); + + it("defers promotion when preserved content leaves no room in the file budget", async (workspaceDir) => { + await writeDailyMemoryNote(workspaceDir, "2026-04-29", [ + "Keep the Tuesday deployment window in durable memory.", + ]); + await recordMemoryRecalls( + workspaceDir, + "deployment window", + [ + memoryRecallResult( + "memory/2026-04-29.md", + 1, + 1, + 0.95, + "Keep the Tuesday deployment window in durable memory.", + ), + ], + { nowMs: Date.parse("2026-04-29T10:00:00.000Z") }, + ); + const ranked = await rankAllCandidates(workspaceDir); + const memoryPath = path.join(workspaceDir, "MEMORY.md"); + const oversizedBase = `# Long-Term Memory\n\n${"u".repeat(8_800)}\n`; + await fs.writeFile(memoryPath, oversizedBase, "utf-8"); + const consolidate = vi.fn(async () => ({ text: "unused" })); + + const deferred = await applyAllCandidates(workspaceDir, ranked, { + agentId: "main", + memoryFileMaxChars: 9_000, + nowMs: Date.parse("2026-04-29T10:00:00.000Z"), + consolidation: { + subagent: { complete: consolidate }, + logger: { info: vi.fn(), warn: vi.fn() }, + }, + }); + + expect(deferred).toMatchObject({ applied: 0, appended: 0, compactedSections: 0 }); + expect(deferred.rejectedCandidates).toEqual([ + expect.objectContaining({ reason: expect.stringContaining("MEMORY.md budget exceeded") }), + ]); + expect(consolidate).toHaveBeenCalledOnce(); + await expect(fs.readFile(memoryPath, "utf-8")).resolves.toBe(oversizedBase); + + await fs.writeFile(memoryPath, "# Long-Term Memory\n\n", "utf-8"); + const retried = await applyAllCandidates(workspaceDir, ranked, { + memoryFileMaxChars: 9_000, + nowMs: Date.parse("2026-04-29T10:05:00.000Z"), + }); + expect(retried.applied).toBe(1); + expect((await fs.readFile(memoryPath, "utf-8")).length).toBeLessThanOrEqual(9_000); + }); + + it("leaves prior promotions intact when fitting would exceed the loss limit", async (workspaceDir) => { + await writeDailyMemoryNote(workspaceDir, "2026-04-29", [ + "Keep the Tuesday deployment window in durable memory.", + ]); + await recordMemoryRecalls( + workspaceDir, + "deployment window", + [ + memoryRecallResult( + "memory/2026-04-29.md", + 1, + 1, + 0.95, + "Keep the Tuesday deployment window in durable memory.", + ), + ], + { nowMs: Date.parse("2026-04-29T10:00:00.000Z") }, + ); + const memoryPath = path.join(workspaceDir, "MEMORY.md"); + const existing = ["2026-04-01", "2026-04-08", "2026-04-15", "2026-04-22"] + .map( + (date, index) => + `## Promoted From Short-Term Memory (${date})\n\n- ${"x".repeat(500)}`, + ) + .join("\n"); + await fs.writeFile(memoryPath, existing, "utf-8"); + + const result = await applyAllCandidates(workspaceDir, await rankAllCandidates(workspaceDir), { + memoryFileMaxChars: 1_600, + maxPriorEntryLossFraction: 0.25, + nowMs: Date.parse("2026-04-29T10:00:00.000Z"), + }); + + expect(result).toMatchObject({ applied: 0, compactedSections: 0 }); + expect(result.rejectedCandidates[0]?.reason).toContain("MEMORY.md budget exceeded"); + await expect(fs.readFile(memoryPath, "utf-8")).resolves.toBe(existing); + }); }); it("defers append-only promotion when recall state changes during rehydration", async (workspaceDir) => { @@ -3195,6 +3339,7 @@ describe("short-term promotion", () => { try { const applied = await applyAllCandidates(workspaceAlias, secondRanked, { memoryFileMaxChars: 400, + maxPriorEntryLossFraction: 1, }); expect(applied.applied).toBe(1); expect(await fs.readFile(targetPath, "utf-8")).toContain(secondSnippet); diff --git a/src/memory-host-sdk/dreaming.test.ts b/src/memory-host-sdk/dreaming.test.ts index 979f5780b934..7172144b57b8 100644 --- a/src/memory-host-sdk/dreaming.test.ts +++ b/src/memory-host-sdk/dreaming.test.ts @@ -10,6 +10,7 @@ import { resolveMemoryDreamingPluginConfig, resolveMemoryDreamingPluginId, resolveMemoryDreamingConfig, + resolveMemoryDreamingWorkspace, resolveMemoryDreamingWorkspaces, } from "./dreaming.js"; @@ -296,6 +297,10 @@ describe("memory dreaming host helpers", () => { expect(resolveMemoryDreamingWorkspaces(cfg)).toEqual([ { workspaceDir, agentIds: ["alpha", "beta"] }, ]); + expect(resolveMemoryDreamingWorkspace(cfg, workspaceAliasDir)).toEqual({ + workspaceDir, + agentIds: ["alpha", "beta"], + }); }); it("includes the runtime primary workspace alongside configured subagent workspaces", () => { diff --git a/src/memory-host-sdk/dreaming.ts b/src/memory-host-sdk/dreaming.ts index 0d88609d5ebb..62eda9e7a5c4 100644 --- a/src/memory-host-sdk/dreaming.ts +++ b/src/memory-host-sdk/dreaming.ts @@ -685,3 +685,14 @@ export function resolveMemoryDreamingWorkspaces( } return [...byWorkspace.values()]; } + +export function resolveMemoryDreamingWorkspace( + cfg: OpenClawConfig, + workspaceDir: string, + options: MemoryDreamingWorkspaceOptions = {}, +): MemoryDreamingWorkspace | undefined { + const workspacePath = resolveWorkspaceStateIdentity(workspaceDir).workspacePath; + return resolveMemoryDreamingWorkspaces(cfg, options).find( + (entry) => resolveWorkspaceStateIdentity(entry.workspaceDir).workspacePath === workspacePath, + ); +} diff --git a/src/plugin-sdk/memory-core-host-status.ts b/src/plugin-sdk/memory-core-host-status.ts index 7874b39647c6..39ae69700a98 100644 --- a/src/plugin-sdk/memory-core-host-status.ts +++ b/src/plugin-sdk/memory-core-host-status.ts @@ -12,6 +12,7 @@ export { isSameMemoryDreamingDay, resolveMemoryDeepDreamingConfig, resolveMemoryDreamingConfig, + resolveMemoryDreamingWorkspace, resolveMemoryDreamingWorkspaces, resolveMemoryLightDreamingConfig, resolveMemoryRemDreamingConfig,