mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 09:39:25 +00:00
fix(memory): honor bootstrap budget during promotion (#142545)
This commit is contained in:
Dallin Romney
GitHub
parent
1cc13c10a7
commit
fc0360dab2
16 changed files with 515 additions and 70 deletions
|
|
@ -92,7 +92,7 @@ source reference.
|
|||
|
||||
The model returns operation decisions, not replacement memory prose. The memory
|
||||
writer applies those decisions to the existing file using each candidate's
|
||||
bounded, sourced entry. An accepted rewrite must:
|
||||
bounded, sourced entry. An accepted rewrite or append compaction must:
|
||||
|
||||
- preserve prior entries within `phases.deep.maxPriorEntryLossFraction`
|
||||
- include every promoted candidate's `Source: path#Lx-Ly` reference
|
||||
|
|
@ -276,7 +276,7 @@ All settings live under `plugins.entries.memory-core.config.dreaming`.
|
|||
Enable or disable the dreaming sweep.
|
||||
</ParamField>
|
||||
<ParamField path="phases.deep.maxPriorEntryLossFraction" type="number" default="0.25">
|
||||
Reject a consolidation rewrite when it removes more than this fraction of prior entries.
|
||||
Reject a consolidation rewrite or append compaction when it removes more than this fraction of prior entries. Append compaction only removes whole machine-generated promotion sections.
|
||||
</ParamField>
|
||||
<ParamField path="frequency" type="string" default="0 3 * * *">
|
||||
Cron cadence for the full dreaming sweep.
|
||||
|
|
|
|||
|
|
@ -185,7 +185,12 @@ touching long-term memory.
|
|||
The consolidation output is accepted only if it passes structural
|
||||
validation, stays within the bootstrap file budget, and does not lose more
|
||||
than a bounded fraction of existing entries. A rejected rewrite falls back
|
||||
to the previous append-only behavior for that sweep.
|
||||
to append-only behavior for that sweep. Promotion uses the smallest configured
|
||||
per-file bootstrap limit among agents sharing the workspace, capped by the
|
||||
writer's own limit. If an append still cannot fit after older generated
|
||||
sections are removed, the writer preserves `MEMORY.md` unchanged and leaves
|
||||
the candidates eligible for a later sweep instead of committing an oversized
|
||||
file.
|
||||
|
||||
**Write safety.** Replacing `MEMORY.md` uses optimistic concurrency: the
|
||||
content hash captured when consolidation input was built is re-checked
|
||||
|
|
|
|||
|
|
@ -737,7 +737,7 @@ For conceptual behavior and slash commands, see [Dreaming](/concepts/dreaming).
|
|||
| `frequency` | `string` | `0 3 * * *` | Optional cron cadence for the full dreaming sweep |
|
||||
| `model` | `string` | default model | Optional Dream Diary subagent model override |
|
||||
| `phases.deep.maxPromotedSnippetTokens` | `number` | `160` | Maximum estimated tokens kept from each short-term recall snippet promoted into `MEMORY.md`; provenance metadata remains visible |
|
||||
| `phases.deep.maxPriorEntryLossFraction` | `number` | `0.25` | Reject a consolidation rewrite that removes more than this fraction of prior entries |
|
||||
| `phases.deep.maxPriorEntryLossFraction` | `number` | `0.25` | Reject consolidation or append compaction that removes more than this fraction of prior entries |
|
||||
|
||||
### Example
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import path from "node:path";
|
|||
import { resolveMemorySearchStaleness } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import {
|
||||
resolveMemoryDreamingConfig,
|
||||
resolveMemoryDreamingWorkspaces,
|
||||
resolveMemoryDreamingWorkspace,
|
||||
resolveMemoryDeepDreamingConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import {
|
||||
|
|
@ -32,6 +32,7 @@ import type {
|
|||
MemoryPromoteExplainOptions,
|
||||
MemorySearchCommandOptions,
|
||||
} from "./cli.types.js";
|
||||
import { resolveMemoryPromotionFileMaxChars } from "./memory-budget.js";
|
||||
import { forgetMemoryEntries } from "./memory-forget.js";
|
||||
import { captureMemoryRebuildNotice } from "./memory-rebuild-notice.js";
|
||||
import { formatMemoryVectorDegradedWriteReason } from "./memory/manager-vector-warning.js";
|
||||
|
|
@ -415,11 +416,12 @@ export async function runMemoryPromote(
|
|||
let applyResult: Awaited<ReturnType<typeof applyShortTermPromotions>> | undefined;
|
||||
if (opts.apply) {
|
||||
try {
|
||||
const workspaceAgentIds = resolveMemoryDreamingWorkspace(cfg, workspaceDir)?.agentIds ?? [
|
||||
agentId,
|
||||
];
|
||||
applyResult = await applyShortTermPromotions({
|
||||
agentId,
|
||||
workspaceAgentIds: resolveMemoryDreamingWorkspaces(cfg).find(
|
||||
(workspace) => path.resolve(workspace.workspaceDir) === path.resolve(workspaceDir),
|
||||
)?.agentIds,
|
||||
workspaceAgentIds,
|
||||
workspaceDir,
|
||||
candidates,
|
||||
limit: opts.limit,
|
||||
|
|
@ -428,6 +430,11 @@ export async function runMemoryPromote(
|
|||
minUniqueQueries: opts.minUniqueQueries ?? dreaming.minUniqueQueries,
|
||||
maxAgeDays: dreaming.maxAgeDays,
|
||||
maxPromotedSnippetTokens: dreaming.maxPromotedSnippetTokens,
|
||||
maxPriorEntryLossFraction: dreaming.maxPriorEntryLossFraction,
|
||||
memoryFileMaxChars: resolveMemoryPromotionFileMaxChars({
|
||||
cfg,
|
||||
agentIds: workspaceAgentIds,
|
||||
}),
|
||||
timezone: dreaming.timezone,
|
||||
});
|
||||
} catch (err) {
|
||||
|
|
|
|||
|
|
@ -3477,6 +3477,144 @@ describe("memory cli", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("honors the configured prior-entry loss limit during CLI promotion", async () => {
|
||||
await withTempWorkspace(async (workspaceDir) => {
|
||||
const promotionSection = (date: string, index: number) =>
|
||||
[
|
||||
`## Promoted From Short-Term Memory (${date})`,
|
||||
`<!-- openclaw-memory-promotion:legacy-${index} -->`,
|
||||
`- ${"x".repeat(350)}`,
|
||||
"",
|
||||
].join("\n");
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "MEMORY.md"),
|
||||
[0, 1, 2, 3]
|
||||
.map((index) => promotionSection(`2026-04-${String(index + 1).padStart(2, "0")}`, index))
|
||||
.join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
await writeDailyMemoryNote(workspaceDir, "2026-04-10", ["Retain the release checklist."]);
|
||||
await recordShortTermRecalls({
|
||||
workspaceDir,
|
||||
query: "release checklist",
|
||||
results: [
|
||||
{
|
||||
path: "memory/2026-04-10.md",
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
score: 0.91,
|
||||
snippet: "Retain the release checklist.",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
});
|
||||
getRuntimeConfig.mockReturnValue({
|
||||
agents: {
|
||||
list: [{ id: "main", default: true, workspace: workspaceDir, bootstrapMaxChars: 1_400 }],
|
||||
},
|
||||
plugins: {
|
||||
entries: {
|
||||
"memory-core": {
|
||||
config: { dreaming: { phases: { deep: { maxPriorEntryLossFraction: 1 } } } },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const close = vi.fn(async () => {});
|
||||
mockManager({ status: () => makeMemoryStatus({ workspaceDir }), close });
|
||||
|
||||
await runMemoryCli([
|
||||
"promote",
|
||||
"--apply",
|
||||
"--min-score",
|
||||
"0",
|
||||
"--min-recall-count",
|
||||
"0",
|
||||
"--min-unique-queries",
|
||||
"0",
|
||||
]);
|
||||
|
||||
const memory = await fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8");
|
||||
expect(memory).toContain("Retain the release checklist.");
|
||||
expect(memory.length).toBeLessThanOrEqual(1_400);
|
||||
expect(close).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"uses the smallest bootstrap cap across CLI workspace symlink aliases",
|
||||
async () => {
|
||||
await withTempWorkspace(async (workspaceDir) => {
|
||||
const workspaceAliasDir = `${workspaceDir}-alias`;
|
||||
await fs.symlink(workspaceDir, workspaceAliasDir, "dir");
|
||||
const existingMemory = `# Long-Term Memory\n\n${"x".repeat(9_100)}\n`;
|
||||
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), existingMemory, "utf-8");
|
||||
await writeDailyMemoryNote(workspaceDir, "2026-04-01", ["Shared workspace fact."]);
|
||||
await recordShortTermRecalls({
|
||||
workspaceDir: workspaceAliasDir,
|
||||
query: "shared workspace",
|
||||
results: [
|
||||
{
|
||||
path: "memory/2026-04-01.md",
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
score: 0.91,
|
||||
snippet: "Shared workspace fact.",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
});
|
||||
getRuntimeConfig.mockReturnValue({
|
||||
agents: {
|
||||
list: [
|
||||
{
|
||||
id: "alpha",
|
||||
default: true,
|
||||
workspace: workspaceDir,
|
||||
bootstrapMaxChars: 9_000,
|
||||
},
|
||||
{ id: "beta", workspace: workspaceAliasDir, bootstrapMaxChars: 12_000 },
|
||||
],
|
||||
},
|
||||
});
|
||||
const close = vi.fn(async () => {});
|
||||
mockManager({
|
||||
status: () => makeMemoryStatus({ workspaceDir: workspaceAliasDir }),
|
||||
close,
|
||||
});
|
||||
|
||||
const writeJson = spyRuntimeJson(defaultRuntime);
|
||||
await runMemoryCli([
|
||||
"promote",
|
||||
"--agent",
|
||||
"beta",
|
||||
"--apply",
|
||||
"--json",
|
||||
"--min-score",
|
||||
"0",
|
||||
"--min-recall-count",
|
||||
"0",
|
||||
"--min-unique-queries",
|
||||
"0",
|
||||
]);
|
||||
|
||||
const payload = firstWrittenJsonArg<{
|
||||
candidates: unknown[];
|
||||
apply: { appliedCandidates: unknown[]; rejectedCandidates: Array<{ reason: string }> };
|
||||
}>(writeJson);
|
||||
expect(payload?.candidates).toHaveLength(1);
|
||||
expect(payload?.apply.appliedCandidates).toEqual([]);
|
||||
expect(payload?.apply.rejectedCandidates).toEqual([
|
||||
expect.objectContaining({ reason: expect.stringContaining("budget") }),
|
||||
]);
|
||||
expect(await fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8")).toBe(
|
||||
existingMemory,
|
||||
);
|
||||
expect(close).toHaveBeenCalled();
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("names apply-time rejections without ranking blocked origins", async () => {
|
||||
await withTempWorkspace(async (workspaceDir) => {
|
||||
const relativePath = "memory/2026-04-02.md";
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import { peekSystemEventEntries } from "openclaw/plugin-sdk/system-event-runtime
|
|||
import { appendFailedDreamingEvent } from "./dreaming-events.js";
|
||||
import type { NarrativePhaseData } from "./dreaming-narrative.js";
|
||||
import { formatErrorMessage, includesSystemEventToken } from "./dreaming-shared.js";
|
||||
import { resolveMemoryPromotionFileMaxChars } from "./memory-budget.js";
|
||||
import type { PromotionRejectionCategory } from "./short-term-promotion-types.js";
|
||||
|
||||
const RUNTIME_CRON_RECONCILE_INTERVAL_MS = 60_000;
|
||||
|
|
@ -638,6 +639,10 @@ async function runShortTermDreamingPromotionIfTriggered(params: {
|
|||
maxAgeDays: params.config.maxAgeDays,
|
||||
maxPromotedSnippetTokens: params.config.maxPromotedSnippetTokens,
|
||||
maxPriorEntryLossFraction: params.config.maxPriorEntryLossFraction,
|
||||
memoryFileMaxChars: resolveMemoryPromotionFileMaxChars({
|
||||
cfg: params.cfg,
|
||||
agentIds,
|
||||
}),
|
||||
consolidation: {
|
||||
...(params.subagent ? { subagent: params.subagent } : {}),
|
||||
...(params.config.execution?.model ? { model: params.config.execution.model } : {}),
|
||||
|
|
@ -675,7 +680,10 @@ async function runShortTermDreamingPromotionIfTriggered(params: {
|
|||
`memory-core: dreaming applied details [workspace=${workspaceDir}] ${appliedSummary}`,
|
||||
);
|
||||
}
|
||||
const deepHasContent = candidates.length > 0 || applied.applied > 0;
|
||||
const hasReportableRejections = applied.rejectedCandidates.some(
|
||||
({ category }) => category !== "memory budget",
|
||||
);
|
||||
const deepHasContent = repair.changed || applied.applied > 0 || hasReportableRejections;
|
||||
await writeDeepDreamingReport({
|
||||
workspaceDir,
|
||||
bodyLines: reportLines,
|
||||
|
|
@ -685,14 +693,12 @@ async function runShortTermDreamingPromotionIfTriggered(params: {
|
|||
storage: params.config.storage ?? { mode: "separate", separateReports: false },
|
||||
});
|
||||
// Generate dream diary narrative from promoted memories.
|
||||
if (candidates.length > 0 || applied.applied > 0) {
|
||||
if (applied.applied > 0) {
|
||||
const data: NarrativePhaseData = {
|
||||
phase: "deep",
|
||||
snippets: candidates.map((c) => c.snippet).filter(Boolean),
|
||||
snippets: applied.appliedCandidates.map((c) => c.snippet).filter(Boolean),
|
||||
promotions: applied.appliedCandidates.map((c) => c.snippet).filter(Boolean),
|
||||
sourceEntryKeys: [
|
||||
...new Set([...candidates, ...applied.appliedCandidates].map((c) => c.key)),
|
||||
],
|
||||
sourceEntryKeys: [...new Set(applied.appliedCandidates.map((c) => c.key))],
|
||||
};
|
||||
if (!params.subagent) {
|
||||
await appendFallbackNarrativeEntry({
|
||||
|
|
|
|||
18
extensions/memory-core/src/memory-budget-append.ts
Normal file
18
extensions/memory-core/src/memory-budget-append.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import { compactMemoryForBudget, type CompactMemoryParams } from "./memory-budget.js";
|
||||
|
||||
/** Compose the complete post-promotion file from the bounded compaction result. */
|
||||
export function buildBudgetedMemoryAppend(params: CompactMemoryParams): {
|
||||
content: string;
|
||||
droppedDates: string[];
|
||||
} {
|
||||
const compaction = compactMemoryForBudget(params);
|
||||
const header = compaction.compacted.trim().length > 0 ? "" : "# Long-Term Memory\n\n";
|
||||
const base =
|
||||
compaction.compacted.length === 0 || compaction.compacted.endsWith("\n")
|
||||
? compaction.compacted
|
||||
: `${compaction.compacted}\n`;
|
||||
return {
|
||||
content: `${header}${base}${params.newSection}`,
|
||||
droppedDates: compaction.droppedDates,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,9 +1,40 @@
|
|||
// Memory Core tests cover memory budget plugin behavior.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { compactMemoryForBudget, DEFAULT_MEMORY_FILE_MAX_CHARS } from "./memory-budget.js";
|
||||
import {
|
||||
compactMemoryForBudget,
|
||||
DEFAULT_MEMORY_FILE_MAX_CHARS,
|
||||
resolveMemoryPromotionFileMaxChars,
|
||||
} from "./memory-budget.js";
|
||||
|
||||
const PROMOTION_MARKER_LINE = "<!-- openclaw-memory-promotion:memory/short-term.md#entry -->";
|
||||
|
||||
describe("promotion file budget resolution", () => {
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: { bootstrapMaxChars: 9_500 },
|
||||
list: [
|
||||
{ id: "alpha", bootstrapMaxChars: 12_000 },
|
||||
{ id: "beta", bootstrapMaxChars: 9_000 },
|
||||
],
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
it("uses the smallest bootstrap cap among agents sharing the workspace", () => {
|
||||
expect(resolveMemoryPromotionFileMaxChars({ cfg, agentIds: ["alpha", "beta"] })).toBe(9_000);
|
||||
});
|
||||
|
||||
it("retains the promotion writer ceiling when the agent cap is larger", () => {
|
||||
expect(resolveMemoryPromotionFileMaxChars({ cfg, agentIds: ["alpha"] })).toBe(
|
||||
DEFAULT_MEMORY_FILE_MAX_CHARS,
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to the configured default for an unlisted workspace owner", () => {
|
||||
expect(resolveMemoryPromotionFileMaxChars({ cfg, agentIds: ["gamma"] })).toBe(9_500);
|
||||
});
|
||||
});
|
||||
|
||||
function promotionSection(date: string, sizeChars: number): string {
|
||||
const heading = `## Promoted From Short-Term Memory (${date})\n`;
|
||||
const marker = `${PROMOTION_MARKER_LINE}\n`;
|
||||
|
|
@ -114,6 +145,24 @@ describe("compactMemoryForBudget — bounded MEMORY.md compaction (regression fo
|
|||
expect(result.compacted).not.toContain("Promoted From Short-Term Memory");
|
||||
});
|
||||
|
||||
it("stops before compaction would exceed the prior-entry loss limit", () => {
|
||||
const existing = [
|
||||
promotionSection("2026-04-10", 500),
|
||||
promotionSection("2026-04-15", 500),
|
||||
promotionSection("2026-04-20", 500),
|
||||
promotionSection("2026-04-25", 500),
|
||||
].join("\n");
|
||||
const result = compactMemoryForBudget({
|
||||
existingMemory: existing,
|
||||
newSection: `\n${promotionSection("2026-04-29", 500)}`,
|
||||
budgetChars: 1_400,
|
||||
maxPriorEntryLossFraction: 0.25,
|
||||
});
|
||||
|
||||
expect(result.droppedDates).toEqual(["2026-04-10"]);
|
||||
expect(result.compacted).toContain("(2026-04-15)");
|
||||
});
|
||||
|
||||
it("returns existing unchanged when the file has no promotion sections (cannot compact)", () => {
|
||||
const existing = "# Long-Term Memory\n\nLots of user content here.\n".repeat(50);
|
||||
const newSection = `\n${promotionSection("2026-04-29", 200)}`;
|
||||
|
|
|
|||
|
|
@ -14,6 +14,9 @@
|
|||
* is preserved unconditionally.
|
||||
*/
|
||||
|
||||
import { resolveAgentConfig } from "openclaw/plugin-sdk/agent-scope-runtime";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
|
||||
const PROMOTION_SECTION_HEADING_RE = /^## Promoted From Short-Term Memory \(([^)]+)\)\s*$/;
|
||||
|
||||
const PROMOTION_SUBSECTION_HEADING_RE = /^### (?:Global|Project: .+?)\s*$/;
|
||||
|
|
@ -32,6 +35,35 @@ const SETEXT_HEADING_UNDERLINE_RE = /^ {0,3}(?:=+|-+)[ \t]*$/;
|
|||
*/
|
||||
export const DEFAULT_MEMORY_FILE_MAX_CHARS = 10_000;
|
||||
|
||||
/**
|
||||
* Keep promotion output within every consuming agent's per-file bootstrap
|
||||
* budget. An unconfigured bootstrap limit stays above the promotion writer's
|
||||
* own ceiling, so only explicit lower limits need to reduce the budget here.
|
||||
*/
|
||||
export function resolveMemoryPromotionFileMaxChars(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
agentIds: readonly string[];
|
||||
}): number {
|
||||
const defaultBootstrapLimit = params.cfg?.agents?.defaults?.bootstrapMaxChars;
|
||||
const agentIds: Array<string | undefined> =
|
||||
params.agentIds.length > 0 ? [...new Set(params.agentIds)] : [undefined];
|
||||
let limit = DEFAULT_MEMORY_FILE_MAX_CHARS;
|
||||
|
||||
for (const agentId of agentIds) {
|
||||
const configuredLimit = agentId
|
||||
? (resolveAgentConfig(params.cfg ?? {}, agentId)?.bootstrapMaxChars ?? defaultBootstrapLimit)
|
||||
: defaultBootstrapLimit;
|
||||
if (
|
||||
typeof configuredLimit === "number" &&
|
||||
Number.isFinite(configuredLimit) &&
|
||||
configuredLimit > 0
|
||||
) {
|
||||
limit = Math.min(limit, Math.floor(configuredLimit));
|
||||
}
|
||||
}
|
||||
return limit;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reserve for writer-side overhead that the helper does not see directly:
|
||||
* the `# Long-Term Memory\n\n` header re-emitted when compaction empties
|
||||
|
|
@ -44,7 +76,7 @@ const WRITE_OVERHEAD_RESERVE = 21;
|
|||
|
||||
type MemoryBlock =
|
||||
| { kind: "preserved"; text: string }
|
||||
| { kind: "promotion"; date: string; text: string };
|
||||
| { kind: "promotion"; date: string; text: string; entryCount: number };
|
||||
|
||||
function isGeneratedPromotionBlock(lines: string[]): boolean {
|
||||
let sawEntry = false;
|
||||
|
|
@ -125,7 +157,12 @@ function parseMemoryBlocks(content: string): MemoryBlock[] {
|
|||
}
|
||||
const text = currentLines.join("\n");
|
||||
if (currentKind === "promotion" && currentDate && isGeneratedPromotionBlock(currentLines)) {
|
||||
blocks.push({ kind: "promotion", date: currentDate, text });
|
||||
blocks.push({
|
||||
kind: "promotion",
|
||||
date: currentDate,
|
||||
text,
|
||||
entryCount: currentLines.filter((line) => PROMOTION_ENTRY_MARKER_RE.test(line)).length,
|
||||
});
|
||||
} else {
|
||||
blocks.push({ kind: "preserved", text });
|
||||
}
|
||||
|
|
@ -167,10 +204,12 @@ function joinBlocks(blocks: MemoryBlock[]): string {
|
|||
return blocks.map((block) => block.text).join("\n");
|
||||
}
|
||||
|
||||
type CompactMemoryParams = {
|
||||
export type CompactMemoryParams = {
|
||||
existingMemory: string;
|
||||
newSection: string;
|
||||
budgetChars: number;
|
||||
/** Maximum fraction of existing generated entries that this write may remove. */
|
||||
maxPriorEntryLossFraction?: number;
|
||||
};
|
||||
|
||||
type CompactMemoryResult = {
|
||||
|
|
@ -190,10 +229,9 @@ type CompactMemoryResult = {
|
|||
* - Promotion sections are dropped in ascending date order (oldest first).
|
||||
* - If `existingMemory + newSection` already fits the budget, the existing
|
||||
* memory is returned unchanged.
|
||||
* - If the budget cannot be satisfied even by dropping every promotion
|
||||
* section, the function drops them all and returns; the caller writes
|
||||
* the new section anyway. This is the "log and continue" failure mode —
|
||||
* refusing the new write would silently swallow the freshest material.
|
||||
* - Compaction stops before exceeding `maxPriorEntryLossFraction` when set.
|
||||
* - If the budget cannot be satisfied within that loss bound, the caller owns
|
||||
* the final fit check and may defer the new section without rewriting memory.
|
||||
*/
|
||||
export function compactMemoryForBudget(params: CompactMemoryParams): CompactMemoryResult {
|
||||
const { existingMemory, newSection, budgetChars } = params;
|
||||
|
|
@ -212,9 +250,19 @@ export function compactMemoryForBudget(params: CompactMemoryParams): CompactMemo
|
|||
const blocks = parseMemoryBlocks(existingMemory);
|
||||
const promotionEntries = blocks
|
||||
.map((block, index) =>
|
||||
block.kind === "promotion" ? { index, date: block.date, length: block.text.length } : null,
|
||||
block.kind === "promotion"
|
||||
? {
|
||||
index,
|
||||
date: block.date,
|
||||
length: block.text.length,
|
||||
entryCount: block.entryCount,
|
||||
}
|
||||
: null,
|
||||
)
|
||||
.filter(
|
||||
(entry): entry is { index: number; date: string; length: number; entryCount: number } =>
|
||||
entry !== null,
|
||||
)
|
||||
.filter((entry): entry is { index: number; date: string; length: number } => entry !== null)
|
||||
.toSorted((a, b) => a.date.localeCompare(b.date));
|
||||
|
||||
if (promotionEntries.length === 0) {
|
||||
|
|
@ -223,6 +271,9 @@ export function compactMemoryForBudget(params: CompactMemoryParams): CompactMemo
|
|||
|
||||
const droppedIndices = new Set<number>();
|
||||
const droppedDates: string[] = [];
|
||||
const totalEntryCount = promotionEntries.reduce((total, entry) => total + entry.entryCount, 0);
|
||||
const maxLossFraction = Math.max(0, Math.min(1, params.maxPriorEntryLossFraction ?? 1));
|
||||
let droppedEntryCount = 0;
|
||||
let projectedExistingSize = existingMemory.length;
|
||||
// Block boundaries cost one newline each in joinBlocks; subtract a
|
||||
// newline along with the block text so the projection stays honest.
|
||||
|
|
@ -232,8 +283,15 @@ export function compactMemoryForBudget(params: CompactMemoryParams): CompactMemo
|
|||
if (projectedExistingSize + newSection.length <= effectiveBudget) {
|
||||
break;
|
||||
}
|
||||
if (
|
||||
totalEntryCount > 0 &&
|
||||
(droppedEntryCount + entry.entryCount) / totalEntryCount > maxLossFraction
|
||||
) {
|
||||
break;
|
||||
}
|
||||
droppedIndices.add(entry.index);
|
||||
droppedDates.push(entry.date);
|
||||
droppedEntryCount += entry.entryCount;
|
||||
projectedExistingSize = Math.max(0, projectedExistingSize - entry.length - blockSeparatorCost);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -375,6 +375,7 @@ describe("memory forget", () => {
|
|||
candidates,
|
||||
nowMs,
|
||||
memoryFileMaxChars: 450,
|
||||
maxPriorEntryLossFraction: 1,
|
||||
...thresholds,
|
||||
});
|
||||
expect(promoted.appended).toBe(1);
|
||||
|
|
|
|||
|
|
@ -21,7 +21,8 @@ import {
|
|||
isPromotionOriginBlocked,
|
||||
} from "./dreaming-consolidation-candidates.js";
|
||||
import { applyMemoryConsolidationPlan, consolidateMemory } from "./dreaming-consolidation.js";
|
||||
import { compactMemoryForBudget, DEFAULT_MEMORY_FILE_MAX_CHARS } from "./memory-budget.js";
|
||||
import { buildBudgetedMemoryAppend } from "./memory-budget-append.js";
|
||||
import { DEFAULT_MEMORY_FILE_MAX_CHARS } from "./memory-budget.js";
|
||||
import { pruneMemoryEntryOrigins, reserveMemoryEntryOrigins } from "./memory-entry-origins.js";
|
||||
import { withMemoryWorkspaceLock } from "./memory-workspace-lock.js";
|
||||
import {
|
||||
|
|
@ -137,13 +138,6 @@ function formatPromotedSnippetForMemory(rawSnippet: string, maxTokens: number):
|
|||
return truncatePromotedSnippet(normalized || "(no snippet captured)", maxTokens);
|
||||
}
|
||||
|
||||
function withTrailingNewline(content: string): string {
|
||||
if (!content) {
|
||||
return "";
|
||||
}
|
||||
return content.endsWith("\n") ? content : `${content}\n`;
|
||||
}
|
||||
|
||||
function consolidationCandidateFingerprint(candidate: PromotionCandidate): string {
|
||||
return JSON.stringify({
|
||||
key: candidate.key,
|
||||
|
|
@ -416,6 +410,10 @@ export async function applyShortTermPromotions(
|
|||
typeof options.memoryFileMaxChars === "number" && Number.isFinite(options.memoryFileMaxChars)
|
||||
? Math.max(0, Math.floor(options.memoryFileMaxChars))
|
||||
: DEFAULT_MEMORY_FILE_MAX_CHARS;
|
||||
const maxPriorEntryLossFraction = Math.max(
|
||||
0,
|
||||
Math.min(1, options.maxPriorEntryLossFraction ?? 0.25),
|
||||
);
|
||||
const consolidationPlan =
|
||||
options.agentId && options.consolidation?.subagent && toAppend.length > 0
|
||||
? await consolidateMemory({
|
||||
|
|
@ -424,10 +422,7 @@ export async function applyShortTermPromotions(
|
|||
existingMemory,
|
||||
candidates: toAppend,
|
||||
...(options.consolidation.model ? { model: options.consolidation.model } : {}),
|
||||
maxPriorEntryLossFraction: Math.max(
|
||||
0,
|
||||
Math.min(1, options.maxPriorEntryLossFraction ?? 0.25),
|
||||
),
|
||||
maxPriorEntryLossFraction,
|
||||
memoryFileMaxChars: budgetChars,
|
||||
...(typeof options.maxPromotedSnippetTokens === "number"
|
||||
? { maxPromotedSnippetTokens: options.maxPromotedSnippetTokens }
|
||||
|
|
@ -522,10 +517,7 @@ export async function applyShortTermPromotions(
|
|||
nowMs,
|
||||
...(options.timezone ? { timezone: options.timezone } : {}),
|
||||
memoryFileMaxChars: budgetChars,
|
||||
maxPriorEntryLossFraction: Math.max(
|
||||
0,
|
||||
Math.min(1, options.maxPriorEntryLossFraction ?? 0.25),
|
||||
),
|
||||
maxPriorEntryLossFraction,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -604,37 +596,44 @@ export async function applyShortTermPromotions(
|
|||
if (toAppend.length > 0) {
|
||||
// Model absence or rejected output preserves the shipped append-only
|
||||
// promotion contract, so a deep sweep never loses eligible memories.
|
||||
const section = buildPromotionSection(
|
||||
toAppend,
|
||||
nowMs,
|
||||
options.timezone,
|
||||
options.maxPromotedSnippetTokens,
|
||||
);
|
||||
const compaction = compactMemoryForBudget({
|
||||
const appendPlan = buildBudgetedMemoryAppend({
|
||||
existingMemory,
|
||||
newSection: section,
|
||||
newSection: buildPromotionSection(
|
||||
toAppend,
|
||||
nowMs,
|
||||
options.timezone,
|
||||
options.maxPromotedSnippetTokens,
|
||||
),
|
||||
budgetChars,
|
||||
maxPriorEntryLossFraction,
|
||||
});
|
||||
const droppedDates = compaction.droppedDates;
|
||||
const baseMemory = compaction.compacted;
|
||||
const header = baseMemory.trim().length > 0 ? "" : "# Long-Term Memory\n\n";
|
||||
const content = `${header}${withTrailingNewline(baseMemory)}${section}`;
|
||||
// Append fallback keeps the historical read-modify-replace contract. Policy accepts
|
||||
// its external-editor race because OpenClaw writers remain serialized by this sweep lock.
|
||||
await commitMemoryContent({
|
||||
filePath: memoryWritePath,
|
||||
tempPrefix: `${path.basename(memoryPath)}.promotion`,
|
||||
expectedHash: hashMemoryContent(existingMemory),
|
||||
expectedContent: existingMemory,
|
||||
allowInPlaceFallback: true,
|
||||
content,
|
||||
});
|
||||
committedMemoryContent = content;
|
||||
for (const candidate of toAppend) {
|
||||
successfulCandidates.set(candidate.key, candidate);
|
||||
const { content, droppedDates } = appendPlan;
|
||||
if (budgetChars > 0 && content.length > budgetChars) {
|
||||
const reason = `MEMORY.md budget exceeded (${content.length} > ${budgetChars} chars)`;
|
||||
for (const candidate of toAppend) {
|
||||
reject(candidate.key, "memory budget", reason);
|
||||
}
|
||||
options.consolidation?.logger.info(
|
||||
`memory-core: deferred ${toAppend.length} promotion candidate(s) because ${reason}.`,
|
||||
);
|
||||
} else {
|
||||
// Append fallback keeps the historical read-modify-replace contract. Policy accepts
|
||||
// its external-editor race because OpenClaw writers remain serialized by this sweep lock.
|
||||
await commitMemoryContent({
|
||||
filePath: memoryWritePath,
|
||||
tempPrefix: `${path.basename(memoryPath)}.promotion`,
|
||||
expectedHash: hashMemoryContent(existingMemory),
|
||||
expectedContent: existingMemory,
|
||||
allowInPlaceFallback: true,
|
||||
content,
|
||||
});
|
||||
committedMemoryContent = content;
|
||||
for (const candidate of toAppend) {
|
||||
successfulCandidates.set(candidate.key, candidate);
|
||||
}
|
||||
compactedDates = droppedDates;
|
||||
appendedCandidates = toAppend.length;
|
||||
}
|
||||
compactedDates = droppedDates;
|
||||
appendedCandidates = toAppend.length;
|
||||
}
|
||||
}
|
||||
if (rewriteSkippedReason) {
|
||||
|
|
|
|||
|
|
@ -153,9 +153,9 @@ export type ApplyShortTermPromotionsOptions = {
|
|||
/**
|
||||
* Maximum size of MEMORY.md on disk after a promotion write, in
|
||||
* characters. When the post-write size would exceed this budget, the
|
||||
* oldest auto-promotion sections are compacted out before write so the
|
||||
* file stays bounded and bootstrap injection keeps reaching new
|
||||
* sessions. Pass `0` to disable compaction. Defaults to
|
||||
* oldest auto-promotion sections may be compacted out, within
|
||||
* `maxPriorEntryLossFraction`, so the file stays bounded and bootstrap
|
||||
* injection keeps reaching new sessions. Pass `0` to disable compaction. Defaults to
|
||||
* `DEFAULT_MEMORY_FILE_MAX_CHARS`. See #73691.
|
||||
*/
|
||||
memoryFileMaxChars?: number;
|
||||
|
|
@ -166,6 +166,7 @@ export type ApplyShortTermPromotionsOptions = {
|
|||
* metadata.
|
||||
*/
|
||||
maxPromotedSnippetTokens?: number;
|
||||
/** Maximum fraction of prior entries a promotion write may remove. */
|
||||
maxPriorEntryLossFraction?: number;
|
||||
consolidation?: {
|
||||
subagent?: import("./dreaming-narrative.js").DreamingCompletion;
|
||||
|
|
@ -190,6 +191,7 @@ export type PromotionRejectionCategory =
|
|||
| "selection limit"
|
||||
| "source rehydration"
|
||||
| "source changed"
|
||||
| "memory budget"
|
||||
| "candidate changed";
|
||||
|
||||
export type ApplyShortTermPromotionsResult = {
|
||||
|
|
|
|||
|
|
@ -1869,6 +1869,61 @@ describe("short-term promotion", () => {
|
|||
await expectEnoent(fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8"));
|
||||
});
|
||||
|
||||
it("excludes a candidate downgraded during consolidation from the fallback write", async (workspaceDir) => {
|
||||
const relativePath = "memory/2026-04-01.md";
|
||||
const firstSnippet = "Gateway binds loopback on the admin host.";
|
||||
const secondSnippet = "Backups use the encrypted archive bucket.";
|
||||
await writeDailyMemoryNote(workspaceDir, "2026-04-01", [firstSnippet, secondSnippet]);
|
||||
await recordMemoryRecalls(
|
||||
workspaceDir,
|
||||
"operations policy",
|
||||
[firstSnippet, secondSnippet].map((snippet, index) => ({
|
||||
path: relativePath,
|
||||
startLine: index + 1,
|
||||
endLine: index + 1,
|
||||
score: 0.92,
|
||||
snippet,
|
||||
source: "memory" as const,
|
||||
provenance: {
|
||||
originClass: "agent" as const,
|
||||
sessionKind: "interactive" as const,
|
||||
observedAt: Date.parse("2026-04-01T12:00:00.000Z"),
|
||||
},
|
||||
})),
|
||||
);
|
||||
const ranked = await rankAllCandidates(workspaceDir);
|
||||
expect(ranked).toHaveLength(2);
|
||||
|
||||
const complete = vi.fn(async () => {
|
||||
const store = await testing.readRecallStore(workspaceDir, new Date().toISOString());
|
||||
const downgraded = expectDefined(
|
||||
Object.values(store.entries).find((entry) => entry.startLine === 1),
|
||||
"downgraded recall entry",
|
||||
);
|
||||
downgraded.provenance = {
|
||||
originClass: "untrusted",
|
||||
sessionKind: "interactive",
|
||||
observedAt: Date.parse("2026-04-01T12:05:00.000Z"),
|
||||
};
|
||||
await testing.writeRawRecallStore(workspaceDir, store);
|
||||
return { text: "invalid consolidation response" };
|
||||
});
|
||||
const applied = await applyAllCandidates(workspaceDir, ranked, {
|
||||
agentId: "main",
|
||||
consolidation: {
|
||||
subagent: { complete },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
},
|
||||
});
|
||||
|
||||
expect(complete).toHaveBeenCalledOnce();
|
||||
expect(applied.appliedCandidates.map((candidate) => candidate.startLine)).toEqual([2]);
|
||||
expect(applied.rejectedCandidates.map(({ candidate }) => candidate.startLine)).toEqual([1]);
|
||||
const memory = await fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8");
|
||||
expect(memory).not.toContain(firstSnippet);
|
||||
expect(memory).toContain(secondSnippet);
|
||||
});
|
||||
|
||||
it("does not double-prefix promoted snippets that are already markdown bullets", async (workspaceDir) => {
|
||||
await writeDailyMemoryNote(workspaceDir, "2026-04-01", [
|
||||
"alpha",
|
||||
|
|
@ -2926,6 +2981,7 @@ describe("short-term promotion", () => {
|
|||
return await applyAllCandidates(workspaceDir, await rankAllCandidates(workspaceDir), {
|
||||
nowMs,
|
||||
memoryFileMaxChars: 1_400,
|
||||
maxPriorEntryLossFraction: 1,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -3058,6 +3114,94 @@ describe("short-term promotion", () => {
|
|||
expect((await fs.stat(memoryPath)).mode & 0o7777).toBe(0o640);
|
||||
}
|
||||
});
|
||||
|
||||
it("defers promotion when preserved content leaves no room in the file budget", async (workspaceDir) => {
|
||||
await writeDailyMemoryNote(workspaceDir, "2026-04-29", [
|
||||
"Keep the Tuesday deployment window in durable memory.",
|
||||
]);
|
||||
await recordMemoryRecalls(
|
||||
workspaceDir,
|
||||
"deployment window",
|
||||
[
|
||||
memoryRecallResult(
|
||||
"memory/2026-04-29.md",
|
||||
1,
|
||||
1,
|
||||
0.95,
|
||||
"Keep the Tuesday deployment window in durable memory.",
|
||||
),
|
||||
],
|
||||
{ nowMs: Date.parse("2026-04-29T10:00:00.000Z") },
|
||||
);
|
||||
const ranked = await rankAllCandidates(workspaceDir);
|
||||
const memoryPath = path.join(workspaceDir, "MEMORY.md");
|
||||
const oversizedBase = `# Long-Term Memory\n\n${"u".repeat(8_800)}\n`;
|
||||
await fs.writeFile(memoryPath, oversizedBase, "utf-8");
|
||||
const consolidate = vi.fn(async () => ({ text: "unused" }));
|
||||
|
||||
const deferred = await applyAllCandidates(workspaceDir, ranked, {
|
||||
agentId: "main",
|
||||
memoryFileMaxChars: 9_000,
|
||||
nowMs: Date.parse("2026-04-29T10:00:00.000Z"),
|
||||
consolidation: {
|
||||
subagent: { complete: consolidate },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
},
|
||||
});
|
||||
|
||||
expect(deferred).toMatchObject({ applied: 0, appended: 0, compactedSections: 0 });
|
||||
expect(deferred.rejectedCandidates).toEqual([
|
||||
expect.objectContaining({ reason: expect.stringContaining("MEMORY.md budget exceeded") }),
|
||||
]);
|
||||
expect(consolidate).toHaveBeenCalledOnce();
|
||||
await expect(fs.readFile(memoryPath, "utf-8")).resolves.toBe(oversizedBase);
|
||||
|
||||
await fs.writeFile(memoryPath, "# Long-Term Memory\n\n", "utf-8");
|
||||
const retried = await applyAllCandidates(workspaceDir, ranked, {
|
||||
memoryFileMaxChars: 9_000,
|
||||
nowMs: Date.parse("2026-04-29T10:05:00.000Z"),
|
||||
});
|
||||
expect(retried.applied).toBe(1);
|
||||
expect((await fs.readFile(memoryPath, "utf-8")).length).toBeLessThanOrEqual(9_000);
|
||||
});
|
||||
|
||||
it("leaves prior promotions intact when fitting would exceed the loss limit", async (workspaceDir) => {
|
||||
await writeDailyMemoryNote(workspaceDir, "2026-04-29", [
|
||||
"Keep the Tuesday deployment window in durable memory.",
|
||||
]);
|
||||
await recordMemoryRecalls(
|
||||
workspaceDir,
|
||||
"deployment window",
|
||||
[
|
||||
memoryRecallResult(
|
||||
"memory/2026-04-29.md",
|
||||
1,
|
||||
1,
|
||||
0.95,
|
||||
"Keep the Tuesday deployment window in durable memory.",
|
||||
),
|
||||
],
|
||||
{ nowMs: Date.parse("2026-04-29T10:00:00.000Z") },
|
||||
);
|
||||
const memoryPath = path.join(workspaceDir, "MEMORY.md");
|
||||
const existing = ["2026-04-01", "2026-04-08", "2026-04-15", "2026-04-22"]
|
||||
.map(
|
||||
(date, index) =>
|
||||
`## Promoted From Short-Term Memory (${date})\n<!-- openclaw-memory-promotion:legacy-${index} -->\n- ${"x".repeat(500)}`,
|
||||
)
|
||||
.join("\n");
|
||||
await fs.writeFile(memoryPath, existing, "utf-8");
|
||||
|
||||
const result = await applyAllCandidates(workspaceDir, await rankAllCandidates(workspaceDir), {
|
||||
memoryFileMaxChars: 1_600,
|
||||
maxPriorEntryLossFraction: 0.25,
|
||||
nowMs: Date.parse("2026-04-29T10:00:00.000Z"),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ applied: 0, compactedSections: 0 });
|
||||
expect(result.rejectedCandidates[0]?.reason).toContain("MEMORY.md budget exceeded");
|
||||
await expect(fs.readFile(memoryPath, "utf-8")).resolves.toBe(existing);
|
||||
});
|
||||
});
|
||||
|
||||
it("defers append-only promotion when recall state changes during rehydration", async (workspaceDir) => {
|
||||
|
|
@ -3195,6 +3339,7 @@ describe("short-term promotion", () => {
|
|||
try {
|
||||
const applied = await applyAllCandidates(workspaceAlias, secondRanked, {
|
||||
memoryFileMaxChars: 400,
|
||||
maxPriorEntryLossFraction: 1,
|
||||
});
|
||||
expect(applied.applied).toBe(1);
|
||||
expect(await fs.readFile(targetPath, "utf-8")).toContain(secondSnippet);
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
resolveMemoryDreamingPluginConfig,
|
||||
resolveMemoryDreamingPluginId,
|
||||
resolveMemoryDreamingConfig,
|
||||
resolveMemoryDreamingWorkspace,
|
||||
resolveMemoryDreamingWorkspaces,
|
||||
} from "./dreaming.js";
|
||||
|
||||
|
|
@ -296,6 +297,10 @@ describe("memory dreaming host helpers", () => {
|
|||
expect(resolveMemoryDreamingWorkspaces(cfg)).toEqual([
|
||||
{ workspaceDir, agentIds: ["alpha", "beta"] },
|
||||
]);
|
||||
expect(resolveMemoryDreamingWorkspace(cfg, workspaceAliasDir)).toEqual({
|
||||
workspaceDir,
|
||||
agentIds: ["alpha", "beta"],
|
||||
});
|
||||
});
|
||||
|
||||
it("includes the runtime primary workspace alongside configured subagent workspaces", () => {
|
||||
|
|
|
|||
|
|
@ -685,3 +685,14 @@ export function resolveMemoryDreamingWorkspaces(
|
|||
}
|
||||
return [...byWorkspace.values()];
|
||||
}
|
||||
|
||||
export function resolveMemoryDreamingWorkspace(
|
||||
cfg: OpenClawConfig,
|
||||
workspaceDir: string,
|
||||
options: MemoryDreamingWorkspaceOptions = {},
|
||||
): MemoryDreamingWorkspace | undefined {
|
||||
const workspacePath = resolveWorkspaceStateIdentity(workspaceDir).workspacePath;
|
||||
return resolveMemoryDreamingWorkspaces(cfg, options).find(
|
||||
(entry) => resolveWorkspaceStateIdentity(entry.workspaceDir).workspacePath === workspacePath,
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ export {
|
|||
isSameMemoryDreamingDay,
|
||||
resolveMemoryDeepDreamingConfig,
|
||||
resolveMemoryDreamingConfig,
|
||||
resolveMemoryDreamingWorkspace,
|
||||
resolveMemoryDreamingWorkspaces,
|
||||
resolveMemoryLightDreamingConfig,
|
||||
resolveMemoryRemDreamingConfig,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue