From c46cc1e2d458c042ea63a4ea4331ac4d120a564f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 08:46:34 -0700 Subject: [PATCH] refactor(channels): unify inbound replay protection (#109117) * refactor(channels): unify inbound replay-guard orchestration on shared factory * fix(plugin-sdk): bind replay-guard settlement to claim handles * fix(plugin-sdk): claim-handle settlement (rest) * fix(discord): align skipped queue cleanup * style(feishu): simplify bot menu claim settlement * refactor(plugin-sdk): extract replay dedupe contracts --- .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- .../discord/src/monitor/inbound-dedupe.ts | 56 +--- .../discord/src/monitor/inbound-job.test.ts | 11 +- extensions/discord/src/monitor/inbound-job.ts | 7 +- .../discord/src/monitor/message-handler.ts | 43 ++- .../discord/src/monitor/message-run-queue.ts | 49 +--- extensions/feishu/src/bot.test.ts | 10 +- extensions/feishu/src/bot.ts | 12 +- extensions/feishu/src/dedup-state.ts | 17 +- extensions/feishu/src/dedup.test.ts | 58 +--- extensions/feishu/src/dedup.ts | 60 ++-- extensions/feishu/src/monitor.account.ts | 7 +- .../feishu/src/monitor.bot-menu-handler.ts | 27 +- .../src/monitor.comment-notice-handler.ts | 26 +- extensions/feishu/src/monitor.comment.test.ts | 39 ++- .../src/monitor.message-handler.test.ts | 1 - .../feishu/src/monitor.message-handler.ts | 213 ++++++++------ .../feishu/src/monitor.reaction.test.ts | 44 ++- .../src/monitor/inbound-dedupe.test.ts | 77 +---- .../imessage/src/monitor/inbound-dedupe.ts | 70 ++--- .../imessage/src/monitor/monitor-provider.ts | 53 ++-- extensions/line/src/bot-handlers.ts | 132 ++++----- extensions/matrix/doctor-contract-api.test.ts | 16 +- .../matrix/src/matrix/monitor/handler.test.ts | 116 ++++---- .../matrix/src/matrix/monitor/handler.ts | 23 +- .../src/matrix/monitor/inbound-dedupe.test.ts | 64 ++-- .../src/matrix/monitor/inbound-dedupe.ts | 67 ++--- .../matrix/src/matrix/monitor/index.test.ts | 18 +- .../src/mattermost/monitor-replay.ts | 57 ++-- .../mattermost/src/mattermost/monitor.test.ts | 63 ---- extensions/nextcloud-talk/src/core.test.ts | 40 +-- extensions/nextcloud-talk/src/doctor.test.ts | 2 +- extensions/nextcloud-talk/src/monitor.ts | 34 +-- extensions/nextcloud-talk/src/replay-guard.ts | 99 ++----- extensions/raft/src/gateway.test.ts | 27 +- extensions/raft/src/gateway.ts | 54 ++-- .../bot-handlers.inbound-debounce.runtime.ts | 21 +- ...ot-handlers.inbound-media-group.runtime.ts | 21 +- .../src/bot-handlers.inbound-text.runtime.ts | 27 +- .../src/bot-handlers.inbound.runtime.ts | 21 +- .../bot-handlers.message-events.runtime.ts | 11 +- .../bot-handlers.message-lifecycle.runtime.ts | 33 ++- .../src/bot-handlers.message.runtime.ts | 27 +- .../src/message-dispatch-dedupe.test.ts | 202 +++++++------ .../telegram/src/message-dispatch-dedupe.ts | 116 ++++---- .../telegram/src/polling-session.test.ts | 19 +- extensions/whatsapp/src/inbound/dedupe.ts | 32 +- extensions/whatsapp/src/inbound/monitor.ts | 32 +- extensions/zalo/src/monitor.webhook.ts | 55 ++-- scripts/plugin-sdk-surface-report.mjs | 6 +- src/plugin-sdk/channel-replay-guard.ts | 275 ++++++++++++++++++ src/plugin-sdk/persistent-dedupe.test.ts | 155 ++++++++++ src/plugin-sdk/persistent-dedupe.ts | 157 +++------- src/plugin-sdk/persistent-dedupe.types.ts | 122 ++++++++ 54 files changed, 1561 insertions(+), 1467 deletions(-) create mode 100644 src/plugin-sdk/channel-replay-guard.ts create mode 100644 src/plugin-sdk/persistent-dedupe.test.ts create mode 100644 src/plugin-sdk/persistent-dedupe.types.ts diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 2c5b9e2bce2..e7c4990c80b 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -a1d46522f78815ebda945360e6814772eb0b563350c4f0ee9cf2f414ad9dc948 plugin-sdk-api-baseline.json -2dc9fd508e3875a00113171b8dabe738554c049c0083f5d05a4f1f82cc5320da plugin-sdk-api-baseline.jsonl +e59682fe6e11b4bd6e47d585105b75f417ccc774e7085569336fffd478200be2 plugin-sdk-api-baseline.json +4db490b8a21d1b4ce20f3e4d2f053b034d3ae039c79b2dbba995d13a5c4b0150 plugin-sdk-api-baseline.jsonl diff --git a/extensions/discord/src/monitor/inbound-dedupe.ts b/extensions/discord/src/monitor/inbound-dedupe.ts index c75d97c2717..aa20f244ca5 100644 --- a/extensions/discord/src/monitor/inbound-dedupe.ts +++ b/extensions/discord/src/monitor/inbound-dedupe.ts @@ -1,15 +1,20 @@ // Discord plugin module implements inbound dedupe behavior. -import { createClaimableDedupe, type ClaimableDedupe } from "openclaw/plugin-sdk/persistent-dedupe"; +import { createChannelReplayGuard } from "openclaw/plugin-sdk/persistent-dedupe"; import type { DiscordMessageEvent } from "./listeners.js"; import { resolveDiscordMessageChannelId } from "./message-utils.js"; const RECENT_DISCORD_MESSAGE_TTL_MS = 5 * 60_000; const RECENT_DISCORD_MESSAGE_MAX = 5000; -export function createDiscordInboundReplayGuard(): ClaimableDedupe { - return createClaimableDedupe({ - ttlMs: RECENT_DISCORD_MESSAGE_TTL_MS, - memoryMaxSize: RECENT_DISCORD_MESSAGE_MAX, +type DiscordInboundReplayKeys = string | readonly (string | null | undefined)[] | null | undefined; + +export function createDiscordInboundReplayGuard() { + return createChannelReplayGuard({ + dedupe: { + ttlMs: RECENT_DISCORD_MESSAGE_TTL_MS, + memoryMaxSize: RECENT_DISCORD_MESSAGE_MAX, + }, + buildReplayKey: (keys) => keys, }); } @@ -37,44 +42,3 @@ export function buildDiscordInboundReplayKey(params: { } return `${params.accountId}:${channelId}:${messageId}`; } - -export async function claimDiscordInboundReplay(params: { - replayKey?: string | null; - replayGuard: ClaimableDedupe; -}): Promise { - const replayKey = params.replayKey?.trim(); - if (!replayKey) { - return true; - } - const claim = await params.replayGuard.claim(replayKey); - return claim.kind === "claimed"; -} - -export async function commitDiscordInboundReplay(params: { - replayKeys?: readonly (string | null | undefined)[]; - replayGuard: ClaimableDedupe; -}): Promise { - const replayKeys = normalizeDiscordInboundReplayKeys(params.replayKeys); - await Promise.all(replayKeys.map((replayKey) => params.replayGuard.commit(replayKey))); -} - -export function releaseDiscordInboundReplay(params: { - replayKeys?: readonly (string | null | undefined)[]; - replayGuard: ClaimableDedupe; - error?: unknown; -}): void { - const replayKeys = normalizeDiscordInboundReplayKeys(params.replayKeys); - replayKeys.forEach((replayKey) => params.replayGuard.release(replayKey, { error: params.error })); -} - -function normalizeDiscordInboundReplayKeys( - replayKeys?: readonly (string | null | undefined)[], -): string[] { - return [ - ...new Set( - (replayKeys ?? []) - .map((replayKey) => replayKey?.trim()) - .filter((replayKey): replayKey is string => Boolean(replayKey)), - ), - ]; -} diff --git a/extensions/discord/src/monitor/inbound-job.test.ts b/extensions/discord/src/monitor/inbound-job.test.ts index 0b0082d83ad..849cc20bb33 100644 --- a/extensions/discord/src/monitor/inbound-job.test.ts +++ b/extensions/discord/src/monitor/inbound-job.test.ts @@ -1,5 +1,5 @@ // Discord tests cover inbound job plugin behavior. -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { Message } from "../internal/discord.js"; import { createPartialDiscordChannelWithThrowingGetters } from "../test-support/partial-channel.js"; import { buildDiscordInboundJob, materializeDiscordInboundJob } from "./inbound-job.js"; @@ -143,7 +143,12 @@ describe("buildDiscordInboundJob", () => { it("re-materializes the process context with an overridden abort signal", async () => { const ctx = await createBaseDiscordMessageContext(); - const job = buildDiscordInboundJob(ctx, { replayKeys: ["default:ch-1:m-1"] }); + const replayClaim = { + keys: ["default:ch-1:m-1"] as const, + commit: vi.fn(async () => true), + release: vi.fn(), + }; + const job = buildDiscordInboundJob(ctx, { replayClaims: [replayClaim] }); const overrideAbortController = new AbortController(); const rematerialized = materializeDiscordInboundJob(job, overrideAbortController.signal); @@ -154,7 +159,7 @@ describe("buildDiscordInboundJob", () => { expect(rematerialized.abortSignal).toBe(overrideAbortController.signal); expect(rematerialized.message).toEqual(job.payload.message); expect(rematerialized.data).toEqual(job.payload.data); - expect(job.replayKeys).toEqual(["default:ch-1:m-1"]); + expect(job.replayClaims).toEqual([replayClaim]); }); it("preserves Discord message getters across queued jobs", async () => { diff --git a/extensions/discord/src/monitor/inbound-job.ts b/extensions/discord/src/monitor/inbound-job.ts index 05efe883b1a..a76c775ffa6 100644 --- a/extensions/discord/src/monitor/inbound-job.ts +++ b/extensions/discord/src/monitor/inbound-job.ts @@ -1,4 +1,5 @@ // Discord plugin module implements inbound job behavior. +import type { ChannelReplayClaimHandle } from "openclaw/plugin-sdk/persistent-dedupe"; import { resolveDiscordChannelIdSafe, resolveDiscordChannelInfoSafe, @@ -23,7 +24,7 @@ export type DiscordInboundJob = { queueKey: string; payload: DiscordInboundJobPayload; runtime: DiscordInboundJobRuntime; - replayKeys?: string[]; + replayClaims?: readonly ChannelReplayClaimHandle[]; }; function resolveDiscordInboundJobQueueKey(ctx: DiscordMessagePreflightContext): string { @@ -42,7 +43,7 @@ function resolveDiscordInboundJobQueueKey(ctx: DiscordMessagePreflightContext): export function buildDiscordInboundJob( ctx: DiscordMessagePreflightContext, - options?: { replayKeys?: readonly string[] }, + options?: { replayClaims?: readonly ChannelReplayClaimHandle[] }, ): DiscordInboundJob { const { runtime, @@ -77,7 +78,7 @@ export function buildDiscordInboundJob( threadBindings, discordRestFetch, }, - replayKeys: options?.replayKeys ? [...options.replayKeys] : undefined, + replayClaims: options?.replayClaims, }; } diff --git a/extensions/discord/src/monitor/message-handler.ts b/extensions/discord/src/monitor/message-handler.ts index 75924d595b1..c09c63b7f78 100644 --- a/extensions/discord/src/monitor/message-handler.ts +++ b/extensions/discord/src/monitor/message-handler.ts @@ -4,16 +4,14 @@ import { shouldDebounceTextInbound, } from "openclaw/plugin-sdk/channel-inbound"; import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime"; +import type { ChannelReplayClaimHandle } from "openclaw/plugin-sdk/persistent-dedupe"; import { danger } from "openclaw/plugin-sdk/runtime-env"; import { resolveOpenProviderRuntimeGroupPolicy } from "openclaw/plugin-sdk/runtime-group-policy"; import type { Client } from "../internal/discord.js"; import { buildDiscordInboundReplayKey, - claimDiscordInboundReplay, - commitDiscordInboundReplay, createDiscordInboundReplayGuard, DiscordRetryableInboundError, - releaseDiscordInboundReplay, } from "./inbound-dedupe.js"; import { buildDiscordInboundJob } from "./inbound-job.js"; import type { DiscordMessageEvent, DiscordMessageHandler } from "./listeners.js"; @@ -76,7 +74,6 @@ export function createDiscordMessageHandler( runtime: params.runtime, setStatus: params.setStatus, abortSignal: params.abortSignal, - replayGuard, testing: params.testing, }); @@ -84,7 +81,7 @@ export function createDiscordMessageHandler( data: DiscordMessageEvent; client: Client; abortSignal?: AbortSignal; - replayKey?: string; + replayClaim?: ChannelReplayClaimHandle; }>({ cfg: params.cfg, channel: "discord", @@ -122,14 +119,14 @@ export function createDiscordMessageHandler( if (!last) { return; } - const replayKeys = entries.map((entry) => entry.replayKey).filter(isNonEmptyString); + const replayClaims = entries + .map((entry) => entry.replayClaim) + .filter((claim): claim is ChannelReplayClaimHandle => claim !== undefined); const abortSignal = last.abortSignal; if (abortSignal?.aborted) { - releaseDiscordInboundReplay({ - replayKeys, - error: abortSignal.reason, - replayGuard, - }); + for (const claim of replayClaims) { + claim.release({ error: abortSignal.reason }); + } return; } try { @@ -146,11 +143,11 @@ export function createDiscordMessageHandler( client: last.client, }); if (!ctx) { - await commitDiscordInboundReplay({ replayKeys, replayGuard }); + await Promise.all(replayClaims.map((claim) => claim.commit())); return; } applyImplicitReplyBatchGate(ctx, params.replyToMode, false); - messageRunQueue.enqueue(buildDiscordInboundJob(ctx, { replayKeys })); + messageRunQueue.enqueue(buildDiscordInboundJob(ctx, { replayClaims })); return; } const combinedBaseText = entries @@ -195,7 +192,7 @@ export function createDiscordMessageHandler( client: last.client, }); if (!ctx) { - await commitDiscordInboundReplay({ replayKeys, replayGuard }); + await Promise.all(replayClaims.map((claim) => claim.commit())); return; } applyImplicitReplyBatchGate(ctx, params.replyToMode, true); @@ -212,12 +209,14 @@ export function createDiscordMessageHandler( ctxBatch.MessageSidLast = ids[ids.length - 1]; } } - messageRunQueue.enqueue(buildDiscordInboundJob(ctx, { replayKeys })); + messageRunQueue.enqueue(buildDiscordInboundJob(ctx, { replayClaims })); } catch (error) { if (error instanceof DiscordRetryableInboundError) { - releaseDiscordInboundReplay({ replayKeys, error, replayGuard }); + for (const claim of replayClaims) { + claim.release({ error }); + } } else { - await commitDiscordInboundReplay({ replayKeys, replayGuard }); + await Promise.all(replayClaims.map((claim) => claim.commit())); } throw error; } @@ -245,12 +244,8 @@ export function createDiscordMessageHandler( accountId: params.accountId, data, }); - if ( - !(await claimDiscordInboundReplay({ - replayKey, - replayGuard, - })) - ) { + const replayClaim = await replayGuard.claim(replayKey); + if (replayClaim.kind !== "claimed" && replayClaim.kind !== "invalid") { return; } @@ -258,7 +253,7 @@ export function createDiscordMessageHandler( data, client, abortSignal: options?.abortSignal, - replayKey: replayKey ?? undefined, + ...(replayClaim.kind === "claimed" ? { replayClaim: replayClaim.handle } : {}), }); } catch (err) { params.runtime.error(danger(`handler failed: ${String(err)}`)); diff --git a/extensions/discord/src/monitor/message-run-queue.ts b/extensions/discord/src/monitor/message-run-queue.ts index aa34773bc1f..5f5baf18bae 100644 --- a/extensions/discord/src/monitor/message-run-queue.ts +++ b/extensions/discord/src/monitor/message-run-queue.ts @@ -1,14 +1,8 @@ // Discord plugin module implements message run queue behavior. import { createChannelRunQueue } from "openclaw/plugin-sdk/channel-outbound"; import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime"; -import type { ClaimableDedupe } from "openclaw/plugin-sdk/persistent-dedupe"; import { danger } from "openclaw/plugin-sdk/runtime-env"; -import { - commitDiscordInboundReplay, - createDiscordInboundReplayGuard, - DiscordRetryableInboundError, - releaseDiscordInboundReplay, -} from "./inbound-dedupe.js"; +import { DiscordRetryableInboundError } from "./inbound-dedupe.js"; import { materializeDiscordInboundJob, type DiscordInboundJob } from "./inbound-job.js"; import type { RuntimeEnv } from "./message-handler.preflight.types.js"; import type { DiscordMonitorStatusSink } from "./status.js"; @@ -19,7 +13,6 @@ type DiscordMessageRunQueueParams = { runtime: RuntimeEnv; setStatus?: DiscordMonitorStatusSink; abortSignal?: AbortSignal; - replayGuard?: ClaimableDedupe; testing?: DiscordMessageRunQueueTestingHooks; }; @@ -41,7 +34,6 @@ const loadMessageProcessRuntime = createLazyRuntimeModule( async function processDiscordQueuedMessage(params: { job: DiscordInboundJob; lifecycleSignal?: AbortSignal; - replayGuard: ClaimableDedupe; testing?: DiscordMessageRunQueueTestingHooks; }) { const processDiscordMessageImpl = @@ -53,42 +45,32 @@ async function processDiscordQueuedMessage(params: { : (params.job.runtime.abortSignal ?? params.lifecycleSignal); try { await processDiscordMessageImpl(materializeDiscordInboundJob(params.job, abortSignal)); - await commitDiscordInboundReplay({ - replayKeys: params.job.replayKeys, - replayGuard: params.replayGuard, - }); + await Promise.all(params.job.replayClaims?.map((claim) => claim.commit()) ?? []); } catch (error) { if (error instanceof DiscordRetryableInboundError) { - releaseDiscordInboundReplay({ - replayKeys: params.job.replayKeys, - error, - replayGuard: params.replayGuard, - }); + for (const claim of params.job.replayClaims ?? []) { + claim.release({ error }); + } } else { - await commitDiscordInboundReplay({ - replayKeys: params.job.replayKeys, - replayGuard: params.replayGuard, - }); + await Promise.all(params.job.replayClaims?.map((claim) => claim.commit()) ?? []); } throw error; } } -function cleanupSkippedDiscordQueuedMessage(params: { - job: DiscordInboundJob; - replayGuard: ClaimableDedupe; -}) { - releaseDiscordInboundReplay({ - replayKeys: params.job.replayKeys, - error: new DiscordRetryableInboundError("discord queued run skipped before processing"), - replayGuard: params.replayGuard, - }); +function cleanupSkippedDiscordQueuedMessage(params: { job: DiscordInboundJob }) { + // Typing feedback is created inside processing after admission, so skipped + // jobs only carry replay claims that need reopening for a later retry. + for (const claim of params.job.replayClaims ?? []) { + claim.release({ + error: new DiscordRetryableInboundError("discord queued run skipped before processing"), + }); + } } export function createDiscordMessageRunQueue( params: DiscordMessageRunQueueParams, ): DiscordMessageRunQueue { - const replayGuard = params.replayGuard ?? createDiscordInboundReplayGuard(); const skippedCleanup = new Set(); const runQueue = createChannelRunQueue({ setStatus: params.setStatus, @@ -122,7 +104,7 @@ export function createDiscordMessageRunQueue( return { enqueue(job) { const cleanupSkipped = () => { - cleanupSkippedDiscordQueuedMessage({ job, replayGuard }); + cleanupSkippedDiscordQueuedMessage({ job }); }; if (!lifecycleActive) { cleanupSkipped(); @@ -136,7 +118,6 @@ export function createDiscordMessageRunQueue( await processDiscordQueuedMessage({ job, lifecycleSignal, - replayGuard, testing: params.testing, }); }); diff --git a/extensions/feishu/src/bot.test.ts b/extensions/feishu/src/bot.test.ts index a029d7f084a..684d595fa3c 100644 --- a/extensions/feishu/src/bot.test.ts +++ b/extensions/feishu/src/bot.test.ts @@ -4508,7 +4508,6 @@ describe("createFeishuMessageReceiveHandler media dedupe", () => { resolveDebounceText: ({ event }) => (JSON.parse(event.message.content) as { text: string }).text, hasProcessedMessage: vi.fn(async () => false), - recordProcessedMessage: vi.fn(async () => true), }); await handler(createTextEvent("msg-text-first", "1710000000000", "first")); @@ -4569,7 +4568,6 @@ describe("createFeishuMessageReceiveHandler media dedupe", () => { handleMessage, resolveDebounceText: () => "", hasProcessedMessage: vi.fn(async () => false), - recordProcessedMessage: vi.fn(async () => true), }); const firstEvent = createAudioEvent("file_audio_receive_first"); @@ -4581,16 +4579,16 @@ describe("createFeishuMessageReceiveHandler media dedupe", () => { expect(handleMessage).toHaveBeenCalledTimes(2); const firstCall = mockCallArg<{ event?: FeishuMessageEvent; - processingClaimHeld?: boolean; + processingClaim?: { commit: () => Promise }; }>(handleMessage, 0, 0); expect(firstCall.event).toEqual(firstEvent); - expect(firstCall.processingClaimHeld).toBe(true); + expect(firstCall.processingClaim?.commit).toBeTypeOf("function"); const secondCall = mockCallArg<{ event?: FeishuMessageEvent; - processingClaimHeld?: boolean; + processingClaim?: { commit: () => Promise }; }>(handleMessage, 1, 0); expect(secondCall.event).toEqual(secondEvent); - expect(secondCall.processingClaimHeld).toBe(true); + expect(secondCall.processingClaim?.commit).toBeTypeOf("function"); }); }); /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/extensions/feishu/src/bot.ts b/extensions/feishu/src/bot.ts index 49de2d80905..fd54e20663c 100644 --- a/extensions/feishu/src/bot.ts +++ b/extensions/feishu/src/bot.ts @@ -49,7 +49,11 @@ import type { ClawdbotConfig, RuntimeEnv } from "./bot-runtime-api.js"; import { resolveFeishuSenderName, type FeishuPermissionError } from "./bot-sender-name.js"; import { createFeishuClient } from "./client.js"; import { resolveConfiguredFeishuGroupSessionScope } from "./conversation-id.js"; -import { finalizeFeishuMessageProcessing, recordProcessedFeishuMessage } from "./dedup.js"; +import { + finalizeFeishuMessageProcessing, + recordProcessedFeishuMessage, + type FeishuMessageProcessingClaim, +} from "./dedup.js"; import { resolveFeishuMessageDedupeKey } from "./dedupe-key.js"; import { maybeCreateDynamicAgent } from "./dynamic-agent.js"; import { extractMentionTargets, isMentionForwardRequest } from "./mention.js"; @@ -270,7 +274,7 @@ export async function handleFeishuMessage(params: { channelRuntime?: ReturnType["channel"]; chatHistories?: Map; accountId?: string; - processingClaimHeld?: boolean; + processingClaim?: FeishuMessageProcessingClaim; messageDedupeKey?: string; }): Promise { const { @@ -282,7 +286,7 @@ export async function handleFeishuMessage(params: { channelRuntime, chatHistories, accountId, - processingClaimHeld = false, + processingClaim, messageDedupeKey: messageDedupeKeyOverride, } = params; @@ -300,7 +304,7 @@ export async function handleFeishuMessage(params: { messageId: messageDedupeKey, namespace: account.accountId, log, - claimHeld: processingClaimHeld, + processingClaim, })) ) { log(`feishu: skipping duplicate message ${messageId}`); diff --git a/extensions/feishu/src/dedup-state.ts b/extensions/feishu/src/dedup-state.ts index 729865e3da8..c64c70db796 100644 --- a/extensions/feishu/src/dedup-state.ts +++ b/extensions/feishu/src/dedup-state.ts @@ -1,4 +1,4 @@ -import { createClaimableDedupe } from "openclaw/plugin-sdk/persistent-dedupe"; +import { createChannelReplayGuard } from "openclaw/plugin-sdk/persistent-dedupe"; const DEDUPE_NAMESPACE_PREFIX = "feishu.dedup"; const DEDUP_TTL_MS = 24 * 60 * 60 * 1000; @@ -6,12 +6,15 @@ const MEMORY_MAX_SIZE = 1_000; const STORE_MAX_ENTRIES = 10_000; function createFeishuDedupeGuard() { - return createClaimableDedupe({ - pluginId: "feishu", - namespacePrefix: DEDUPE_NAMESPACE_PREFIX, - ttlMs: DEDUP_TTL_MS, - memoryMaxSize: MEMORY_MAX_SIZE, - stateMaxEntries: STORE_MAX_ENTRIES, + return createChannelReplayGuard({ + dedupe: { + pluginId: "feishu", + namespacePrefix: DEDUPE_NAMESPACE_PREFIX, + ttlMs: DEDUP_TTL_MS, + memoryMaxSize: MEMORY_MAX_SIZE, + stateMaxEntries: STORE_MAX_ENTRIES, + }, + buildReplayKey: (messageId) => messageId, }); } diff --git a/extensions/feishu/src/dedup.test.ts b/extensions/feishu/src/dedup.test.ts index 81ff4583e5a..aa3fa1de04b 100644 --- a/extensions/feishu/src/dedup.test.ts +++ b/extensions/feishu/src/dedup.test.ts @@ -10,7 +10,6 @@ import { finalizeFeishuMessageProcessing, hasProcessedFeishuMessage, recordProcessedFeishuMessage, - releaseFeishuMessageProcessing, warmupDedupFromPluginState, } from "./dedup.js"; @@ -45,45 +44,6 @@ async function restartFeishuDedup(): Promise { } describe("Feishu claimable dedupe", () => { - it("drops a duplicate message within the TTL after commit", async () => { - await expect( - claimUnprocessedFeishuMessage({ messageId: "msg-1", namespace: "account-a" }), - ).resolves.toBe("claimed"); - await expect(recordProcessedFeishuMessage("msg-1", "account-a")).resolves.toBe(true); - - await expect( - claimUnprocessedFeishuMessage({ messageId: "msg-1", namespace: "account-a" }), - ).resolves.toBe("duplicate"); - await expect(hasProcessedFeishuMessage("msg-1", "account-a")).resolves.toBe(true); - await expect(hasProcessedFeishuMessage("msg-1", "account-b")).resolves.toBe(false); - }); - - it("reports an in-flight claim and lets a released claim retry", async () => { - await expect( - claimUnprocessedFeishuMessage({ messageId: "msg-2", namespace: "account-a" }), - ).resolves.toBe("claimed"); - await expect( - claimUnprocessedFeishuMessage({ messageId: "msg-2", namespace: "account-a" }), - ).resolves.toBe("inflight"); - - releaseFeishuMessageProcessing("msg-2", "account-a"); - await expect( - claimUnprocessedFeishuMessage({ messageId: "msg-2", namespace: "account-a" }), - ).resolves.toBe("claimed"); - }); - - it("does not persist released claims across a restart", async () => { - await expect( - claimUnprocessedFeishuMessage({ messageId: "msg-3", namespace: "account-a" }), - ).resolves.toBe("claimed"); - releaseFeishuMessageProcessing("msg-3", "account-a"); - - await restartFeishuDedup(); - await expect( - claimUnprocessedFeishuMessage({ messageId: "msg-3", namespace: "account-a" }), - ).resolves.toBe("claimed"); - }); - it("prevents replay after a restart once a message is committed", async () => { await expect( finalizeFeishuMessageProcessing({ messageId: "msg-4", namespace: "account-a" }), @@ -92,28 +52,32 @@ describe("Feishu claimable dedupe", () => { await restartFeishuDedup(); await expect( claimUnprocessedFeishuMessage({ messageId: "msg-4", namespace: "account-a" }), - ).resolves.toBe("duplicate"); + ).resolves.toEqual({ kind: "duplicate" }); await expect( finalizeFeishuMessageProcessing({ messageId: "msg-4", namespace: "account-a" }), ).resolves.toBe(false); }); it("commits a held claim without reclaiming it", async () => { - await expect( - claimUnprocessedFeishuMessage({ messageId: "msg-5", namespace: "account-a" }), - ).resolves.toBe("claimed"); + const claim = await claimUnprocessedFeishuMessage({ + messageId: "msg-5", + namespace: "account-a", + }); + expect(claim.kind).toBe("claimed"); + if (claim.kind !== "claimed") { + throw new Error(`expected claimed result, received ${claim.kind}`); + } await expect( finalizeFeishuMessageProcessing({ messageId: "msg-5", namespace: "account-a", - claimHeld: true, + processingClaim: claim.handle, }), ).resolves.toBe(true); await expect( finalizeFeishuMessageProcessing({ messageId: "msg-5", namespace: "account-a", - claimHeld: true, }), ).resolves.toBe(false); }); @@ -156,7 +120,7 @@ describe("Feishu claimable dedupe", () => { await expect(recordProcessedFeishuMessage("msg-9", "account-a", log)).resolves.toBe(true); await expect( claimUnprocessedFeishuMessage({ messageId: "msg-9", namespace: "account-a", log }), - ).resolves.toBe("duplicate"); + ).resolves.toEqual({ kind: "duplicate" }); expect(log).toHaveBeenCalledWith( expect.stringContaining("feishu-dedup: persistent state error"), ); diff --git a/extensions/feishu/src/dedup.ts b/extensions/feishu/src/dedup.ts index 9f18fcd270a..f990f3b6958 100644 --- a/extensions/feishu/src/dedup.ts +++ b/extensions/feishu/src/dedup.ts @@ -3,11 +3,18 @@ // the same event once per bot, so handlers claim a dedupe key before // processing, commit once handling is dispatched, and release on retryable // failure so the event can be redelivered. +import type { ChannelReplayClaimHandle } from "openclaw/plugin-sdk/persistent-dedupe"; import { feishuDedupeState } from "./dedup-state.js"; type FeishuDedupeLog = (...args: unknown[]) => void; -type FeishuMessageClaim = "claimed" | "duplicate" | "inflight"; +export type FeishuMessageProcessingClaim = ChannelReplayClaimHandle; + +type FeishuMessageClaim = + | { kind: "claimed"; handle: FeishuMessageProcessingClaim } + | { kind: "duplicate" } + | { kind: "inflight" } + | { kind: "invalid" }; function dedupeKey(messageId: string | undefined | null): string { return messageId?.trim() ?? ""; @@ -30,7 +37,7 @@ function dedupeOptions(namespace: string | undefined, log: FeishuDedupeLog | und /** * Claims a dedupe key for exclusive handling. Duplicate (already committed) - * and in-flight keys are reported; blank keys fail open as claimed so an + * and in-flight keys are reported; blank keys fail open as invalid so an * unidentifiable event is never suppressed. */ export async function claimUnprocessedFeishuMessage(params: { @@ -38,23 +45,14 @@ export async function claimUnprocessedFeishuMessage(params: { namespace?: string; log?: FeishuDedupeLog; }): Promise { - const key = dedupeKey(params.messageId); - if (!key) { - return "claimed"; - } - return (await feishuDedupeState.guard.claim(key, dedupeOptions(params.namespace, params.log))) - .kind; -} - -/** Drops an uncommitted claim so a failed handler can retry the message. */ -export function releaseFeishuMessageProcessing( - messageId: string | undefined | null, - namespace = "global", -): void { - const key = dedupeKey(messageId); - if (key) { - feishuDedupeState.guard.release(key, { namespace }); + const claim = await feishuDedupeState.guard.claim( + params.messageId, + dedupeOptions(params.namespace, params.log), + ); + if (claim.kind === "inflight") { + return { kind: "inflight" }; } + return claim; } /** @@ -66,17 +64,18 @@ export async function finalizeFeishuMessageProcessing(params: { messageId: string | undefined | null; namespace?: string; log?: FeishuDedupeLog; - claimHeld?: boolean; + processingClaim?: FeishuMessageProcessingClaim; }): Promise { const key = dedupeKey(params.messageId); if (!key) { return false; } const options = dedupeOptions(params.namespace, params.log); - if (!params.claimHeld && (await feishuDedupeState.guard.claim(key, options)).kind !== "claimed") { + const claim = params.processingClaim ?? (await feishuDedupeState.guard.claim(key, options)); + if ("kind" in claim && claim.kind !== "claimed") { return false; } - return await feishuDedupeState.guard.commit(key, options); + return await ("kind" in claim ? claim.handle : claim).commit(); } /** Records a handled message so restart/replay cannot dispatch it again; false when already recorded. */ @@ -85,11 +84,8 @@ export async function recordProcessedFeishuMessage( namespace = "global", log?: FeishuDedupeLog, ): Promise { - const key = dedupeKey(messageId); - if (!key) { - return false; - } - return await feishuDedupeState.guard.commit(key, dedupeOptions(namespace, log)); + const claim = await feishuDedupeState.guard.claim(messageId, dedupeOptions(namespace, log)); + return claim.kind === "claimed" ? await claim.handle.commit() : false; } /** Forgets a recorded message so a retryable synthetic event can be handled on redelivery. */ @@ -98,11 +94,7 @@ export async function forgetProcessedFeishuMessage( namespace = "global", log?: FeishuDedupeLog, ): Promise { - const key = dedupeKey(messageId); - if (!key) { - return false; - } - return await feishuDedupeState.guard.forget(key, dedupeOptions(namespace, log)); + return await feishuDedupeState.guard.forget(messageId, dedupeOptions(namespace, log)); } /** Checks recency without claiming or recording. */ @@ -111,11 +103,7 @@ export async function hasProcessedFeishuMessage( namespace = "global", log?: FeishuDedupeLog, ): Promise { - const key = dedupeKey(messageId); - if (!key) { - return false; - } - return await feishuDedupeState.guard.hasRecent(key, dedupeOptions(namespace, log)); + return await feishuDedupeState.guard.hasRecent(messageId, dedupeOptions(namespace, log)); } /** Loads recent persisted entries into memory at account start; returns the loaded count. */ diff --git a/extensions/feishu/src/monitor.account.ts b/extensions/feishu/src/monitor.account.ts index 1cf574e8b26..6103a129e82 100644 --- a/extensions/feishu/src/monitor.account.ts +++ b/extensions/feishu/src/monitor.account.ts @@ -12,11 +12,7 @@ import { import { handleFeishuCardAction, type FeishuCardActionEvent } from "./card-action.js"; import { createEventDispatcher } from "./client.js"; import { isRecord, readString } from "./comment-shared.js"; -import { - hasProcessedFeishuMessage, - recordProcessedFeishuMessage, - warmupDedupFromPluginState, -} from "./dedup.js"; +import { hasProcessedFeishuMessage, warmupDedupFromPluginState } from "./dedup.js"; import { applyBotIdentityState, startBotIdentityRecovery } from "./monitor.bot-identity.js"; import { createFeishuBotMenuHandler } from "./monitor.bot-menu-handler.js"; import { createFeishuDriveCommentNoticeHandler } from "./monitor.comment-notice-handler.js"; @@ -307,7 +303,6 @@ function registerEventHandlers( resolveDebounceText: ({ event, botOpenId, botName }) => parseFeishuMessageEvent(event, botOpenId, botName).content, hasProcessedMessage: hasProcessedFeishuMessage, - recordProcessedMessage: recordProcessedFeishuMessage, getBotOpenId: (id) => botOpenIds.get(id), getBotName: (id) => botNames.get(id), resolveSequentialKey: getFeishuSequentialKey, diff --git a/extensions/feishu/src/monitor.bot-menu-handler.ts b/extensions/feishu/src/monitor.bot-menu-handler.ts index 1dbc91f15ab..223e8125bbd 100644 --- a/extensions/feishu/src/monitor.bot-menu-handler.ts +++ b/extensions/feishu/src/monitor.bot-menu-handler.ts @@ -3,12 +3,7 @@ import { isRecord, readStringValue as readString } from "openclaw/plugin-sdk/str import type { ClawdbotConfig, HistoryEntry, PluginRuntime, RuntimeEnv } from "../runtime-api.js"; import { handleFeishuMessage, type FeishuMessageEvent } from "./bot.js"; import { maybeHandleFeishuQuickActionMenu } from "./card-ux-launcher.js"; -import { - claimUnprocessedFeishuMessage, - forgetProcessedFeishuMessage, - recordProcessedFeishuMessage, - releaseFeishuMessageProcessing, -} from "./dedup.js"; +import { claimUnprocessedFeishuMessage, forgetProcessedFeishuMessage } from "./dedup.js"; import { botNames, botOpenIds } from "./monitor.state.js"; import { isFeishuRetryableSyntheticEventError } from "./monitor.synthetic-error.js"; @@ -104,11 +99,11 @@ export function createFeishuBotMenuHandler(params: { namespace: accountId, log, }); - if (claim === "duplicate") { + if (claim.kind === "duplicate") { log(`feishu[${accountId}]: dropping duplicate bot-menu event for ${syntheticMessageId}`); return; } - if (claim === "inflight") { + if (claim.kind === "inflight") { log(`feishu[${accountId}]: dropping in-flight bot-menu event for ${syntheticMessageId}`); return; } @@ -122,7 +117,7 @@ export function createFeishuBotMenuHandler(params: { channelRuntime: params.channelRuntime, chatHistories, accountId, - processingClaimHeld: true, + processingClaim: claim.kind === "claimed" ? claim.handle : undefined, }); const promise = maybeHandleFeishuQuickActionMenu({ @@ -134,7 +129,9 @@ export function createFeishuBotMenuHandler(params: { }) .then(async (handledMenu) => { if (handledMenu) { - await recordProcessedFeishuMessage(syntheticMessageId, accountId, log); + if (claim.kind === "claimed") { + await claim.handle.commit(); + } return; } return await handleLegacyMenu(); @@ -142,13 +139,13 @@ export function createFeishuBotMenuHandler(params: { .catch(async (err: unknown) => { if (isFeishuRetryableSyntheticEventError(err)) { await forgetProcessedFeishuMessage(syntheticMessageId, accountId, log); - } else { - await recordProcessedFeishuMessage(syntheticMessageId, accountId, log); + if (claim.kind === "claimed") { + claim.handle.release({ error: err }); + } + } else if (claim.kind === "claimed") { + await claim.handle.commit(); } throw err; - }) - .finally(() => { - releaseFeishuMessageProcessing(syntheticMessageId, accountId); }); if (fireAndForget) { promise.catch((err: unknown) => { diff --git a/extensions/feishu/src/monitor.comment-notice-handler.ts b/extensions/feishu/src/monitor.comment-notice-handler.ts index 413a3da51a0..7c903e03291 100644 --- a/extensions/feishu/src/monitor.comment-notice-handler.ts +++ b/extensions/feishu/src/monitor.comment-notice-handler.ts @@ -1,11 +1,7 @@ // Feishu plugin module implements monitor.comment notice handler behavior. import type { ClawdbotConfig, RuntimeEnv } from "../runtime-api.js"; import { handleFeishuCommentEvent } from "./comment-handler.js"; -import { - claimUnprocessedFeishuMessage, - recordProcessedFeishuMessage, - releaseFeishuMessageProcessing, -} from "./dedup.js"; +import { claimUnprocessedFeishuMessage, type FeishuMessageProcessingClaim } from "./dedup.js"; import { parseFeishuDriveCommentNoticeEventPayload } from "./monitor.comment.js"; import { botOpenIds } from "./monitor.state.js"; import { isFeishuRetryableSyntheticEventError } from "./monitor.synthetic-error.js"; @@ -53,20 +49,22 @@ export function createFeishuDriveCommentNoticeHandler(params: { } const eventId = event.event_id?.trim(); const syntheticMessageId = eventId ? `drive-comment:${eventId}` : undefined; + let processingClaim: FeishuMessageProcessingClaim | undefined; if (syntheticMessageId) { const claim = await claimUnprocessedFeishuMessage({ messageId: syntheticMessageId, namespace: accountId, log, }); - if (claim === "duplicate") { + if (claim.kind === "duplicate") { log(`feishu[${accountId}]: dropping duplicate comment event ${syntheticMessageId}`); return; } - if (claim === "inflight") { + if (claim.kind === "inflight") { log(`feishu[${accountId}]: dropping in-flight comment event ${syntheticMessageId}`); return; } + processingClaim = claim.kind === "claimed" ? claim.handle : undefined; } log( `feishu[${accountId}]: received drive comment notice ` + @@ -88,18 +86,14 @@ export function createFeishuDriveCommentNoticeHandler(params: { runtime, }); }); - if (syntheticMessageId) { - await recordProcessedFeishuMessage(syntheticMessageId, accountId, log); - } + await processingClaim?.commit(); } catch (err) { - if (syntheticMessageId && !isFeishuRetryableSyntheticEventError(err)) { - await recordProcessedFeishuMessage(syntheticMessageId, accountId, log); + if (isFeishuRetryableSyntheticEventError(err)) { + processingClaim?.release({ error: err }); + } else { + await processingClaim?.commit(); } throw err; - } finally { - if (syntheticMessageId) { - releaseFeishuMessageProcessing(syntheticMessageId, accountId); - } } }); }; diff --git a/extensions/feishu/src/monitor.comment.test.ts b/extensions/feishu/src/monitor.comment.test.ts index 4af189add8c..48a20027139 100644 --- a/extensions/feishu/src/monitor.comment.test.ts +++ b/extensions/feishu/src/monitor.comment.test.ts @@ -822,13 +822,20 @@ describe("resolveDriveCommentEventTurn", () => { }); describe("drive.notice.comment_add_v1 monitor handler", () => { + let processingClaim: dedup.FeishuMessageProcessingClaim; beforeEach(() => { lastRuntime = createNonExitingRuntimeEnv(); handleFeishuCommentEventMock.mockClear(); createFeishuClientMock.mockReset().mockReturnValue(makeOpenApiClient({}) as never); - vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockResolvedValue("claimed"); - vi.spyOn(dedup, "recordProcessedFeishuMessage").mockResolvedValue(true); - vi.spyOn(dedup, "releaseFeishuMessageProcessing").mockImplementation(() => {}); + processingClaim = { + keys: ["drive-comment:test"], + commit: vi.fn(async () => true), + release: vi.fn(), + }; + vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockResolvedValue({ + kind: "claimed", + handle: processingClaim, + }); }); afterEach(() => { @@ -910,7 +917,7 @@ describe("drive.notice.comment_add_v1 monitor handler", () => { }); it("drops duplicate comment events before dispatch", async () => { - vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockResolvedValue("duplicate"); + vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockResolvedValue({ kind: "duplicate" }); const onComment = await setupCommentMonitorHandler(); await onComment(makeDriveCommentEvent()); @@ -925,23 +932,12 @@ describe("drive.notice.comment_add_v1 monitor handler", () => { await onComment(makeDriveCommentEvent()); await vi.waitFor(() => { - expect(dedup.recordProcessedFeishuMessage).toHaveBeenCalledTimes(1); - expect(dedup.releaseFeishuMessageProcessing).toHaveBeenCalledWith( - "drive-comment:10d9d60b990db39f96a4c2fd357fb877", - "default", - ); + expect(processingClaim.commit).toHaveBeenCalledTimes(1); + expect(processingClaim.release).not.toHaveBeenCalled(); expect(lastRuntime?.error).toHaveBeenCalledWith( "feishu[default]: error handling drive comment notice: Error: post-send failure", ); }); - const [recordedMessageId, recordedNamespace, recordedLogger] = mockCallAt( - dedup.recordProcessedFeishuMessage as ReturnType, - 0, - "Feishu processed-message record", - ); - expect(recordedMessageId).toBe("drive-comment:10d9d60b990db39f96a4c2fd357fb877"); - expect(recordedNamespace).toBe("default"); - expect(typeof recordedLogger).toBe("function"); }); it("releases comment replay without recording when failure is explicitly retryable", async () => { @@ -955,11 +951,10 @@ describe("drive.notice.comment_add_v1 monitor handler", () => { await onComment(makeDriveCommentEvent()); await vi.waitFor(() => { - expect(dedup.recordProcessedFeishuMessage).not.toHaveBeenCalled(); - expect(dedup.releaseFeishuMessageProcessing).toHaveBeenCalledWith( - "drive-comment:10d9d60b990db39f96a4c2fd357fb877", - "default", - ); + expect(processingClaim.commit).not.toHaveBeenCalled(); + expect(processingClaim.release).toHaveBeenCalledWith({ + error: expect.objectContaining({ message: "retry me" }), + }); expect(lastRuntime?.error).toHaveBeenCalledWith( "feishu[default]: error handling drive comment notice: FeishuRetryableSyntheticEventError: retry me", ); diff --git a/extensions/feishu/src/monitor.message-handler.test.ts b/extensions/feishu/src/monitor.message-handler.test.ts index b4f85763777..49de48d45ab 100644 --- a/extensions/feishu/src/monitor.message-handler.test.ts +++ b/extensions/feishu/src/monitor.message-handler.test.ts @@ -54,7 +54,6 @@ function createHandler() { handleMessage, resolveDebounceText: () => "hello", hasProcessedMessage: vi.fn(async () => false), - recordProcessedMessage: vi.fn(async () => true), getBotOpenId: () => "ou_bot", }); diff --git a/extensions/feishu/src/monitor.message-handler.ts b/extensions/feishu/src/monitor.message-handler.ts index 8b1bcb4ad2e..fab556c0dbd 100644 --- a/extensions/feishu/src/monitor.message-handler.ts +++ b/extensions/feishu/src/monitor.message-handler.ts @@ -1,7 +1,7 @@ // Feishu plugin module implements monitor.message handler behavior. import { isRecord, readStringValue as readString } from "openclaw/plugin-sdk/string-coerce-runtime"; import type { ClawdbotConfig, HistoryEntry, PluginRuntime, RuntimeEnv } from "../runtime-api.js"; -import { claimUnprocessedFeishuMessage, releaseFeishuMessageProcessing } from "./dedup.js"; +import { claimUnprocessedFeishuMessage, type FeishuMessageProcessingClaim } from "./dedup.js"; import { resolveFeishuMessageDedupeKey } from "./dedupe-key.js"; import type { FeishuMessageEvent } from "./event-types.js"; import { isMentionForwardRequest } from "./mention.js"; @@ -24,7 +24,7 @@ type FeishuMessageReceiveHandlerContext = { channelRuntime?: PluginRuntime["channel"]; chatHistories?: Map; accountId?: string; - processingClaimHeld?: boolean; + processingClaim?: FeishuMessageProcessingClaim; messageDedupeKey?: string; }) => Promise; resolveDebounceText: (params: { @@ -37,11 +37,6 @@ type FeishuMessageReceiveHandlerContext = { namespace: string, log?: (...args: unknown[]) => void, ) => Promise; - recordProcessedMessage: ( - messageId: string | undefined | null, - namespace: string, - log?: (...args: unknown[]) => void, - ) => Promise; getBotOpenId?: (accountId: string) => string | undefined; getBotName?: (accountId: string) => string | undefined; resolveSequentialKey?: (params: { @@ -110,13 +105,18 @@ function mergeFeishuDebounceMentions( return merged.size > 0 ? Array.from(merged.values()) : undefined; } +type FeishuMessageDebounceEntry = { + event: FeishuMessageEvent; + processingClaim?: FeishuMessageProcessingClaim; +}; + function dedupeFeishuDebounceEntriesByDedupeKey( - entries: FeishuMessageEvent[], -): FeishuMessageEvent[] { + entries: FeishuMessageDebounceEntry[], +): FeishuMessageDebounceEntry[] { const seen = new Set(); - const deduped: FeishuMessageEvent[] = []; + const deduped: FeishuMessageDebounceEntry[] = []; for (const entry of entries) { - const dedupeKey = resolveFeishuMessageDedupeKey(entry); + const dedupeKey = resolveFeishuMessageDedupeKey(entry.event); if (!dedupeKey) { deduped.push(entry); continue; @@ -167,7 +167,6 @@ export function createFeishuMessageReceiveHandler({ handleMessage, resolveDebounceText: resolveText, hasProcessedMessage, - recordProcessedMessage, getBotOpenId = () => undefined, getBotName = () => undefined, resolveSequentialKey = ({ accountId: accountIdLocal, event }) => @@ -188,7 +187,11 @@ export function createFeishuMessageReceiveHandler({ }, }); - const dispatchFeishuMessage = async (event: FeishuMessageEvent, messageDedupeKey?: string) => { + const dispatchFeishuMessage = async ( + event: FeishuMessageEvent, + messageDedupeKey?: string, + processingClaim?: FeishuMessageProcessingClaim, + ) => { const sequentialKey = resolveSequentialKey({ accountId, event, @@ -205,7 +208,7 @@ export function createFeishuMessageReceiveHandler({ channelRuntime, chatHistories, accountId, - processingClaimHeld: true, + processingClaim, messageDedupeKey, }); await enqueue(sequentialKey, task); @@ -226,99 +229,110 @@ export function createFeishuMessageReceiveHandler({ }; const recordSuppressedMessageIds = async ( - entries: FeishuMessageEvent[], + entries: FeishuMessageDebounceEntry[], dispatchDedupeKey?: string, ) => { const keepDedupeKey = dispatchDedupeKey?.trim(); const suppressedIds = new Set( entries - .map((entry) => resolveFeishuMessageDedupeKey(entry)) - .filter((id): id is string => Boolean(id) && (!keepDedupeKey || id !== keepDedupeKey)), + .map((entry) => ({ + id: resolveFeishuMessageDedupeKey(entry.event), + claim: entry.processingClaim, + })) + .filter(({ id }) => Boolean(id) && (!keepDedupeKey || id !== keepDedupeKey)), ); - for (const messageId of suppressedIds) { + for (const suppressed of suppressedIds) { try { - await recordProcessedMessage(messageId, accountId, log); + await suppressed.claim?.commit(); } catch (err) { error( - `feishu[${accountId}]: failed to record merged dedupe id ${messageId}: ${String(err)}`, + `feishu[${accountId}]: failed to record merged dedupe id ${suppressed.id}: ${String(err)}`, ); } } }; - const inboundDebouncer = channelRuntime.debounce.createInboundDebouncer({ - debounceMs: inboundDebounceMs, - buildKey: (event) => { - const chatId = event.message.chat_id?.trim(); - const senderId = resolveSenderDebounceId(event); - if (!chatId || !senderId) { - return null; - } - const rootId = event.message.root_id?.trim(); - const threadKey = rootId ? `thread:${rootId}` : "chat"; - return `feishu:${accountId}:${chatId}:${threadKey}:${senderId}`; - }, - shouldDebounce: (event) => { - if (event.message.message_type !== "text") { - return false; - } - const text = resolveDebounceText(event); - return Boolean(text) && !channelRuntime.commands.isControlCommandMessage(text, cfg); - }, - onFlush: async (entries) => { - const last = entries.at(-1); - if (!last) { - return; - } - if (entries.length === 1) { - await dispatchFeishuMessage(last, resolveFeishuMessageDedupeKey(last)); - return; - } - const dedupedEntries = dedupeFeishuDebounceEntriesByDedupeKey(entries); - const freshEntries: FeishuMessageEvent[] = []; - for (const entry of dedupedEntries) { - if (!(await hasProcessedMessage(resolveFeishuMessageDedupeKey(entry), accountId, log))) { - freshEntries.push(entry); + const inboundDebouncer = + channelRuntime.debounce.createInboundDebouncer({ + debounceMs: inboundDebounceMs, + buildKey: ({ event }) => { + const chatId = event.message.chat_id?.trim(); + const senderId = resolveSenderDebounceId(event); + if (!chatId || !senderId) { + return null; } - } - const dispatchEntry = freshEntries.at(-1); - if (!dispatchEntry) { - return; - } - const dispatchDedupeKey = resolveFeishuMessageDedupeKey(dispatchEntry); - await recordSuppressedMessageIds(dedupedEntries, dispatchDedupeKey); - const combinedText = freshEntries - .map((entry) => resolveDebounceText(entry)) - .filter(Boolean) - .join("\n"); - const mergedMentions = resolveFeishuDebounceMentions({ - entries: freshEntries, - botOpenId: getBotOpenId(accountId), - }); - await dispatchFeishuMessage( - { - ...dispatchEntry, - message: { - ...dispatchEntry.message, - ...(combinedText.trim() - ? { - message_type: "text", - content: JSON.stringify({ text: combinedText }), - } - : {}), - mentions: mergedMentions ?? dispatchEntry.message.mentions, + const rootId = event.message.root_id?.trim(); + const threadKey = rootId ? `thread:${rootId}` : "chat"; + return `feishu:${accountId}:${chatId}:${threadKey}:${senderId}`; + }, + shouldDebounce: ({ event }) => { + if (event.message.message_type !== "text") { + return false; + } + const text = resolveDebounceText(event); + return Boolean(text) && !channelRuntime.commands.isControlCommandMessage(text, cfg); + }, + onFlush: async (entries) => { + const last = entries.at(-1); + if (!last) { + return; + } + if (entries.length === 1) { + await dispatchFeishuMessage( + last.event, + resolveFeishuMessageDedupeKey(last.event), + last.processingClaim, + ); + return; + } + const dedupedEntries = dedupeFeishuDebounceEntriesByDedupeKey(entries); + const freshEntries: FeishuMessageDebounceEntry[] = []; + for (const entry of dedupedEntries) { + if ( + !(await hasProcessedMessage(resolveFeishuMessageDedupeKey(entry.event), accountId, log)) + ) { + freshEntries.push(entry); + } + } + const dispatchEntry = freshEntries.at(-1); + if (!dispatchEntry) { + return; + } + const dispatchDedupeKey = resolveFeishuMessageDedupeKey(dispatchEntry.event); + await recordSuppressedMessageIds(dedupedEntries, dispatchDedupeKey); + const combinedText = freshEntries + .map((entry) => resolveDebounceText(entry.event)) + .filter(Boolean) + .join("\n"); + const mergedMentions = resolveFeishuDebounceMentions({ + entries: freshEntries.map((entry) => entry.event), + botOpenId: getBotOpenId(accountId), + }); + await dispatchFeishuMessage( + { + ...dispatchEntry.event, + message: { + ...dispatchEntry.event.message, + ...(combinedText.trim() + ? { + message_type: "text", + content: JSON.stringify({ text: combinedText }), + } + : {}), + mentions: mergedMentions ?? dispatchEntry.event.message.mentions, + }, }, - }, - dispatchDedupeKey, - ); - }, - onError: (err, entries) => { - for (const entry of entries) { - releaseFeishuMessageProcessing(resolveFeishuMessageDedupeKey(entry), accountId); - } - error(`feishu[${accountId}]: inbound debounce flush failed: ${String(err)}`); - }, - }); + dispatchDedupeKey, + dispatchEntry.processingClaim, + ); + }, + onError: (err, entries) => { + for (const entry of entries) { + entry.processingClaim?.release({ error: err }); + } + error(`feishu[${accountId}]: inbound debounce flush failed: ${String(err)}`); + }, + }); return async (data) => { // Publish message recency before dedupe/debounce; transport liveness is @@ -348,16 +362,21 @@ export function createFeishuMessageReceiveHandler({ namespace: accountId, log, }); - if (claim !== "claimed") { - log(`feishu[${accountId}]: dropping ${claim} event for message ${messageId}`); + if (claim.kind === "duplicate" || claim.kind === "inflight") { + log(`feishu[${accountId}]: dropping ${claim.kind} event for message ${messageId}`); return; } const processMessage = async () => { - await inboundDebouncer.enqueue(event); + await inboundDebouncer.enqueue({ + event, + ...(claim.kind === "claimed" ? { processingClaim: claim.handle } : {}), + }); }; if (fireAndForget) { void processMessage().catch((err: unknown) => { - releaseFeishuMessageProcessing(messageDedupeKey, accountId); + if (claim.kind === "claimed") { + claim.handle.release({ error: err }); + } error(`feishu[${accountId}]: error handling message: ${String(err)}`); }); return; @@ -365,7 +384,9 @@ export function createFeishuMessageReceiveHandler({ try { await processMessage(); } catch (err) { - releaseFeishuMessageProcessing(messageDedupeKey, accountId); + if (claim.kind === "claimed") { + claim.handle.release({ error: err }); + } error(`feishu[${accountId}]: error handling message: ${String(err)}`); } }; diff --git a/extensions/feishu/src/monitor.reaction.test.ts b/extensions/feishu/src/monitor.reaction.test.ts index 9eb1d0f1731..c02fe40e939 100644 --- a/extensions/feishu/src/monitor.reaction.test.ts +++ b/extensions/feishu/src/monitor.reaction.test.ts @@ -233,8 +233,21 @@ function expectParsedFirstDispatchedEvent(botOpenId = "ou_bot") { }; } +function createClaimedFeishuDedupeResult() { + return { + kind: "claimed" as const, + handle: { + keys: ["test"] as const, + commit: async () => true, + release: () => undefined, + }, + }; +} + function setDedupPassThroughMocks(): void { - vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockResolvedValue("claimed"); + vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockResolvedValue( + createClaimedFeishuDedupeResult(), + ); vi.spyOn(dedup, "recordProcessedFeishuMessage").mockResolvedValue(true); vi.spyOn(dedup, "hasProcessedFeishuMessage").mockResolvedValue(false); } @@ -595,7 +608,9 @@ describe("Feishu inbound debounce regressions", () => { }); it("passes prefetched botName through to handleFeishuMessage", async () => { - vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockResolvedValue("claimed"); + vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockResolvedValue( + createClaimedFeishuDedupeResult(), + ); vi.spyOn(dedup, "recordProcessedFeishuMessage").mockResolvedValue(true); vi.spyOn(dedup, "hasProcessedFeishuMessage").mockResolvedValue(false); const onMessage = await setupDebounceMonitor({ botName: "OpenClaw Bot" }); @@ -678,7 +693,9 @@ describe("Feishu inbound debounce regressions", () => { }); it("excludes previously processed retries from combined debounce text", async () => { - vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockResolvedValue("claimed"); + vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockResolvedValue( + createClaimedFeishuDedupeResult(), + ); vi.spyOn(dedup, "recordProcessedFeishuMessage").mockResolvedValue(true); setStaleRetryMocks(); const onMessage = await setupDebounceMonitor(); @@ -704,8 +721,15 @@ describe("Feishu inbound debounce regressions", () => { }); it("uses latest fresh message id when debounce batch ends with stale retry", async () => { - vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockResolvedValue("claimed"); - const recordSpy = vi.spyOn(dedup, "recordProcessedFeishuMessage").mockResolvedValue(true); + const staleCommit = vi.fn(async () => true); + vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockImplementation(async ({ messageId }) => ({ + kind: "claimed", + handle: { + keys: [messageId ?? "test"], + commit: messageId === "om_old_latest_fresh" ? staleCommit : async () => true, + release: () => undefined, + }, + })); setStaleRetryMocks("om_old_latest_fresh"); const onMessage = await setupDebounceMonitor(); @@ -721,15 +745,7 @@ describe("Feishu inbound debounce regressions", () => { expect(dispatched.message.message_id).toBe("om_new_latest_fresh"); const combined = JSON.parse(dispatched.message.content) as { text?: string }; expect(combined.text).toBe("fresh"); - expect(recordSpy).toHaveBeenCalledTimes(1); - const [recordedMessageId, recordedNamespace, recordedLogger] = mockCallAt( - recordSpy, - 0, - "Feishu processed-message record", - ); - expect(recordedMessageId).toBe("om_old_latest_fresh"); - expect(recordedNamespace).toBe("default"); - expect(typeof recordedLogger).toBe("function"); + expect(staleCommit).toHaveBeenCalledTimes(1); }); it("releases early event dedupe when debounced dispatch fails", async () => { diff --git a/extensions/imessage/src/monitor/inbound-dedupe.test.ts b/extensions/imessage/src/monitor/inbound-dedupe.test.ts index b8f1cec69ad..efdf8c2c7ca 100644 --- a/extensions/imessage/src/monitor/inbound-dedupe.test.ts +++ b/extensions/imessage/src/monitor/inbound-dedupe.test.ts @@ -1,14 +1,9 @@ // Imessage tests cover inbound dedupe + stale-backlog age fence behavior. -import { beforeEach, describe, expect, it } from "vitest"; -import { installIMessageStateRuntimeForTest } from "../test-support/runtime.js"; +import { describe, expect, it } from "vitest"; import { buildIMessageInboundReplayKey, - claimIMessageInboundReplay, - commitIMessageInboundReplay, - createIMessageInboundReplayGuard, IMESSAGE_STALE_INBOUND_THRESHOLD_MS, isStaleIMessageBacklog, - releaseIMessageInboundReplay, } from "./inbound-dedupe.js"; import type { IMessagePayload } from "./types.js"; @@ -105,73 +100,3 @@ describe("isStaleIMessageBacklog", () => { expect(isStaleIMessageBacklog(payload({ created_at: "not-a-date" }), now)).toBe(false); }); }); - -describe("createIMessageInboundReplayGuard claim/commit/release", () => { - beforeEach(() => { - installIMessageStateRuntimeForTest(); - }); - - it("claims a key, and a committed key blocks a later claim as a duplicate", async () => { - const guard = createIMessageInboundReplayGuard(); - const message = payload({ guid: "GUID-DEDUPE" }); - const first = await claimIMessageInboundReplay({ guard, accountId: "default", message }); - expect(first.claimed).toBe(true); - expect(first.key).toBe("default:guid:GUID-DEDUPE"); - await commitIMessageInboundReplay({ - guard, - accountId: "default", - keys: first.key ? [first.key] : [], - }); - const second = await claimIMessageInboundReplay({ guard, accountId: "default", message }); - expect(second.claimed).toBe(false); - }); - - it("a released claim is reclaimable so a transient failure can retry", async () => { - const guard = createIMessageInboundReplayGuard(); - const message = payload({ guid: "GUID-RETRY" }); - const first = await claimIMessageInboundReplay({ guard, accountId: "default", message }); - expect(first.claimed).toBe(true); - releaseIMessageInboundReplay({ - guard, - accountId: "default", - keys: first.key ? [first.key] : [], - }); - const second = await claimIMessageInboundReplay({ guard, accountId: "default", message }); - expect(second.claimed).toBe(true); - }); - - it("a held (uncommitted) claim reports a concurrent duplicate as not claimed", async () => { - const guard = createIMessageInboundReplayGuard(); - const message = payload({ guid: "GUID-INFLIGHT" }); - const first = await claimIMessageInboundReplay({ guard, accountId: "default", message }); - expect(first.claimed).toBe(true); - // Second claim while the first is still in flight (not yet committed). - const second = await claimIMessageInboundReplay({ guard, accountId: "default", message }); - expect(second.claimed).toBe(false); - }); - - it("round-trips the composite claim key for a GUID-less row", async () => { - // Regression guard: the exact claimed key (composite, no GUID) must be the - // one committed, or a GUID-less coalesced row would leak an in-flight claim. - const guard = createIMessageInboundReplayGuard(); - const message = payload({ guid: undefined }); - const first = await claimIMessageInboundReplay({ guard, accountId: "default", message }); - expect(first.claimed).toBe(true); - expect(first.key).toBe(buildIMessageInboundReplayKey({ accountId: "default", message })); - await commitIMessageInboundReplay({ - guard, - accountId: "default", - keys: first.key ? [first.key] : [], - }); - const second = await claimIMessageInboundReplay({ guard, accountId: "default", message }); - expect(second.claimed).toBe(false); - }); - - it("fails open: an unidentifiable message claims with no key", async () => { - const guard = createIMessageInboundReplayGuard(); - const message = payload({ guid: undefined, sender: undefined }); - const res = await claimIMessageInboundReplay({ guard, accountId: "default", message }); - expect(res.claimed).toBe(true); - expect(res.key).toBeNull(); - }); -}); diff --git a/extensions/imessage/src/monitor/inbound-dedupe.ts b/extensions/imessage/src/monitor/inbound-dedupe.ts index b93477099de..a91b88c0216 100644 --- a/extensions/imessage/src/monitor/inbound-dedupe.ts +++ b/extensions/imessage/src/monitor/inbound-dedupe.ts @@ -11,7 +11,7 @@ // ROWID but the original (old) send date, so it arrives on the live watch as // a "new" row. The age fence is what recognizes it as stale. import { createHash } from "node:crypto"; -import { createClaimableDedupe, type ClaimableDedupe } from "openclaw/plugin-sdk/persistent-dedupe"; +import { createChannelReplayGuard } from "openclaw/plugin-sdk/persistent-dedupe"; import type { IMessagePayload } from "./types.js"; const IMESSAGE_INBOUND_DEDUPE_PLUGIN_ID = "imessage"; @@ -47,59 +47,27 @@ export const IMESSAGE_RECOVERY_MAX_ROWS = 500; * post-restart re-emit; release on dispatch failure lets a transient failure * retry instead of being permanently suppressed. */ -export function createIMessageInboundReplayGuard(): ClaimableDedupe { - return createClaimableDedupe({ - pluginId: IMESSAGE_INBOUND_DEDUPE_PLUGIN_ID, - namespacePrefix: IMESSAGE_INBOUND_DEDUPE_NAMESPACE_PREFIX, - ttlMs: IMESSAGE_INBOUND_DEDUPE_TTL_MS, - memoryMaxSize: IMESSAGE_INBOUND_DEDUPE_MEMORY_MAX, - stateMaxEntries: IMESSAGE_INBOUND_DEDUPE_STATE_MAX_ENTRIES, +type IMessageInboundReplayEvent = + | { accountId: string; message: IMessagePayload } + | { accountId: string; keys: readonly string[] }; + +export function createIMessageInboundReplayGuard() { + return createChannelReplayGuard({ + dedupe: { + pluginId: IMESSAGE_INBOUND_DEDUPE_PLUGIN_ID, + namespacePrefix: IMESSAGE_INBOUND_DEDUPE_NAMESPACE_PREFIX, + ttlMs: IMESSAGE_INBOUND_DEDUPE_TTL_MS, + memoryMaxSize: IMESSAGE_INBOUND_DEDUPE_MEMORY_MAX, + stateMaxEntries: IMESSAGE_INBOUND_DEDUPE_STATE_MAX_ENTRIES, + }, + buildReplayKey: (event) => + "message" in event + ? buildIMessageInboundReplayKey({ accountId: event.accountId, message: event.message }) + : event.keys, + namespace: (event) => event.accountId, }); } -/** - * Claim a message before handling. Returns the key to commit/release later, and - * `claimed=false` when a recent copy already owns the key (duplicate/inflight) - * so the caller drops it. A message with no derivable key fails open (claimed, - * key=null) so it is always handled and nothing to commit. - */ -export async function claimIMessageInboundReplay(params: { - guard: ClaimableDedupe; - accountId: string; - message: IMessagePayload; -}): Promise<{ claimed: boolean; key: string | null }> { - const key = buildIMessageInboundReplayKey({ - accountId: params.accountId, - message: params.message, - }); - if (!key) { - return { claimed: true, key: null }; - } - const claim = await params.guard.claim(key, { namespace: params.accountId }); - return { claimed: claim.kind === "claimed", key }; -} - -export async function commitIMessageInboundReplay(params: { - guard: ClaimableDedupe; - accountId: string; - keys: readonly string[]; -}): Promise { - for (const key of new Set(params.keys)) { - await params.guard.commit(key, { namespace: params.accountId }); - } -} - -export function releaseIMessageInboundReplay(params: { - guard: ClaimableDedupe; - accountId: string; - keys: readonly string[]; - error?: unknown; -}): void { - for (const key of new Set(params.keys)) { - params.guard.release(key, { namespace: params.accountId, error: params.error }); - } -} - /** * Stable replay key for an inbound message. Prefers the Apple GUID (globally * unique, survives chat.db rowid churn). Falls back to a composite of the diff --git a/extensions/imessage/src/monitor/monitor-provider.ts b/extensions/imessage/src/monitor/monitor-provider.ts index 1708800c176..0eb93d4fa26 100644 --- a/extensions/imessage/src/monitor/monitor-provider.ts +++ b/extensions/imessage/src/monitor/monitor-provider.ts @@ -27,6 +27,7 @@ import { recordInboundSession } from "openclaw/plugin-sdk/conversation-runtime"; import { expectDefined } from "openclaw/plugin-sdk/expect-runtime"; import { normalizeScpRemoteHost } from "openclaw/plugin-sdk/host-runtime"; import { isInboundPathAllowed, kindFromMime } from "openclaw/plugin-sdk/media-runtime"; +import type { ChannelReplayClaimHandle } from "openclaw/plugin-sdk/persistent-dedupe"; import { DEFAULT_GROUP_HISTORY_LIMIT, type HistoryEntry } from "openclaw/plugin-sdk/reply-history"; import { resolveTextChunkLimit, type GetReplyOptions } from "openclaw/plugin-sdk/reply-runtime"; import { dispatchInboundMessage } from "openclaw/plugin-sdk/reply-runtime"; @@ -86,14 +87,11 @@ import { } from "./group-allowlist-warnings.js"; import { buildIMessageInboundReplayKey, - claimIMessageInboundReplay, - commitIMessageInboundReplay, createIMessageInboundReplayGuard, IMESSAGE_RECOVERY_MAX_AGE_MS, IMESSAGE_RECOVERY_MAX_ROWS, IMESSAGE_STALE_INBOUND_THRESHOLD_MS, isStaleIMessageBacklog, - releaseIMessageInboundReplay, } from "./inbound-dedupe.js"; import { buildDirectIMessageReplyTarget, @@ -685,11 +683,9 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P const { debouncer: inboundDebouncer } = createChannelInboundDebouncer<{ message: IMessagePayload; - // Exact replay-guard key claimed for this row at ingestion (GUID or, for a - // GUID-less row, the composite fallback). Carried through so flush commits - // or releases the same key it claimed, even after a debounce merge rewrites - // the payload identity. null when the row had no derivable key (fail open). - replayKey: string | null; + // The ingestion claim owns the exact GUID/composite key even when debounce + // later rewrites the payload identity. Missing handles fail open. + replayClaim?: ChannelReplayClaimHandle; }>({ cfg, channel: "imessage", @@ -746,28 +742,21 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P // dispatch throws so a transient failure can retry on a later re-emit. Per // unit so a failure in one bucket entry cannot strand another's claim. const dispatchUnit = async ( - unitEntries: { message: IMessagePayload; replayKey: string | null }[], + unitEntries: { message: IMessagePayload; replayClaim?: ChannelReplayClaimHandle }[], message: IMessagePayload, ) => { - const keys = unitEntries - .map((entry) => entry.replayKey) - .filter((key): key is string => key !== null); + const replayClaims = unitEntries + .map((entry) => entry.replayClaim) + .filter((claim): claim is ChannelReplayClaimHandle => claim !== undefined); try { await handleMessageNow(message); - await commitIMessageInboundReplay({ - guard: inboundReplayGuard, - accountId: accountInfo.accountId, - keys, - }); + await Promise.all(replayClaims.map((claim) => claim.commit())); advanceRecoveryCursorAfterHandled(unitEntries); } catch (err) { holdRecoveryCursorBeforeFailedRows(unitEntries); - releaseIMessageInboundReplay({ - guard: inboundReplayGuard, - accountId: accountInfo.accountId, - keys, - error: err, - }); + for (const claim of replayClaims) { + claim.release({ error: err }); + } runtime.error?.(`imessage: inbound dispatch failed: ${String(err)}`); } }; @@ -791,7 +780,10 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P // Standalone URL preview rows merge with the immediately preceding row; // already-complete URL messages flush any pending ordinary row first. if (messages.some(hasIMessageUrlBalloonBundleID)) { - let pending: { message: IMessagePayload; replayKey: string | null } | null = null; + let pending: { + message: IMessagePayload; + replayClaim?: ChannelReplayClaimHandle; + } | null = null; for (const entry of entries) { if (isStandaloneIMessageUrlPreviewPayload(entry.message) && pending) { const unitEntries = [pending, entry]; @@ -1587,8 +1579,7 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P message, }); if (suppressedKey) { - await commitIMessageInboundReplay({ - guard: inboundReplayGuard, + await inboundReplayGuard.shouldProcess({ accountId: accountInfo.accountId, keys: [suppressedKey], }); @@ -1606,19 +1597,21 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P // transient dispatch failure (see handleMessageNow) so a failed message can // still retry on a later re-emit. Claimed only once we will actually enqueue // so a dropped row never leaks an uncommitted claim. - const replay = await claimIMessageInboundReplay({ - guard: inboundReplayGuard, + const replay = await inboundReplayGuard.claim({ accountId: accountInfo.accountId, message: repairedMessage, }); - if (!replay.claimed) { + if (replay.kind === "duplicate" || replay.kind === "inflight") { logVerbose( `imessage: dropping duplicate inbound notification account=${accountInfo.accountId}`, ); return; } trackPendingRecoveryReplayRow(repairedMessage); - await inboundDebouncer.enqueue({ message: repairedMessage, replayKey: replay.key }); + await inboundDebouncer.enqueue({ + message: repairedMessage, + ...(replay.kind === "claimed" ? { replayClaim: replay.handle } : {}), + }); }; await waitForTransportReady({ diff --git a/extensions/line/src/bot-handlers.ts b/extensions/line/src/bot-handlers.ts index 7917d5e854c..172f6bb38ca 100644 --- a/extensions/line/src/bot-handlers.ts +++ b/extensions/line/src/bot-handlers.ts @@ -10,7 +10,7 @@ import { resolvePairingIdLabel, upsertChannelPairingRequest, } from "openclaw/plugin-sdk/conversation-runtime"; -import { createClaimableDedupe, type ClaimableDedupe } from "openclaw/plugin-sdk/persistent-dedupe"; +import { createChannelReplayGuard } from "openclaw/plugin-sdk/persistent-dedupe"; import { DEFAULT_GROUP_HISTORY_LIMIT, createChannelHistoryWindow, @@ -80,19 +80,25 @@ interface LineHandlerContext { const LINE_WEBHOOK_REPLAY_WINDOW_MS = 10 * 60 * 1000; const LINE_WEBHOOK_REPLAY_MAX_ENTRIES = 4096; -type LineWebhookReplayCache = ClaimableDedupe; function normalizeLineIngressEntry(value: string): string | null { return normalizeLineAllowEntry(value) || null; } -export function createLineWebhookReplayCache(): LineWebhookReplayCache { - return createClaimableDedupe({ - ttlMs: LINE_WEBHOOK_REPLAY_WINDOW_MS, - memoryMaxSize: LINE_WEBHOOK_REPLAY_MAX_ENTRIES, +type LineReplayEvent = { event: WebhookEvent; accountId: string }; + +export function createLineWebhookReplayCache() { + return createChannelReplayGuard({ + dedupe: { + ttlMs: LINE_WEBHOOK_REPLAY_WINDOW_MS, + memoryMaxSize: LINE_WEBHOOK_REPLAY_MAX_ENTRIES, + }, + buildReplayKey: ({ event, accountId }) => buildLineWebhookReplayKey(event, accountId)?.key, }); } +type LineWebhookReplayCache = ReturnType; + function buildLineWebhookReplayKey( event: WebhookEvent, accountId: string, @@ -125,39 +131,6 @@ function buildLineWebhookReplayKey( return { key: `${accountId}|${event.type}|${sourceId}|${eventId}`, eventId: `event:${eventId}` }; } -type LineReplayCandidate = { - key: string; - eventId: string; - cache: LineWebhookReplayCache; -}; - -function getLineReplayCandidate( - event: WebhookEvent, - context: LineHandlerContext, -): LineReplayCandidate | null { - const replay = buildLineWebhookReplayKey(event, context.account.accountId); - const cache = context.replayCache; - if (!replay || !cache) { - return null; - } - return { key: replay.key, eventId: replay.eventId, cache }; -} - -async function claimLineReplayEvent( - candidate: LineReplayCandidate, -): Promise<{ skip: true; inFlightResult?: Promise } | { skip: false }> { - const claim = await candidate.cache.claim(candidate.key); - if (claim.kind === "claimed") { - return { skip: false }; - } - if (claim.kind === "inflight") { - logVerbose(`line: skipped in-flight replayed webhook event ${candidate.eventId}`); - return { skip: true, inFlightResult: claim.pending.then(() => undefined) }; - } - logVerbose(`line: skipped replayed webhook event ${candidate.eventId}`); - return { skip: true }; -} - function resolveLineGroupConfig(params: { config: ResolvedLineAccount["config"]; groupId?: string; @@ -573,49 +546,30 @@ export async function handleLineWebhookEvents( ): Promise { let firstError: unknown; for (const event of events) { - const replayCandidate = getLineReplayCandidate(event, context); - const replaySkip = replayCandidate ? await claimLineReplayEvent(replayCandidate) : null; - if (replaySkip?.skip) { - if (replaySkip.inFlightResult) { + try { + if (!context.replayCache) { + await handleLineWebhookEvent(event, context); + continue; + } + const replayEvent = { event, accountId: context.account.accountId }; + const result = await context.replayCache.processGuarded( + replayEvent, + async () => await handleLineWebhookEvent(event, context), + { onError: "commit" }, + ); + const replayId = buildLineWebhookReplayKey(event, context.account.accountId)?.eventId; + if (result.kind === "inflight") { + logVerbose(`line: skipped in-flight replayed webhook event ${replayId ?? "unknown"}`); try { - await replaySkip.inFlightResult; + await result.pending; } catch (err) { context.runtime.error?.(danger(`line: replayed in-flight event failed: ${String(err)}`)); firstError ??= err; } - } - continue; - } - try { - switch (event.type) { - case "message": - await handleMessageEvent(event, context); - break; - case "follow": - await handleFollowEvent(event, context); - break; - case "unfollow": - await handleUnfollowEvent(event, context); - break; - case "join": - await handleJoinEvent(event, context); - break; - case "leave": - await handleLeaveEvent(event, context); - break; - case "postback": - await handlePostbackEvent(event, context); - break; - default: - logVerbose(`line: unhandled event type: ${(event as WebhookEvent).type}`); - } - if (replayCandidate) { - await replayCandidate.cache.commit(replayCandidate.key); + } else if (result.kind === "duplicate") { + logVerbose(`line: skipped replayed webhook event ${replayId ?? "unknown"}`); } } catch (err) { - if (replayCandidate) { - await replayCandidate.cache.commit(replayCandidate.key); - } context.runtime.error?.(danger(`line: event handler failed: ${String(err)}`)); firstError ??= err; } @@ -625,6 +579,34 @@ export async function handleLineWebhookEvents( } } +async function handleLineWebhookEvent( + event: WebhookEvent, + context: LineHandlerContext, +): Promise { + switch (event.type) { + case "message": + await handleMessageEvent(event, context); + break; + case "follow": + await handleFollowEvent(event, context); + break; + case "unfollow": + await handleUnfollowEvent(event, context); + break; + case "join": + await handleJoinEvent(event, context); + break; + case "leave": + await handleLeaveEvent(event, context); + break; + case "postback": + await handlePostbackEvent(event, context); + break; + default: + logVerbose(`line: unhandled event type: ${(event as WebhookEvent).type}`); + } +} + function toLintErrorObject(value: unknown, fallbackMessage: string): Error { if (value instanceof Error) { return value; diff --git a/extensions/matrix/doctor-contract-api.test.ts b/extensions/matrix/doctor-contract-api.test.ts index df7e95ce3e4..3950a5fc93c 100644 --- a/extensions/matrix/doctor-contract-api.test.ts +++ b/extensions/matrix/doctor-contract-api.test.ts @@ -470,15 +470,21 @@ describe("matrix doctor contract state migrations", () => { auth: { accountId: "ops" }, env: dedupeEnv, }); - await expect(opsDeduper.claimEvent({ roomId, eventId: "$committed" })).resolves.toBe(false); - await expect(opsDeduper.claimEvent({ roomId, eventId: "$expired" })).resolves.toBe(true); + await expect(opsDeduper.claim({ roomId, eventId: "$committed" })).resolves.toEqual({ + kind: "duplicate", + }); + const expiredClaim = await opsDeduper.claim({ roomId, eventId: "$expired" }); + expect(expiredClaim.kind).toBe("claimed"); + if (expiredClaim.kind === "claimed") { + expiredClaim.handle.release(); + } const homeDeduper = createMatrixInboundEventDeduper({ auth: { accountId: "home" }, env: dedupeEnv, }); - await expect(homeDeduper.claimEvent({ roomId, eventId: "$json-committed" })).resolves.toBe( - false, - ); + await expect(homeDeduper.claim({ roomId, eventId: "$json-committed" })).resolves.toEqual({ + kind: "duplicate", + }); // Legacy sources are retired and the migration is idempotent. await expect(legacyStore.entries()).resolves.toEqual([]); diff --git a/extensions/matrix/src/matrix/monitor/handler.test.ts b/extensions/matrix/src/matrix/monitor/handler.test.ts index b54420a40d7..40c444ed5f6 100644 --- a/extensions/matrix/src/matrix/monitor/handler.test.ts +++ b/extensions/matrix/src/matrix/monitor/handler.test.ts @@ -2558,9 +2558,7 @@ describe("matrix monitor handler live allowlist reload", () => { describe("matrix monitor handler durable inbound dedupe", () => { it("skips replayed inbound events before session recording", async () => { const inboundDeduper = { - claimEvent: vi.fn(async () => false), - commitEvent: vi.fn(async () => undefined), - releaseEvent: vi.fn(), + claim: vi.fn(async () => ({ kind: "duplicate" as const })), }; const { handler, recordInboundSession } = createMatrixHandlerTestHarness({ inboundDeduper, @@ -2578,27 +2576,29 @@ describe("matrix monitor handler durable inbound dedupe", () => { }), ); - expect(inboundDeduper.claimEvent).toHaveBeenCalledWith({ + expect(inboundDeduper.claim).toHaveBeenCalledWith({ roomId: "!room:example.org", eventId: "$dup", }); expect(recordInboundSession).not.toHaveBeenCalled(); - expect(inboundDeduper.commitEvent).not.toHaveBeenCalled(); - expect(inboundDeduper.releaseEvent).not.toHaveBeenCalled(); }); it("commits inbound events only after queued replies finish delivering", async () => { const callOrder: string[] = []; + const commit = vi.fn(async () => { + callOrder.push("commit"); + return true; + }); + const release = vi.fn(() => { + callOrder.push("release"); + }); const inboundDeduper = { - claimEvent: vi.fn(async () => { + claim: vi.fn(async () => { callOrder.push("claim"); - return true; - }), - commitEvent: vi.fn(async () => { - callOrder.push("commit"); - }), - releaseEvent: vi.fn(() => { - callOrder.push("release"); + return { + kind: "claimed" as const, + handle: { keys: ["test"] as const, commit, release }, + }; }), }; const recordInboundSession = vi.fn(async () => { @@ -2652,14 +2652,17 @@ describe("matrix monitor handler durable inbound dedupe", () => { "dispatch-idle", "commit", ]); - expect(inboundDeduper.releaseEvent).not.toHaveBeenCalled(); + expect(release).not.toHaveBeenCalled(); }); it("commits a claimed event when bot loop protection suppresses dispatch", async () => { + const commit = vi.fn(async () => true); + const release = vi.fn(); const inboundDeduper = { - claimEvent: vi.fn(async () => true), - commitEvent: vi.fn(async () => undefined), - releaseEvent: vi.fn(), + claim: vi.fn(async () => ({ + kind: "claimed" as const, + handle: { keys: ["test"] as const, commit, release }, + })), }; const runPrepared = vi.fn( async (turn: { ctxPayload: Record; routeSessionKey: string }) => ({ @@ -2690,18 +2693,18 @@ describe("matrix monitor handler durable inbound dedupe", () => { ); expect(recordInboundSession).not.toHaveBeenCalled(); - expect(inboundDeduper.commitEvent).toHaveBeenCalledWith({ - roomId: "!room:example.org", - eventId: "$bot-loop-drop", - }); - expect(inboundDeduper.releaseEvent).not.toHaveBeenCalled(); + expect(commit).toHaveBeenCalledOnce(); + expect(release).not.toHaveBeenCalled(); }); it("releases a claimed event when reply dispatch fails before completion", async () => { + const commit = vi.fn(async () => true); + const release = vi.fn(); const inboundDeduper = { - claimEvent: vi.fn(async () => true), - commitEvent: vi.fn(async () => undefined), - releaseEvent: vi.fn(), + claim: vi.fn(async () => ({ + kind: "claimed" as const, + handle: { keys: ["test"] as const, commit, release }, + })), }; const runtime = { error: vi.fn(), @@ -2726,19 +2729,19 @@ describe("matrix monitor handler durable inbound dedupe", () => { }), ); - expect(inboundDeduper.commitEvent).not.toHaveBeenCalled(); - expect(inboundDeduper.releaseEvent).toHaveBeenCalledWith({ - roomId: "!room:example.org", - eventId: "$release-on-error", - }); + expect(commit).not.toHaveBeenCalled(); + expect(release).toHaveBeenCalledOnce(); expectRuntimeErrorContaining(runtime.error, "matrix handler failed"); }); it("keeps replay committed when queued final delivery fails after a generic error", async () => { + const commit = vi.fn(async () => true); + const release = vi.fn(); const inboundDeduper = { - claimEvent: vi.fn(async () => true), - commitEvent: vi.fn(async () => undefined), - releaseEvent: vi.fn(), + claim: vi.fn(async () => ({ + kind: "claimed" as const, + handle: { keys: ["test"] as const, commit, release }, + })), }; const runtime = { error: vi.fn(), @@ -2771,21 +2774,21 @@ describe("matrix monitor handler durable inbound dedupe", () => { }), ); - expect(inboundDeduper.commitEvent).toHaveBeenCalledWith({ - roomId: "!room:example.org", - eventId: "$release-on-final-delivery-error", - }); - expect(inboundDeduper.releaseEvent).not.toHaveBeenCalled(); + expect(commit).toHaveBeenCalledOnce(); + expect(release).not.toHaveBeenCalled(); expectRuntimeErrorContaining(runtime.error, "matrix final reply failed"); }); it.each(["tool", "block"] as const)( "keeps replay committed when queued %s delivery fails after a generic error and no final reply exists", async (kind) => { + const commit = vi.fn(async () => true); + const release = vi.fn(); const inboundDeduper = { - claimEvent: vi.fn(async () => true), - commitEvent: vi.fn(async () => undefined), - releaseEvent: vi.fn(), + claim: vi.fn(async () => ({ + kind: "claimed" as const, + handle: { keys: ["test"] as const, commit, release }, + })), }; const runtime = { error: vi.fn(), @@ -2822,27 +2825,28 @@ describe("matrix monitor handler durable inbound dedupe", () => { }), ); - expect(inboundDeduper.commitEvent).toHaveBeenCalledWith({ - roomId: "!room:example.org", - eventId: `$release-on-${kind}-delivery-error`, - }); - expect(inboundDeduper.releaseEvent).not.toHaveBeenCalled(); + expect(commit).toHaveBeenCalledOnce(); + expect(release).not.toHaveBeenCalled(); expectRuntimeErrorContaining(runtime.error, `matrix ${kind} reply failed`); }, ); it("commits a claimed event when dispatch completes without a final reply", async () => { const callOrder: string[] = []; + const commit = vi.fn(async () => { + callOrder.push("commit"); + return true; + }); + const release = vi.fn(() => { + callOrder.push("release"); + }); const inboundDeduper = { - claimEvent: vi.fn(async () => { + claim: vi.fn(async () => { callOrder.push("claim"); - return true; - }), - commitEvent: vi.fn(async () => { - callOrder.push("commit"); - }), - releaseEvent: vi.fn(() => { - callOrder.push("release"); + return { + kind: "claimed" as const, + handle: { keys: ["test"] as const, commit, release }, + }; }), }; const { handler } = createMatrixHandlerTestHarness({ @@ -2868,7 +2872,7 @@ describe("matrix monitor handler durable inbound dedupe", () => { ); expect(callOrder).toEqual(["claim", "record", "dispatch", "commit"]); - expect(inboundDeduper.releaseEvent).not.toHaveBeenCalled(); + expect(release).not.toHaveBeenCalled(); }); }); diff --git a/extensions/matrix/src/matrix/monitor/handler.ts b/extensions/matrix/src/matrix/monitor/handler.ts index 091bb2f2a78..2984b205c10 100644 --- a/extensions/matrix/src/matrix/monitor/handler.ts +++ b/extensions/matrix/src/matrix/monitor/handler.ts @@ -212,7 +212,7 @@ type MatrixMonitorHandlerParams = { startupMs: number; startupGraceMs: number; dropPreStartupMessages: boolean; - inboundDeduper?: Pick; + inboundDeduper?: Pick; directTracker: { isDirectMessage: (params: { roomId: string; @@ -577,7 +577,9 @@ export function createMatrixRoomMessageHandler(params: MatrixMonitorHandlerParam return async (roomId: string, event: MatrixRawEvent) => { const eventId = typeof event.event_id === "string" ? event.event_id.trim() : ""; - let claimedInboundEvent = false; + let inboundReplayClaim: + | import("openclaw/plugin-sdk/persistent-dedupe").ChannelReplayClaimHandle + | undefined; let draftStreamRef: MatrixDraftStreamHandle | undefined; let draftConsumed = false; try { @@ -614,11 +616,11 @@ export function createMatrixRoomMessageHandler(params: MatrixMonitorHandlerParam const eventTs = event.origin_server_ts; const eventAge = event.unsigned?.age; const commitInboundEventIfClaimed = async () => { - if (!claimedInboundEvent || !inboundDeduper || !eventId) { + if (!inboundReplayClaim) { return; } - await inboundDeduper.commitEvent({ roomId, eventId }); - claimedInboundEvent = false; + await inboundReplayClaim.commit(); + inboundReplayClaim = undefined; }; const readIngressPrefix = async () => { const selfUserId = await client.getUserId(); @@ -664,8 +666,11 @@ export function createMatrixRoomMessageHandler(params: MatrixMonitorHandlerParam return undefined; } if (eventId && inboundDeduper) { - claimedInboundEvent = await inboundDeduper.claimEvent({ roomId, eventId }); - if (!claimedInboundEvent) { + const claim = await inboundDeduper.claim({ roomId, eventId }); + // Missing identifiers fail open; committed and in-flight events do not. + if (claim.kind === "claimed") { + inboundReplayClaim = claim.handle; + } else if (claim.kind !== "invalid") { logVerboseMessage(`matrix: skip duplicate inbound event room=${roomId} id=${eventId}`); return undefined; } @@ -2590,9 +2595,7 @@ export function createMatrixRoomMessageHandler(params: MatrixMonitorHandlerParam await redactMatrixDraftEvent(client, roomId, draftEventId); } } - if (claimedInboundEvent && inboundDeduper && eventId) { - inboundDeduper.releaseEvent({ roomId, eventId }); - } + inboundReplayClaim?.release(); } }; } diff --git a/extensions/matrix/src/matrix/monitor/inbound-dedupe.test.ts b/extensions/matrix/src/matrix/monitor/inbound-dedupe.test.ts index dde8123a45f..fed81369516 100644 --- a/extensions/matrix/src/matrix/monitor/inbound-dedupe.test.ts +++ b/extensions/matrix/src/matrix/monitor/inbound-dedupe.test.ts @@ -27,61 +27,43 @@ describe("Matrix inbound event dedupe", () => { const auth = { accountId: "ops" } as const; const event = { roomId: "!room:example.org", eventId: "$event-1" } as const; - it("drops a duplicate event after commit", async () => { - const deduper = createMatrixInboundEventDeduper({ auth, env: createStateEnv() }); - - await expect(deduper.claimEvent(event)).resolves.toBe(true); - await deduper.commitEvent(event); - await expect(deduper.claimEvent(event)).resolves.toBe(false); - }); - - it("reports an in-flight claim as a duplicate", async () => { - const deduper = createMatrixInboundEventDeduper({ auth, env: createStateEnv() }); - - await expect(deduper.claimEvent(event)).resolves.toBe(true); - await expect(deduper.claimEvent(event)).resolves.toBe(false); - }); - it("persists committed events across restarts", async () => { const env = createStateEnv(); const first = createMatrixInboundEventDeduper({ auth, env }); - await expect(first.claimEvent(event)).resolves.toBe(true); - await first.commitEvent(event); + const firstClaim = await first.claim(event); + expect(firstClaim.kind).toBe("claimed"); + if (firstClaim.kind === "claimed") { + await firstClaim.handle.commit(); + } // A fresh instance has an empty memory layer, so the duplicate verdict // must come from the persisted plugin-state SQLite rows. const second = createMatrixInboundEventDeduper({ auth, env }); - await expect(second.claimEvent(event)).resolves.toBe(false); - }); - - it("lets a released claim retry, including after a restart", async () => { - const env = createStateEnv(); - const first = createMatrixInboundEventDeduper({ auth, env }); - await expect(first.claimEvent(event)).resolves.toBe(true); - first.releaseEvent(event); - await expect(first.claimEvent(event)).resolves.toBe(true); - first.releaseEvent(event); - - const second = createMatrixInboundEventDeduper({ auth, env }); - await expect(second.claimEvent(event)).resolves.toBe(true); + await expect(second.claim(event)).resolves.toEqual({ kind: "duplicate" }); }); it("scopes dedupe state per account", async () => { const env = createStateEnv(); const ops = createMatrixInboundEventDeduper({ auth: { accountId: "ops" }, env }); - await expect(ops.claimEvent(event)).resolves.toBe(true); - await ops.commitEvent(event); + const opsClaim = await ops.claim(event); + expect(opsClaim.kind).toBe("claimed"); + if (opsClaim.kind === "claimed") { + await opsClaim.handle.commit(); + } const home = createMatrixInboundEventDeduper({ auth: { accountId: "home" }, env }); - await expect(home.claimEvent(event)).resolves.toBe(true); + await expect(home.claim(event)).resolves.toMatchObject({ kind: "claimed" }); }); it("fails open for events without usable identifiers", async () => { const deduper = createMatrixInboundEventDeduper({ auth, env: createStateEnv() }); - await expect(deduper.claimEvent({ roomId: " ", eventId: "$x" })).resolves.toBe(true); - await expect(deduper.claimEvent({ roomId: " ", eventId: "$x" })).resolves.toBe(true); - await expect(deduper.commitEvent({ roomId: "!r:x", eventId: "" })).resolves.toBeUndefined(); + await expect(deduper.claim({ roomId: " ", eventId: "$x" })).resolves.toEqual({ + kind: "invalid", + }); + await expect(deduper.claim({ roomId: " ", eventId: "$x" })).resolves.toEqual({ + kind: "invalid", + }); }); it("keeps committed events in memory when plugin-state persistence fails", async () => { @@ -95,9 +77,13 @@ describe("Matrix inbound event dedupe", () => { env: { ...process.env, OPENCLAW_STATE_DIR: path.join(filePath, "nested") }, }); - await expect(deduper.claimEvent(event)).resolves.toBe(true); - await expect(deduper.commitEvent(event)).resolves.toBeUndefined(); - await expect(deduper.claimEvent(event)).resolves.toBe(false); + const claim = await deduper.claim(event); + expect(claim.kind).toBe("claimed"); + if (claim.kind !== "claimed") { + throw new Error(`expected claimed result, received ${claim.kind}`); + } + await expect(claim.handle.commit()).resolves.toBe(true); + await expect(deduper.claim(event)).resolves.toEqual({ kind: "duplicate" }); expect(warnSpy).toHaveBeenCalledWith( "MatrixInboundDedupe", "Matrix inbound dedupe persistence failed:", diff --git a/extensions/matrix/src/matrix/monitor/inbound-dedupe.ts b/extensions/matrix/src/matrix/monitor/inbound-dedupe.ts index 43c446928e0..e5befe6b619 100644 --- a/extensions/matrix/src/matrix/monitor/inbound-dedupe.ts +++ b/extensions/matrix/src/matrix/monitor/inbound-dedupe.ts @@ -3,7 +3,7 @@ // (account, room, event) is claimed before handling and committed only after // reply dispatch succeeds; release on retryable failure reopens the event. import { - createClaimableDedupe, + createChannelReplayGuard, resolvePersistentDedupePluginStateNamespace, } from "openclaw/plugin-sdk/persistent-dedupe"; import type { MatrixAuth } from "../client/types.js"; @@ -22,15 +22,6 @@ export const MATRIX_INBOUND_DEDUPE_TTL_MS = 30 * 24 * 60 * 60 * 1000; const MATRIX_INBOUND_DEDUPE_MEMORY_MAX = 5_000; export const MATRIX_INBOUND_DEDUPE_STATE_MAX_ENTRIES = 20_000; -export type MatrixInboundEventDeduper = { - /** True when the caller now owns the event; false for committed or in-flight duplicates. */ - claimEvent: (params: { roomId: string; eventId: string }) => Promise; - /** Records a handled event so restart/replay cannot dispatch it again. */ - commitEvent: (params: { roomId: string; eventId: string }) => Promise; - /** Drops an uncommitted claim so a failed dispatch can retry the event. */ - releaseEvent: (params: { roomId: string; eventId: string }) => void; -}; - function resolveMatrixInboundDedupeAccountId(accountId: string): string { return accountId.trim() || "default"; } @@ -61,43 +52,25 @@ export function resolveMatrixInboundDedupeStateNamespace(): string { export function createMatrixInboundEventDeduper(params: { auth: Pick; env?: NodeJS.ProcessEnv; -}): MatrixInboundEventDeduper { - const guard = createClaimableDedupe({ - pluginId: MATRIX_INBOUND_DEDUPE_PLUGIN_ID, - namespacePrefix: MATRIX_INBOUND_DEDUPE_NAMESPACE_PREFIX, - ttlMs: MATRIX_INBOUND_DEDUPE_TTL_MS, - memoryMaxSize: MATRIX_INBOUND_DEDUPE_MEMORY_MAX, - stateMaxEntries: MATRIX_INBOUND_DEDUPE_STATE_MAX_ENTRIES, - ...(params.env ? { env: params.env } : {}), - // Persistence is best effort: a broken state DB must never block inbound - // handling, so disk errors log and the memory layer keeps deduping. - onDiskError: (err) => { - LogService.warn("MatrixInboundDedupe", "Matrix inbound dedupe persistence failed:", err); - }, - }); +}) { const accountId = params.auth.accountId; - const namespace = MATRIX_INBOUND_DEDUPE_NAMESPACE; - return { - claimEvent: async (ids) => { - const key = buildMatrixInboundDedupeEventKey({ accountId, ...ids }); - if (!key) { - // Fail open: never suppress an event we cannot identify. - return true; - } - return (await guard.claim(key, { namespace })).kind === "claimed"; + return createChannelReplayGuard<{ roomId: string; eventId: string }>({ + dedupe: { + pluginId: MATRIX_INBOUND_DEDUPE_PLUGIN_ID, + namespacePrefix: MATRIX_INBOUND_DEDUPE_NAMESPACE_PREFIX, + ttlMs: MATRIX_INBOUND_DEDUPE_TTL_MS, + memoryMaxSize: MATRIX_INBOUND_DEDUPE_MEMORY_MAX, + stateMaxEntries: MATRIX_INBOUND_DEDUPE_STATE_MAX_ENTRIES, + ...(params.env ? { env: params.env } : {}), + // Persistence is best effort: a broken state DB must never block inbound + // handling, so disk errors log and the memory layer keeps deduping. + onDiskError: (err) => { + LogService.warn("MatrixInboundDedupe", "Matrix inbound dedupe persistence failed:", err); + }, }, - commitEvent: async (ids) => { - const key = buildMatrixInboundDedupeEventKey({ accountId, ...ids }); - if (!key) { - return; - } - await guard.commit(key, { namespace }); - }, - releaseEvent: (ids) => { - const key = buildMatrixInboundDedupeEventKey({ accountId, ...ids }); - if (key) { - guard.release(key, { namespace }); - } - }, - }; + buildReplayKey: (event) => buildMatrixInboundDedupeEventKey({ accountId, ...event }), + namespace: () => MATRIX_INBOUND_DEDUPE_NAMESPACE, + }); } + +export type MatrixInboundEventDeduper = ReturnType; diff --git a/extensions/matrix/src/matrix/monitor/index.test.ts b/extensions/matrix/src/matrix/monitor/index.test.ts index 08f375586e1..0e3da868bf7 100644 --- a/extensions/matrix/src/matrix/monitor/index.test.ts +++ b/extensions/matrix/src/matrix/monitor/index.test.ts @@ -49,10 +49,13 @@ const hoisted = vi.hoisted(() => { const accountConfig = { dm: {}, }; + const inboundReplayClaim = { + keys: ["test"] as const, + commit: vi.fn(async () => true), + release: vi.fn(), + }; const inboundDeduper = { - claimEvent: vi.fn(async () => true), - commitEvent: vi.fn(async () => undefined), - releaseEvent: vi.fn(), + claim: vi.fn(async () => ({ kind: "claimed" as const, handle: inboundReplayClaim })), }; const createMatrixInboundEventDeduper = vi.fn(() => inboundDeduper); const client = Object.assign(createEmitter(), { @@ -118,6 +121,7 @@ const hoisted = vi.hoisted(() => { getMemberDisplayName, getRoomInfo, inboundDeduper, + inboundReplayClaim, logger, registeredOnRoomMessage: null as null | ((roomId: string, event: unknown) => Promise), releaseSharedClientInstance, @@ -471,9 +475,11 @@ describe("monitorMatrixProvider", () => { hoisted.client.hasPersistedSyncState.mockReset().mockReturnValue(false); hoisted.client.stopSyncWithoutPersist.mockReset(); hoisted.client.drainPendingDecryptions.mockReset().mockResolvedValue(undefined); - hoisted.inboundDeduper.claimEvent.mockReset().mockResolvedValue(true); - hoisted.inboundDeduper.commitEvent.mockReset().mockResolvedValue(undefined); - hoisted.inboundDeduper.releaseEvent.mockReset(); + hoisted.inboundDeduper.claim + .mockReset() + .mockResolvedValue({ kind: "claimed" as const, handle: hoisted.inboundReplayClaim }); + hoisted.inboundReplayClaim.commit.mockReset().mockResolvedValue(true); + hoisted.inboundReplayClaim.release.mockReset(); hoisted.createMatrixInboundEventDeduper.mockReset().mockReturnValue(hoisted.inboundDeduper); hoisted.backfillMatrixAuthDeviceIdAfterStartup.mockReset().mockResolvedValue(undefined); hoisted.runMatrixStartupMaintenance.mockReset().mockResolvedValue(undefined); diff --git a/extensions/mattermost/src/mattermost/monitor-replay.ts b/extensions/mattermost/src/mattermost/monitor-replay.ts index 909ae09ce06..130e6b02e9f 100644 --- a/extensions/mattermost/src/mattermost/monitor-replay.ts +++ b/extensions/mattermost/src/mattermost/monitor-replay.ts @@ -1,56 +1,41 @@ // Mattermost plugin module owns replay-guarded post processing. -import { createClaimableDedupe, type ClaimableDedupe } from "openclaw/plugin-sdk/persistent-dedupe"; -import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { createChannelReplayGuard } from "openclaw/plugin-sdk/persistent-dedupe"; const RECENT_MATTERMOST_MESSAGE_TTL_MS = 5 * 60_000; const RECENT_MATTERMOST_MESSAGE_MAX = 2000; -const recentInboundMessages = createClaimableDedupe({ - ttlMs: RECENT_MATTERMOST_MESSAGE_TTL_MS, - memoryMaxSize: RECENT_MATTERMOST_MESSAGE_MAX, -}); - function buildMattermostInboundReplayKeys(params: { accountId: string; messageIds: string[]; }): string[] { - return uniqueStrings(params.messageIds.map((id) => `${params.accountId}:${id.trim()}`)).filter( - (key) => !key.endsWith(":"), - ); + return params.messageIds.map((id) => (id.trim() ? `${params.accountId}:${id.trim()}` : "")); } +function createMattermostInboundReplayGuard() { + return createChannelReplayGuard<{ accountId: string; messageIds: string[] }>({ + dedupe: { + ttlMs: RECENT_MATTERMOST_MESSAGE_TTL_MS, + memoryMaxSize: RECENT_MATTERMOST_MESSAGE_MAX, + }, + buildReplayKey: buildMattermostInboundReplayKeys, + }); +} + +type MattermostInboundReplayGuard = ReturnType; +const recentInboundMessages = createMattermostInboundReplayGuard(); + export async function processMattermostReplayGuardedPost(params: { accountId: string; messageIds: string[]; handlePost: () => Promise; - replayGuard?: ClaimableDedupe; + replayGuard?: MattermostInboundReplayGuard; }): Promise<"processed" | "duplicate"> { const replayGuard = params.replayGuard ?? recentInboundMessages; - const replayKeys = buildMattermostInboundReplayKeys({ + const event = { accountId: params.accountId, messageIds: params.messageIds, + }; + const result = await replayGuard.processGuarded(event, params.handlePost, { + onError: "commit", }); - if (replayKeys.length === 0) { - await params.handlePost(); - return "processed"; - } - - const claimedKeys: string[] = []; - for (const replayKey of replayKeys) { - const claim = await replayGuard.claim(replayKey); - if (claim.kind === "claimed") { - claimedKeys.push(replayKey); - } - } - if (claimedKeys.length === 0) { - return "duplicate"; - } - - try { - await params.handlePost(); - await Promise.all(claimedKeys.map((replayKey) => replayGuard.commit(replayKey))); - return "processed"; - } catch (error) { - await Promise.all(claimedKeys.map((replayKey) => replayGuard.commit(replayKey))); - throw error; - } + return result.kind === "processed" ? "processed" : "duplicate"; } diff --git a/extensions/mattermost/src/mattermost/monitor.test.ts b/extensions/mattermost/src/mattermost/monitor.test.ts index 8d22cf5ee69..8e9ca88f5be 100644 --- a/extensions/mattermost/src/mattermost/monitor.test.ts +++ b/extensions/mattermost/src/mattermost/monitor.test.ts @@ -1,5 +1,4 @@ // Mattermost tests cover monitor plugin behavior. -import { createClaimableDedupe } from "openclaw/plugin-sdk/persistent-dedupe"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../runtime-api.js"; import { resolveMattermostAccount } from "./accounts.js"; @@ -18,7 +17,6 @@ import { } from "./monitor-context.js"; import { deliverMattermostReplyWithDraftPreview } from "./monitor-draft-delivery.js"; import { evaluateMattermostMentionGate } from "./monitor-gating.js"; -import { processMattermostReplayGuardedPost } from "./monitor-replay.js"; type MattermostMentionGateInput = Parameters[0]; type MattermostRequireMentionResolverInput = Parameters< @@ -957,67 +955,6 @@ describe("resolveMattermostPendingHistoryKey", () => { }); }); -describe("processMattermostReplayGuardedPost", () => { - it("skips duplicate message batches after a successful commit", async () => { - const replayGuard = createClaimableDedupe({ - ttlMs: 10_000, - memoryMaxSize: 100, - }); - const handlePost = vi.fn(async () => undefined); - - await expect( - processMattermostReplayGuardedPost({ - replayGuard, - accountId: "acct", - messageIds: ["post-1"], - handlePost, - }), - ).resolves.toBe("processed"); - await expect( - processMattermostReplayGuardedPost({ - replayGuard, - accountId: "acct", - messageIds: ["post-1"], - handlePost, - }), - ).resolves.toBe("duplicate"); - - expect(handlePost).toHaveBeenCalledTimes(1); - }); - - it("keeps replay committed after a non-retryable failure", async () => { - const replayGuard = createClaimableDedupe({ - ttlMs: 10_000, - memoryMaxSize: 100, - }); - const visibleSideEffect = vi.fn(); - const handlePost = vi.fn(async () => { - visibleSideEffect(); - throw new Error("post-send failure"); - }); - - await expect( - processMattermostReplayGuardedPost({ - replayGuard, - accountId: "acct", - messageIds: ["post-3"], - handlePost, - }), - ).rejects.toThrow("post-send failure"); - await expect( - processMattermostReplayGuardedPost({ - replayGuard, - accountId: "acct", - messageIds: ["post-3"], - handlePost, - }), - ).resolves.toBe("duplicate"); - - expect(handlePost).toHaveBeenCalledTimes(1); - expect(visibleSideEffect).toHaveBeenCalledTimes(1); - }); -}); - describe("buildMattermostModelPickerSelectMessageSid", () => { it("stays stable for the same picker selection", () => { expect( diff --git a/extensions/nextcloud-talk/src/core.test.ts b/extensions/nextcloud-talk/src/core.test.ts index 2a6029e9029..ae439fae377 100644 --- a/extensions/nextcloud-talk/src/core.test.ts +++ b/extensions/nextcloud-talk/src/core.test.ts @@ -250,24 +250,24 @@ describe("nextcloud talk core", () => { const stateDir = await makeTempDir(); const firstGuard = createNextcloudTalkReplayGuard({ stateDir }); - const firstAttempt = await firstGuard.shouldProcessMessage({ + const firstAttempt = await firstGuard.shouldProcess({ accountId: "account-a", roomToken: "room-1", messageId: "msg-1", }); - const replayAttempt = await firstGuard.shouldProcessMessage({ + const replayAttempt = await firstGuard.shouldProcess({ accountId: "account-a", roomToken: "room-1", messageId: "msg-1", }); const secondGuard = createNextcloudTalkReplayGuard({ stateDir }); - const restartReplayAttempt = await secondGuard.shouldProcessMessage({ + const restartReplayAttempt = await secondGuard.shouldProcess({ accountId: "account-a", roomToken: "room-1", messageId: "msg-1", }); - const otherAccountFirstAttempt = await secondGuard.shouldProcessMessage({ + const otherAccountFirstAttempt = await secondGuard.shouldProcess({ accountId: "account-b", roomToken: "room-1", messageId: "msg-1", @@ -279,38 +279,6 @@ describe("nextcloud talk core", () => { expect(otherAccountFirstAttempt).toBe(true); }); - it("releases in-flight replay claims when processing fails", async () => { - const guard = createNextcloudTalkReplayGuard({}); - - const firstClaim = await guard.claimMessage({ - accountId: "account-a", - roomToken: "room-1", - messageId: "msg-claim", - }); - const secondClaim = await guard.claimMessage({ - accountId: "account-a", - roomToken: "room-1", - messageId: "msg-claim", - }); - - expect(firstClaim).toBe("claimed"); - expect(secondClaim).toBe("inflight"); - - guard.releaseMessage({ - accountId: "account-a", - roomToken: "room-1", - messageId: "msg-claim", - error: new Error("transient"), - }); - - const retryClaim = await guard.claimMessage({ - accountId: "account-a", - roomToken: "room-1", - messageId: "msg-claim", - }); - expect(retryClaim).toBe("claimed"); - }); - it("resolves allowlist matches", () => { expect( resolveNextcloudTalkAllowlistMatch({ diff --git a/extensions/nextcloud-talk/src/doctor.test.ts b/extensions/nextcloud-talk/src/doctor.test.ts index ec90aea9fe4..f04be362c97 100644 --- a/extensions/nextcloud-talk/src/doctor.test.ts +++ b/extensions/nextcloud-talk/src/doctor.test.ts @@ -128,7 +128,7 @@ describe("nextcloud-talk doctor", () => { const guard = createNextcloudTalkReplayGuard({ stateDir }); await expect( - guard.shouldProcessMessage({ + guard.shouldProcess({ accountId: "account-a", roomToken: "room-1", messageId: "msg-1", diff --git a/extensions/nextcloud-talk/src/monitor.ts b/extensions/nextcloud-talk/src/monitor.ts index ae5d4e7fc24..630ddf53d82 100644 --- a/extensions/nextcloud-talk/src/monitor.ts +++ b/extensions/nextcloud-talk/src/monitor.ts @@ -67,33 +67,19 @@ export async function processNextcloudTalkReplayGuardedMessage(params: { message: NextcloudTalkInboundMessage; handleMessage: () => Promise; }): Promise<"processed" | "duplicate"> { - const claim = await params.replayGuard.claimMessage({ - accountId: params.accountId, - roomToken: params.message.roomToken, - messageId: params.message.messageId, - }); - if (claim !== "claimed") { - return "duplicate"; - } - - try { - await params.handleMessage(); - await params.replayGuard.commitMessage({ + const result = await params.replayGuard.processGuarded( + { accountId: params.accountId, roomToken: params.message.roomToken, messageId: params.message.messageId, - }); - return "processed"; - } catch (error) { - // Failures are treated as non-retryable because the handler may already - // have produced a visible side effect, and replaying the webhook would duplicate it. - await params.replayGuard.commitMessage({ - accountId: params.accountId, - roomToken: params.message.roomToken, - messageId: params.message.messageId, - }); - throw error; - } + }, + params.handleMessage, + { + // The handler may have produced a visible side effect before failing. + onError: "commit", + }, + ); + return result.kind === "processed" ? "processed" : "duplicate"; } function formatError(err: unknown): string { diff --git a/extensions/nextcloud-talk/src/replay-guard.ts b/extensions/nextcloud-talk/src/replay-guard.ts index 288643cc13c..061af5ef8ac 100644 --- a/extensions/nextcloud-talk/src/replay-guard.ts +++ b/extensions/nextcloud-talk/src/replay-guard.ts @@ -1,5 +1,5 @@ // Nextcloud Talk plugin module implements replay guard behavior. -import { createClaimableDedupe } from "openclaw/plugin-sdk/persistent-dedupe"; +import { createChannelReplayGuard } from "openclaw/plugin-sdk/persistent-dedupe"; export const NEXTCLOUD_TALK_PLUGIN_ID = "nextcloud-talk"; export const NEXTCLOUD_TALK_REPLAY_DEDUPE_NAMESPACE_PREFIX = "replay-dedupe"; @@ -7,7 +7,10 @@ const DEFAULT_REPLAY_TTL_MS = 24 * 60 * 60 * 1000; const DEFAULT_MEMORY_MAX_SIZE = 1_000; const DEFAULT_STATE_MAX_ENTRIES = 10_000; -function buildReplayKey(params: { roomToken: string; messageId: string }): string | null { +function buildNextcloudTalkReplayKey(params: { + roomToken: string; + messageId: string; +}): string | null { const roomToken = params.roomToken.trim(); const messageId = params.messageId.trim(); if (!roomToken || !messageId) { @@ -26,99 +29,33 @@ type NextcloudTalkReplayGuardOptions = { onDiskError?: (error: unknown) => void; }; -export type NextcloudTalkReplayGuard = { - claimMessage: (params: { - accountId: string; - roomToken: string; - messageId: string; - }) => Promise<"claimed" | "duplicate" | "inflight" | "invalid">; - commitMessage: (params: { - accountId: string; - roomToken: string; - messageId: string; - }) => Promise; - releaseMessage: (params: { - accountId: string; - roomToken: string; - messageId: string; - error?: unknown; - }) => void; - shouldProcessMessage: (params: { - accountId: string; - roomToken: string; - messageId: string; - }) => Promise; +type NextcloudTalkReplayEvent = { + accountId: string; + roomToken: string; + messageId: string; }; -export function createNextcloudTalkReplayGuard( - options: NextcloudTalkReplayGuardOptions, -): NextcloudTalkReplayGuard { +export function createNextcloudTalkReplayGuard(options: NextcloudTalkReplayGuardOptions) { const stateDir = options.stateDir?.trim(); const baseOptions = { ttlMs: options.ttlMs ?? DEFAULT_REPLAY_TTL_MS, memoryMaxSize: options.memoryMaxSize ?? DEFAULT_MEMORY_MAX_SIZE, }; - const dedupe = createClaimableDedupe( - stateDir + return createChannelReplayGuard({ + dedupe: stateDir ? { ...baseOptions, pluginId: NEXTCLOUD_TALK_PLUGIN_ID, namespacePrefix: NEXTCLOUD_TALK_REPLAY_DEDUPE_NAMESPACE_PREFIX, stateMaxEntries: options.stateMaxEntries ?? options.fileMaxEntries ?? DEFAULT_STATE_MAX_ENTRIES, - env: { - ...process.env, - OPENCLAW_STATE_DIR: stateDir, - }, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, onDiskError: options.onDiskError, } : baseOptions, - ); - - return { - claimMessage: async ({ accountId, roomToken, messageId }) => { - const replayKey = buildReplayKey({ roomToken, messageId }); - if (!replayKey) { - return "invalid"; - } - const result = await dedupe.claim(replayKey, { - namespace: accountId, - }); - return result.kind; - }, - commitMessage: async ({ accountId, roomToken, messageId }) => { - const replayKey = buildReplayKey({ roomToken, messageId }); - if (!replayKey) { - return true; - } - return await dedupe.commit(replayKey, { - namespace: accountId, - }); - }, - releaseMessage: ({ accountId, roomToken, messageId, error }) => { - const replayKey = buildReplayKey({ roomToken, messageId }); - if (!replayKey) { - return; - } - dedupe.release(replayKey, { - namespace: accountId, - error, - }); - }, - shouldProcessMessage: async ({ accountId, roomToken, messageId }) => { - const replayKey = buildReplayKey({ roomToken, messageId }); - if (!replayKey) { - return true; - } - const result = await dedupe.claim(replayKey, { - namespace: accountId, - }); - if (result.kind !== "claimed") { - return false; - } - return await dedupe.commit(replayKey, { - namespace: accountId, - }); - }, - }; + buildReplayKey: buildNextcloudTalkReplayKey, + namespace: (event) => event.accountId, + }); } + +export type NextcloudTalkReplayGuard = ReturnType; diff --git a/extensions/raft/src/gateway.test.ts b/extensions/raft/src/gateway.test.ts index 56a9284935f..a8a9e3cc051 100644 --- a/extensions/raft/src/gateway.test.ts +++ b/extensions/raft/src/gateway.test.ts @@ -3,7 +3,7 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import type { ChannelGatewayContext } from "openclaw/plugin-sdk/channel-contract"; -import { createClaimableDedupe } from "openclaw/plugin-sdk/persistent-dedupe"; +import { createChannelReplayGuard } from "openclaw/plugin-sdk/persistent-dedupe"; import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { ResolvedRaftAccount } from "./accounts.js"; @@ -104,21 +104,26 @@ function createContext(accountId = "default") { ctx: ctx as unknown as ChannelGatewayContext, controller: new AbortController(), run, - wakeDedupe: createClaimableDedupe({ - ttlMs: 0, - memoryMaxSize: 10_000, + wakeDedupe: createChannelReplayGuard<{ accountId: string; key: string }>({ + dedupe: { ttlMs: 0, memoryMaxSize: 10_000 }, + buildReplayKey: (event) => event.key, + namespace: (event) => event.accountId, }), }; } function createPersistentWakeDedupe(stateDir: string) { - return createClaimableDedupe({ - ttlMs: 24 * 60 * 60 * 1000, - memoryMaxSize: 1_000, - pluginId: "raft", - namespacePrefix: "raft-wake-dedupe", - stateMaxEntries: 10_000, - env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + return createChannelReplayGuard<{ accountId: string; key: string }>({ + dedupe: { + ttlMs: 24 * 60 * 60 * 1000, + memoryMaxSize: 1_000, + pluginId: "raft", + namespacePrefix: "raft-wake-dedupe", + stateMaxEntries: 10_000, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }, + buildReplayKey: (event) => event.key, + namespace: (event) => event.accountId, }); } diff --git a/extensions/raft/src/gateway.ts b/extensions/raft/src/gateway.ts index c21cdfcf495..1bddcb9bbad 100644 --- a/extensions/raft/src/gateway.ts +++ b/extensions/raft/src/gateway.ts @@ -7,7 +7,7 @@ import type { Socket } from "node:net"; import type { ChannelGatewayContext } from "openclaw/plugin-sdk/channel-contract"; import { keepHttpServerTaskAlive, waitUntilAbort } from "openclaw/plugin-sdk/channel-outbound"; import { KeyedAsyncQueue } from "openclaw/plugin-sdk/keyed-async-queue"; -import { createClaimableDedupe, type ClaimableDedupe } from "openclaw/plugin-sdk/persistent-dedupe"; +import { createChannelReplayGuard } from "openclaw/plugin-sdk/persistent-dedupe"; import { safeEqualSecret } from "openclaw/plugin-sdk/security-runtime"; import { RAFT_CHANNEL_ID, type ResolvedRaftAccount } from "./accounts.js"; import { dispatchRaftWake } from "./inbound.js"; @@ -42,10 +42,33 @@ const WAKE_EVENT_ID_FIELDS = [ type RaftBridgeProcess = Pick & Pick; +type RaftWakeReplayEvent = { accountId: string; key: string }; + +function createRaftWakeReplayGuard(params?: { + env?: NodeJS.ProcessEnv; + onDiskError?: (error: unknown) => void; +}) { + return createChannelReplayGuard({ + dedupe: { + ttlMs: WAKE_DEDUPE_TTL_MS, + memoryMaxSize: WAKE_DEDUPE_MEMORY_MAX_SIZE, + pluginId: RAFT_CHANNEL_ID, + namespacePrefix: "raft-wake-dedupe", + stateMaxEntries: WAKE_DEDUPE_STATE_MAX_ENTRIES, + ...(params?.env ? { env: params.env } : {}), + ...(params?.onDiskError ? { onDiskError: params.onDiskError } : {}), + }, + buildReplayKey: (event) => event.key, + namespace: (event) => event.accountId, + }); +} + +type RaftWakeReplayGuard = ReturnType; + type RaftGatewayDeps = { createToken?: () => string; spawnBridge?: (params: { profile: string; endpoint: string; token: string }) => RaftBridgeProcess; - wakeDedupe?: ClaimableDedupe; + wakeDedupe?: RaftWakeReplayGuard; }; class WakeRequestError extends Error { @@ -224,12 +247,7 @@ export async function startRaftGatewayAccount( const wakeQueue = new KeyedAsyncQueue(); const wakeDedupe = deps.wakeDedupe ?? - createClaimableDedupe({ - ttlMs: WAKE_DEDUPE_TTL_MS, - memoryMaxSize: WAKE_DEDUPE_MEMORY_MAX_SIZE, - pluginId: RAFT_CHANNEL_ID, - namespacePrefix: "raft-wake-dedupe", - stateMaxEntries: WAKE_DEDUPE_STATE_MAX_ENTRIES, + createRaftWakeReplayGuard({ onDiskError: (error) => { ctx.log?.warn?.(`Raft wake dedupe storage failed: ${String(error)}`); }, @@ -292,23 +310,21 @@ export async function startRaftGatewayAccount( if (ctx.abortSignal?.aborted) { throw new WakeRequestError(503, "Raft Gateway is stopping."); } - const claim = await wakeDedupe.claim(dedupeKey, { namespace: ctx.accountId }); - if (claim.kind === "duplicate") { + const result = await wakeDedupe.processGuarded( + { accountId: ctx.accountId, key: dedupeKey }, + async () => { + await dispatchRaftWake({ ctx }); + }, + ); + if (result.kind === "duplicate") { return false; } - if (claim.kind === "inflight") { - if (await claim.pending) { + if (result.kind === "inflight") { + if (await result.pending) { return false; } throw new WakeRequestError(503, "Raft wake delivery is retrying."); } - try { - await dispatchRaftWake({ ctx }); - } catch (error) { - wakeDedupe.release(dedupeKey, { namespace: ctx.accountId, error }); - throw error; - } - await wakeDedupe.commit(dedupeKey, { namespace: ctx.accountId }); return true; }); sendJson(response, 202, { diff --git a/extensions/telegram/src/bot-handlers.inbound-debounce.runtime.ts b/extensions/telegram/src/bot-handlers.inbound-debounce.runtime.ts index d4804e83ead..25f69e70483 100644 --- a/extensions/telegram/src/bot-handlers.inbound-debounce.runtime.ts +++ b/extensions/telegram/src/bot-handlers.inbound-debounce.runtime.ts @@ -14,6 +14,7 @@ import type { RegisterTelegramHandlerParams } from "./bot-native-commands.js"; import type { TelegramSpooledReplayDeferredParticipant } from "./bot-processing-outcome.js"; import { getTelegramTextParts } from "./bot/helpers.js"; import type { TelegramContext } from "./bot/types.js"; +import type { TelegramMessageDispatchReplayClaim } from "./message-dispatch-dedupe.js"; type TelegramDebounceLane = "default" | "forward"; @@ -29,7 +30,7 @@ export type TelegramDebounceEntry = { threadId?: number; promptContextMinTimestampMs?: number; promptContextAmbientWatermark?: TelegramAmbientTranscriptWatermark; - dispatchDedupeKeys: string[]; + dispatchDedupeClaims: TelegramMessageDispatchReplayClaim[]; spooledReplayParticipant?: TelegramSpooledReplayDeferredParticipant; }; @@ -41,8 +42,8 @@ export function createTelegramInboundDebounceRuntime( promptContextBoundaryOptions, latestPromptContextMinTimestampMs, latestPromptContextAmbientWatermark, - mergeDispatchDedupeKeys, - releaseDispatchDedupeKeys, + mergeDispatchDedupeClaims, + releaseDispatchDedupeClaims, buildFailedProcessingResult, settleSpooledReplayParticipants, spooledReplayOptions, @@ -115,7 +116,7 @@ export function createTelegramInboundDebounceRuntime( ), ...spooledReplayOptions(participants), }, - dispatchDedupeKeys: last.dispatchDedupeKeys, + dispatchDedupeClaims: last.dispatchDedupeClaims, spooledReplayParticipants: participants, }); settleSpooledReplayParticipants(participants, result); @@ -127,8 +128,8 @@ export function createTelegramInboundDebounceRuntime( .join("\n"); const combinedMedia = entries.flatMap((entry) => entry.allMedia); if (!combinedText.trim() && combinedMedia.length === 0) { - releaseDispatchDedupeKeys( - mergeDispatchDedupeKeys(...entries.map((entry) => entry.dispatchDedupeKeys)), + releaseDispatchDedupeClaims( + mergeDispatchDedupeClaims(...entries.map((entry) => entry.dispatchDedupeClaims)), ); settleSpooledReplayParticipants(participants, { kind: "skipped" }); return; @@ -161,8 +162,8 @@ export function createTelegramInboundDebounceRuntime( ), ...spooledReplayOptions(participants), }, - dispatchDedupeKeys: mergeDispatchDedupeKeys( - ...entries.map((entry) => entry.dispatchDedupeKeys), + dispatchDedupeClaims: mergeDispatchDedupeClaims( + ...entries.map((entry) => entry.dispatchDedupeClaims), ), spooledReplayParticipants: participants, }); @@ -199,8 +200,8 @@ export function createTelegramInboundDebounceRuntime( } }, onCancel: (items) => { - releaseDispatchDedupeKeys( - mergeDispatchDedupeKeys(...items.map((item) => item.dispatchDedupeKeys)), + releaseDispatchDedupeClaims( + mergeDispatchDedupeClaims(...items.map((item) => item.dispatchDedupeClaims)), ); settleSpooledReplayParticipants( items diff --git a/extensions/telegram/src/bot-handlers.inbound-media-group.runtime.ts b/extensions/telegram/src/bot-handlers.inbound-media-group.runtime.ts index f522f8cd097..47173566bac 100644 --- a/extensions/telegram/src/bot-handlers.inbound-media-group.runtime.ts +++ b/extensions/telegram/src/bot-handlers.inbound-media-group.runtime.ts @@ -28,6 +28,7 @@ import { getTelegramTextParts, hasBotMention } from "./bot/helpers.js"; import type { TelegramContext } from "./bot/types.js"; import { isTelegramForumServiceMessage } from "./forum-service-message.js"; import { resolveTelegramCommandIngressAuthorization } from "./ingress.js"; +import type { TelegramMessageDispatchReplayClaim } from "./message-dispatch-dedupe.js"; type MediaAuthorization = { authorizationCfg: OpenClawConfig; @@ -49,7 +50,7 @@ type TelegramMediaGroupInput = MediaAuthorization & { storeAllowFrom: string[]; promptContextMinTimestampMs?: number; promptContextAmbientWatermark?: TelegramAmbientTranscriptWatermark; - dispatchDedupeKeys: string[]; + dispatchDedupeClaims: TelegramMessageDispatchReplayClaim[]; }; type BufferedMediaGroupEntry = MediaGroupEntry & @@ -88,8 +89,8 @@ export function createTelegramInboundMediaGroupRuntime( promptContextBoundaryOptions, latestPromptContextMinTimestampMs, latestPromptContextAmbientWatermark, - mergeDispatchDedupeKeys, - releaseDispatchDedupeKeys, + mergeDispatchDedupeClaims, + releaseDispatchDedupeClaims, buildFailedProcessingResult, settleSpooledReplayParticipants, createSpooledReplayParticipantForBufferedWork, @@ -217,12 +218,12 @@ export function createTelegramInboundMediaGroupRuntime( const primary = entry.messages.find((item) => item.msg.caption || item.msg.text) ?? entry.messages[0]; if (!primary) { - releaseDispatchDedupeKeys(entry.dispatchDedupeKeys); + releaseDispatchDedupeClaims(entry.dispatchDedupeClaims); settleSpooledReplayParticipants(entry.spooledReplayParticipants, { kind: "skipped" }); return; } if (await shouldSkipMediaDownloadForUnaddressedMentionGroup({ ...entry, ...primary })) { - releaseDispatchDedupeKeys(entry.dispatchDedupeKeys); + releaseDispatchDedupeClaims(entry.dispatchDedupeClaims); settleSpooledReplayParticipants(entry.spooledReplayParticipants, { kind: "skipped" }); return; } @@ -293,12 +294,12 @@ export function createTelegramInboundMediaGroupRuntime( ), ...spooledReplayOptions(entry.spooledReplayParticipants), }, - dispatchDedupeKeys: entry.dispatchDedupeKeys, + dispatchDedupeClaims: entry.dispatchDedupeClaims, spooledReplayParticipants: entry.spooledReplayParticipants, }); settleSpooledReplayParticipants(entry.spooledReplayParticipants, result); } catch (error) { - releaseDispatchDedupeKeys(entry.dispatchDedupeKeys, error); + releaseDispatchDedupeClaims(entry.dispatchDedupeClaims, error); settleSpooledReplayParticipants( entry.spooledReplayParticipants, buildFailedProcessingResult(error), @@ -336,9 +337,9 @@ export function createTelegramInboundMediaGroupRuntime( existing.promptContextAmbientWatermark, input.promptContextAmbientWatermark, ); - existing.dispatchDedupeKeys = mergeDispatchDedupeKeys( - existing.dispatchDedupeKeys, - input.dispatchDedupeKeys, + existing.dispatchDedupeClaims = mergeDispatchDedupeClaims( + existing.dispatchDedupeClaims, + input.dispatchDedupeClaims, ); existing.timer = setTimeout(() => { buffer.delete(key); diff --git a/extensions/telegram/src/bot-handlers.inbound-text.runtime.ts b/extensions/telegram/src/bot-handlers.inbound-text.runtime.ts index 6a5010374f9..64c0771726c 100644 --- a/extensions/telegram/src/bot-handlers.inbound-text.runtime.ts +++ b/extensions/telegram/src/bot-handlers.inbound-text.runtime.ts @@ -7,6 +7,7 @@ import type { TelegramAmbientTranscriptWatermark } from "./bot-message-context.t import type { RegisterTelegramHandlerParams } from "./bot-native-commands.js"; import type { TelegramSpooledReplayDeferredParticipant } from "./bot-processing-outcome.js"; import type { TelegramContext } from "./bot/types.js"; +import type { TelegramMessageDispatchReplayClaim } from "./message-dispatch-dedupe.js"; type TextFragmentEntry = { key: string; @@ -14,7 +15,7 @@ type TextFragmentEntry = { messages: Array<{ msg: Message; ctx: TelegramContext; receivedAtMs: number }>; promptContextMinTimestampMs?: number; promptContextAmbientWatermark?: TelegramAmbientTranscriptWatermark; - dispatchDedupeKeys: string[]; + dispatchDedupeClaims: TelegramMessageDispatchReplayClaim[]; spooledReplayParticipants: TelegramSpooledReplayDeferredParticipant[]; timer: ReturnType; }; @@ -30,7 +31,7 @@ type TelegramTextFragmentInput = { isAuthorizedAbortControlMessage: () => Promise; promptContextMinTimestampMs?: number; promptContextAmbientWatermark?: TelegramAmbientTranscriptWatermark; - dispatchDedupeKeys: string[]; + dispatchDedupeClaims: TelegramMessageDispatchReplayClaim[]; }; export function createTelegramInboundTextRuntime( @@ -41,8 +42,8 @@ export function createTelegramInboundTextRuntime( promptContextBoundaryOptions, latestPromptContextMinTimestampMs, latestPromptContextAmbientWatermark, - mergeDispatchDedupeKeys, - releaseDispatchDedupeKeys, + mergeDispatchDedupeClaims, + releaseDispatchDedupeClaims, buildFailedProcessingResult, settleSpooledReplayParticipants, createSpooledReplayParticipantForBufferedWork, @@ -66,13 +67,13 @@ export function createTelegramInboundTextRuntime( const first = entry.messages[0]; const last = entry.messages.at(-1); if (!first || !last) { - releaseDispatchDedupeKeys(entry.dispatchDedupeKeys); + releaseDispatchDedupeClaims(entry.dispatchDedupeClaims); settleSpooledReplayParticipants(entry.spooledReplayParticipants, { kind: "skipped" }); return; } const combinedText = entry.messages.map((message) => message.msg.text ?? "").join(""); if (!combinedText.trim()) { - releaseDispatchDedupeKeys(entry.dispatchDedupeKeys); + releaseDispatchDedupeClaims(entry.dispatchDedupeClaims); settleSpooledReplayParticipants(entry.spooledReplayParticipants, { kind: "skipped" }); return; } @@ -99,12 +100,12 @@ export function createTelegramInboundTextRuntime( ), ...spooledReplayOptions(entry.spooledReplayParticipants), }, - dispatchDedupeKeys: entry.dispatchDedupeKeys, + dispatchDedupeClaims: entry.dispatchDedupeClaims, spooledReplayParticipants: entry.spooledReplayParticipants, }); settleSpooledReplayParticipants(entry.spooledReplayParticipants, result); } catch (error) { - releaseDispatchDedupeKeys(entry.dispatchDedupeKeys, error); + releaseDispatchDedupeClaims(entry.dispatchDedupeClaims, error); settleSpooledReplayParticipants( entry.spooledReplayParticipants, buildFailedProcessingResult(error), @@ -159,9 +160,9 @@ export function createTelegramInboundTextRuntime( existing.promptContextAmbientWatermark, params.promptContextAmbientWatermark, ); - existing.dispatchDedupeKeys = mergeDispatchDedupeKeys( - existing.dispatchDedupeKeys, - params.dispatchDedupeKeys, + existing.dispatchDedupeClaims = mergeDispatchDedupeClaims( + existing.dispatchDedupeClaims, + params.dispatchDedupeClaims, ); scheduleFlush(existing); return true; @@ -178,7 +179,7 @@ export function createTelegramInboundTextRuntime( key, storeAllowFrom: params.storeAllowFrom, messages: [{ msg: params.msg, ctx: params.ctx, receivedAtMs: nowMs }], - dispatchDedupeKeys: params.dispatchDedupeKeys, + dispatchDedupeClaims: params.dispatchDedupeClaims, spooledReplayParticipants: participant ? [participant] : [], ...promptContextBoundaryOptions( params.promptContextMinTimestampMs, @@ -199,7 +200,7 @@ export function createTelegramInboundTextRuntime( if (existing) { clearTimeout(existing.timer); buffer.delete(key); - releaseDispatchDedupeKeys(existing.dispatchDedupeKeys); + releaseDispatchDedupeClaims(existing.dispatchDedupeClaims); settleSpooledReplayParticipants(existing.spooledReplayParticipants, { kind: "skipped" }); } } diff --git a/extensions/telegram/src/bot-handlers.inbound.runtime.ts b/extensions/telegram/src/bot-handlers.inbound.runtime.ts index c10142695dd..8ec68bc7fd2 100644 --- a/extensions/telegram/src/bot-handlers.inbound.runtime.ts +++ b/extensions/telegram/src/bot-handlers.inbound.runtime.ts @@ -36,6 +36,7 @@ import { resolveMedia } from "./bot/delivery.resolve-media.js"; import { getTelegramTextParts } from "./bot/helpers.js"; import type { TelegramContext } from "./bot/types.js"; import { resolveTelegramCommandIngressAuthorization } from "./ingress.js"; +import type { TelegramMessageDispatchReplayClaim } from "./message-dispatch-dedupe.js"; export function createTelegramHandlerInboundRuntime( { @@ -55,7 +56,7 @@ export function createTelegramHandlerInboundRuntime( const { mediaRuntimeWithAbort, promptContextBoundaryOptions, - releaseDispatchDedupeKeys, + releaseDispatchDedupeClaims, createSpooledReplayParticipantForBufferedWork, } = messageRuntime; const { @@ -105,7 +106,7 @@ export function createTelegramHandlerInboundRuntime( oversizeLogMessage: string; promptContextMinTimestampMs?: number; promptContextAmbientWatermark?: TelegramAmbientTranscriptWatermark; - dispatchDedupeKeys: string[]; + dispatchDedupeClaims: TelegramMessageDispatchReplayClaim[]; }) => { const { authorizationCfg, @@ -127,7 +128,7 @@ export function createTelegramHandlerInboundRuntime( oversizeLogMessage, promptContextMinTimestampMs, promptContextAmbientWatermark, - dispatchDedupeKeys, + dispatchDedupeClaims, } = params; const messageText = getTelegramTextParts(msg).text; @@ -170,7 +171,7 @@ export function createTelegramHandlerInboundRuntime( isAuthorizedAbortControlMessage, promptContextMinTimestampMs, promptContextAmbientWatermark, - dispatchDedupeKeys, + dispatchDedupeClaims, }) ) { return; @@ -195,7 +196,7 @@ export function createTelegramHandlerInboundRuntime( topicConfig, promptContextMinTimestampMs, promptContextAmbientWatermark, - dispatchDedupeKeys, + dispatchDedupeClaims, }) ) { return; @@ -218,7 +219,7 @@ export function createTelegramHandlerInboundRuntime( topicConfig, }) ) { - releaseDispatchDedupeKeys(dispatchDedupeKeys); + releaseDispatchDedupeClaims(dispatchDedupeClaims); return; } @@ -249,7 +250,7 @@ export function createTelegramHandlerInboundRuntime( }).catch(() => {}); } logger.warn({ chatId, error: String(mediaErr) }, oversizeLogMessage); - releaseDispatchDedupeKeys(dispatchDedupeKeys); + releaseDispatchDedupeClaims(dispatchDedupeClaims); return; } logger.warn({ chatId, error: String(mediaErr) }, "media fetch failed"); @@ -270,7 +271,7 @@ export function createTelegramHandlerInboundRuntime( }), }).catch(() => {}); } - releaseDispatchDedupeKeys(dispatchDedupeKeys, retryable ? mediaErr : undefined); + releaseDispatchDedupeClaims(dispatchDedupeClaims, retryable ? mediaErr : undefined); return; } @@ -279,7 +280,7 @@ export function createTelegramHandlerInboundRuntime( const hasText = Boolean(getTelegramTextParts(msg).text.trim()); if (msg.sticker && !media && !hasText) { logVerbose("telegram: skipping sticker-only message (unsupported sticker type)"); - releaseDispatchDedupeKeys(dispatchDedupeKeys); + releaseDispatchDedupeClaims(dispatchDedupeClaims); return; } @@ -327,7 +328,7 @@ export function createTelegramHandlerInboundRuntime( debounceLane, botUsername, ...promptContextBoundaryOptions(promptContextMinTimestampMs, promptContextAmbientWatermark), - dispatchDedupeKeys, + dispatchDedupeClaims, }; if ( debounceEntry.debounceKey && diff --git a/extensions/telegram/src/bot-handlers.message-events.runtime.ts b/extensions/telegram/src/bot-handlers.message-events.runtime.ts index 76b6a7a188c..33514f1e419 100644 --- a/extensions/telegram/src/bot-handlers.message-events.runtime.ts +++ b/extensions/telegram/src/bot-handlers.message-events.runtime.ts @@ -19,6 +19,7 @@ import { } from "./bot/helpers.js"; import { TelegramPairingStoreReadError } from "./bot/helpers.js"; import type { TelegramContext, TelegramGetChat } from "./bot/types.js"; +import type { TelegramMessageDispatchReplayClaim } from "./message-dispatch-dedupe.js"; export function registerTelegramMessageHandlers( { bot, opts, runtime, shouldSkipUpdate }: RegisterTelegramHandlerParams, @@ -29,7 +30,7 @@ export function registerTelegramMessageHandlers( const { normalizePromptContextMinTimestampMs, promptContextBoundaryOptions, - releaseDispatchDedupeKeys, + releaseDispatchDedupeClaims, claimMessageDispatchDedupe, buildSyntheticContext, resolveTelegramSessionState, @@ -122,7 +123,7 @@ export function registerTelegramMessageHandlers( }; const handleInboundMessageLike = async (event: InboundTelegramEvent) => { - let dispatchDedupeKeys: string[] = []; + let dispatchDedupeClaims: TelegramMessageDispatchReplayClaim[] = []; try { if (shouldSkipUpdate(event.ctxForDedupe)) { return; @@ -177,7 +178,7 @@ export function registerTelegramMessageHandlers( if (!dispatchDedupe.process) { return; } - dispatchDedupeKeys = dispatchDedupe.keys; + dispatchDedupeClaims = dispatchDedupe.claims; await recordMessageForReplyChain( event.msg, resolvedThreadId ?? dmThreadId, @@ -201,11 +202,11 @@ export function registerTelegramMessageHandlers( topicConfig, sendOversizeWarning: event.sendOversizeWarning, oversizeLogMessage: event.oversizeLogMessage, - dispatchDedupeKeys, + dispatchDedupeClaims, ...promptContextBoundaryOptions(promptContextMinTimestampMs, promptContextAmbientWatermark), }); } catch (err) { - releaseDispatchDedupeKeys(dispatchDedupeKeys, err); + releaseDispatchDedupeClaims(dispatchDedupeClaims, err); runtime.error?.(danger(`${event.errorMessage}: ${String(err)}`)); const spooledReplay = isTelegramSpooledReplayUpdate(event.ctx.update); if (err instanceof TelegramPairingStoreReadError || spooledReplay) { diff --git a/extensions/telegram/src/bot-handlers.message-lifecycle.runtime.ts b/extensions/telegram/src/bot-handlers.message-lifecycle.runtime.ts index c371e8c1759..3954f83e7cb 100644 --- a/extensions/telegram/src/bot-handlers.message-lifecycle.runtime.ts +++ b/extensions/telegram/src/bot-handlers.message-lifecycle.runtime.ts @@ -1,7 +1,6 @@ // Telegram dispatch dedupe, replay settlement, and synthetic-message helpers. import type { Message } from "grammy/types"; import { danger, logVerbose } from "openclaw/plugin-sdk/runtime-env"; -import { normalizeStringEntries } from "openclaw/plugin-sdk/string-coerce-runtime"; import type { TelegramAmbientTranscriptWatermark, TelegramMessageContextOptions, @@ -24,6 +23,7 @@ import { commitTelegramMessageDispatchReplay, createTelegramMessageDispatchReplayGuard, releaseTelegramMessageDispatchReplay, + type TelegramMessageDispatchReplayClaim, } from "./message-dispatch-dedupe.js"; export function createTelegramMessageLifecycleRuntime({ @@ -68,17 +68,20 @@ export function createTelegramMessageLifecycleRuntime({ ...watermarks: Array ): TelegramAmbientTranscriptWatermark | undefined => watermarks.findLast((watermark) => watermark !== undefined); - const mergeDispatchDedupeKeys = (...groups: Array) => [ - ...new Set(normalizeStringEntries(groups.flatMap((group) => group ?? []))), - ]; - const releaseDispatchDedupeKeys = (keys: readonly string[], error?: unknown) => { - releaseTelegramMessageDispatchReplay({ guard: replayGuard, keys, error }); + const mergeDispatchDedupeClaims = ( + ...groups: Array + ) => [...new Set(groups.flatMap((group) => group ?? []))]; + const releaseDispatchDedupeClaims = ( + claims: readonly TelegramMessageDispatchReplayClaim[], + error?: unknown, + ) => { + releaseTelegramMessageDispatchReplay({ claims, error }); }; - const commitDispatchDedupeKeys = async ( - keys: readonly string[], + const commitDispatchDedupeClaims = async ( + claims: readonly TelegramMessageDispatchReplayClaim[], options: { requirePersistent?: boolean } = {}, ) => { - await commitTelegramMessageDispatchReplay({ guard: replayGuard, keys, ...options }); + await commitTelegramMessageDispatchReplay({ guard: replayGuard, claims, ...options }); }; const buildFailedProcessingResult = (error: unknown): TelegramMessageProcessingResult => ({ kind: "failed-retryable", @@ -125,13 +128,15 @@ export function createTelegramMessageLifecycleRuntime({ participants.length > 0 ? { spooledReplay: true } : {}; const claimMessageDispatchDedupe = async ( msg: Message, - ): Promise<{ process: true; keys: string[] } | { process: false }> => { + ): Promise< + { process: true; claims: TelegramMessageDispatchReplayClaim[] } | { process: false } + > => { const claim = await claimTelegramMessageDispatchReplay({ guard: replayGuard, accountId, msg }); if (claim.kind === "duplicate") { logVerbose(`telegram dispatch dedupe: skipped message ${msg.chat.id}:${msg.message_id}`); return { process: false }; } - return { process: true, keys: claim.kind === "claimed" ? [claim.key] : [] }; + return { process: true, claims: claim.kind === "claimed" ? [claim.handle] : [] }; }; const buildSyntheticTextMessage = (params: { base: Message; @@ -171,9 +176,9 @@ export function createTelegramMessageLifecycleRuntime({ promptContextBoundaryOptions, latestPromptContextMinTimestampMs, latestPromptContextAmbientWatermark, - mergeDispatchDedupeKeys, - releaseDispatchDedupeKeys, - commitDispatchDedupeKeys, + mergeDispatchDedupeClaims, + releaseDispatchDedupeClaims, + commitDispatchDedupeClaims, buildFailedProcessingResult, settleSpooledReplayParticipants, beginSpooledReplaySettlementHolds, diff --git a/extensions/telegram/src/bot-handlers.message.runtime.ts b/extensions/telegram/src/bot-handlers.message.runtime.ts index 0e0be185a48..a52c7480bcf 100644 --- a/extensions/telegram/src/bot-handlers.message.runtime.ts +++ b/extensions/telegram/src/bot-handlers.message.runtime.ts @@ -29,6 +29,7 @@ import { resolveTelegramForumThreadId } from "./bot/helpers.js"; import type { TelegramContext } from "./bot/types.js"; import { resolveTelegramScopedGroupConfig } from "./group-config-helpers.js"; import type { TelegramCachedMessageNode, TelegramReplyChainEntry } from "./message-cache.js"; +import type { TelegramMessageDispatchReplayClaim } from "./message-dispatch-dedupe.js"; import { resolveTelegramPromptMediaPath } from "./prompt-media-path.js"; export function createTelegramHandlerMessageRuntime({ @@ -86,9 +87,9 @@ export function createTelegramHandlerMessageRuntime({ promptContextBoundaryOptions, latestPromptContextMinTimestampMs, latestPromptContextAmbientWatermark, - mergeDispatchDedupeKeys, - releaseDispatchDedupeKeys, - commitDispatchDedupeKeys, + mergeDispatchDedupeClaims, + releaseDispatchDedupeClaims, + commitDispatchDedupeClaims, buildFailedProcessingResult, settleSpooledReplayParticipants, beginSpooledReplaySettlementHolds, @@ -161,7 +162,7 @@ export function createTelegramHandlerMessageRuntime({ promptContextMessageSelection?: TelegramPromptContextMessageSelection; storeAllowFrom: string[]; options?: TelegramMessageContextOptions; - dispatchDedupeKeys?: string[]; + dispatchDedupeClaims?: TelegramMessageDispatchReplayClaim[]; spooledReplayParticipants?: readonly TelegramSpooledReplayDeferredParticipant[]; spooledReplayAbortSignal?: AbortSignal; }): Promise => { @@ -226,7 +227,7 @@ export function createTelegramHandlerMessageRuntime({ ingressSpooledReplayParticipants, ); try { - await commitDispatchDedupeKeys(params.dispatchDedupeKeys ?? [], { + await commitDispatchDedupeClaims(params.dispatchDedupeClaims ?? [], { requirePersistent: true, }); } catch (error) { @@ -236,8 +237,8 @@ export function createTelegramHandlerMessageRuntime({ releaseSettlementHolds("discard-pending"); dispatchDedupeCommitted = true; } else { - releaseDispatchDedupeKeys( - params.dispatchDedupeKeys ?? [], + releaseDispatchDedupeClaims( + params.dispatchDedupeClaims ?? [], result.kind === "failed-retryable" ? result.error : undefined, ); } @@ -361,7 +362,7 @@ export function createTelegramHandlerMessageRuntime({ cfg: runtimeCfg, telegramCfg: runtimeTelegramCfg, onDispatchStart: async () => { - await commitDispatchDedupeKeys(params.dispatchDedupeKeys ?? []); + await commitDispatchDedupeClaims(params.dispatchDedupeClaims ?? []); dispatchDedupeCommitted = true; }, spooledReplayAbortSignal: params.spooledReplayAbortSignal, @@ -382,9 +383,9 @@ export function createTelegramHandlerMessageRuntime({ return await finalizeSpooledReplayResult(result); } if (result.kind === "completed" && !dispatchDedupeCommitted) { - await commitDispatchDedupeKeys(params.dispatchDedupeKeys ?? []); + await commitDispatchDedupeClaims(params.dispatchDedupeClaims ?? []); } else if (result.kind !== "completed" && !dispatchDedupeCommitted) { - releaseDispatchDedupeKeys(params.dispatchDedupeKeys ?? []); + releaseDispatchDedupeClaims(params.dispatchDedupeClaims ?? []); } return result; } catch (err) { @@ -392,7 +393,7 @@ export function createTelegramHandlerMessageRuntime({ return await finalizeSpooledReplayResult(buildFailedProcessingResult(err)); } if (!dispatchDedupeCommitted) { - releaseDispatchDedupeKeys(params.dispatchDedupeKeys ?? [], err); + releaseDispatchDedupeClaims(params.dispatchDedupeClaims ?? [], err); } throw err; } @@ -404,8 +405,8 @@ export function createTelegramHandlerMessageRuntime({ promptContextBoundaryOptions, latestPromptContextMinTimestampMs, latestPromptContextAmbientWatermark, - mergeDispatchDedupeKeys, - releaseDispatchDedupeKeys, + mergeDispatchDedupeClaims, + releaseDispatchDedupeClaims, buildFailedProcessingResult, settleSpooledReplayParticipants, createSpooledReplayParticipantForBufferedWork, diff --git a/extensions/telegram/src/message-dispatch-dedupe.test.ts b/extensions/telegram/src/message-dispatch-dedupe.test.ts index 855c42d2674..6b9c9c26769 100644 --- a/extensions/telegram/src/message-dispatch-dedupe.test.ts +++ b/extensions/telegram/src/message-dispatch-dedupe.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import type { Message } from "grammy/types"; +import type { ChannelReplayClaimHandle } from "openclaw/plugin-sdk/persistent-dedupe"; import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { @@ -42,20 +43,34 @@ function storedReplayKey(accountId: string, msg: Message): string { function createTestReplayGuard( params: { - commit?: TelegramMessageDispatchReplayGuard["commit"]; - forget?: TelegramMessageDispatchReplayGuard["forget"]; - release?: TelegramMessageDispatchReplayGuard["release"]; + forget?: ( + key: string, + options?: Parameters[1], + ) => Promise; } = {}, ): TelegramMessageDispatchReplayGuard { + const eventKey = (event: Parameters[0]): string => + "keys" in event ? (event.keys?.[0] ?? "") : ""; return { - claim: async () => ({ kind: "claimed" }), - commit: params.commit ?? (async () => true), - forget: params.forget ?? (async () => true), - hasRecent: async () => false, + claim: async () => ({ kind: "invalid" }), + forget: async (event, options) => + await (params.forget ?? (async () => true))(eventKey(event), options), warmup: async () => 0, - clearMemory: () => {}, - memorySize: () => 0, - release: params.release ?? (() => {}), + }; +} + +function createTestClaim(params: { + key: string; + commit?: ( + key: string, + options?: Parameters[0], + ) => Promise; + release?: (key: string, options?: { error?: unknown }) => void; +}): ChannelReplayClaimHandle { + return { + keys: [params.key], + commit: async (options) => await (params.commit ?? (async () => true))(params.key, options), + release: (options) => (params.release ?? (() => {}))(params.key, options), }; } @@ -94,16 +109,13 @@ describe("Telegram message dispatch replay guard", () => { msg: message(), }); - expect(first).toEqual({ - kind: "claimed", - key: storedReplayKey("default", message()), - }); if (first.kind !== "claimed") { throw new Error("expected initial claim"); } + expect(first.handle.keys).toEqual([storedReplayKey("default", message())]); await commitTelegramMessageDispatchReplay({ guard: writer, - keys: [first.key], + claims: [first.handle], }); const reader = createTelegramMessageDispatchReplayGuard(); @@ -118,18 +130,28 @@ describe("Telegram message dispatch replay guard", () => { it("preserves concurrent commits", async () => { const writer = createTelegramMessageDispatchReplayGuard(); - const keys = Array.from({ length: 400 }, (_, index) => - storedReplayKey("default", message({ messageId: index + 1 })), + const claims = await Promise.all( + Array.from({ length: 400 }, async (_, index) => { + const claim = await claimTelegramMessageDispatchReplay({ + guard: writer, + accountId: "default", + msg: message({ messageId: index + 1 }), + }); + if (claim.kind !== "claimed") { + throw new Error(`expected claim ${index + 1}`); + } + return claim.handle; + }), ); await commitTelegramMessageDispatchReplay({ guard: writer, - keys, + claims, }); const reader = createTelegramMessageDispatchReplayGuard(); await expect(reader.warmup(TELEGRAM_MESSAGE_DISPATCH_DEDUPE_NAMESPACE)).resolves.toBe( - keys.length, + claims.length, ); }); @@ -138,23 +160,27 @@ describe("Telegram message dispatch replay guard", () => { const firstGate = createDeferred(); const secondGate = createDeferred(); const secondStarted = createDeferred(); - const guard = createTestReplayGuard({ - commit: async (key) => { - events.push(`start:${key}`); - if (key === "first") { - await firstGate.promise; - } else if (key === "second") { - secondStarted.resolve(); - await secondGate.promise; - } - events.push(`finish:${key}`); - return true; - }, - }); + const guard = createTestReplayGuard(); + const claims = ["first", "second", "third"].map((key) => + createTestClaim({ + key, + commit: async (keyLocal) => { + events.push(`start:${keyLocal}`); + if (keyLocal === "first") { + await firstGate.promise; + } else if (keyLocal === "second") { + secondStarted.resolve(); + await secondGate.promise; + } + events.push(`finish:${keyLocal}`); + return true; + }, + }), + ); const commit = commitTelegramMessageDispatchReplay({ guard, - keys: ["first", "second", "third"], + claims, }); expect(events).toEqual(["start:first"]); @@ -177,20 +203,24 @@ describe("Telegram message dispatch replay guard", () => { it("propagates per-key disk errors and stops the commit sequence", async () => { const diskError = new Error("dedupe disk write failed"); const commitCalls: string[] = []; - const guard = createTestReplayGuard({ - commit: async (key, options) => { - commitCalls.push(key); - if (key === "second") { - options?.onDiskError?.(diskError); - } - return true; - }, - }); + const guard = createTestReplayGuard(); + const claims = ["first", "second", "third"].map((key) => + createTestClaim({ + key, + commit: async (keyLocal, options) => { + commitCalls.push(keyLocal); + if (keyLocal === "second") { + options?.onDiskError?.(diskError); + } + return true; + }, + }), + ); await expect( commitTelegramMessageDispatchReplay({ guard, - keys: ["first", "second", "third"], + claims, requirePersistent: true, }), ).rejects.toBe(diskError); @@ -199,7 +229,9 @@ describe("Telegram message dispatch replay guard", () => { it("keeps live dispatch commits fail-open on dedupe disk errors", async () => { const diskError = new Error("dedupe disk write failed"); - const guard = createTestReplayGuard({ + const guard = createTestReplayGuard(); + const claim = createTestClaim({ + key: "live-message", commit: async (_key, options) => { options?.onDiskError?.(diskError); return true; @@ -209,7 +241,7 @@ describe("Telegram message dispatch replay guard", () => { await expect( commitTelegramMessageDispatchReplay({ guard, - keys: ["live-message"], + claims: [claim], }), ).resolves.toBeUndefined(); }); @@ -221,27 +253,32 @@ describe("Telegram message dispatch replay guard", () => { const forgetCalls: string[] = []; const releaseCalls: string[] = []; const guard = createTestReplayGuard({ - commit: async (key, options) => { - commitCalls.push(key); - committed.add(key); - if (key === "second") { - options?.onDiskError?.(diskError); - } - return true; - }, forget: async (key) => { forgetCalls.push(key); committed.delete(key); return true; }, - release: (key) => { - releaseCalls.push(key); - }, }); const keys = ["first", "second", "third"]; + const claims = keys.map((key) => + createTestClaim({ + key, + commit: async (keyLocal, options) => { + commitCalls.push(keyLocal); + committed.add(keyLocal); + if (keyLocal === "second") { + options?.onDiskError?.(diskError); + } + return true; + }, + release: (keyLocal) => { + releaseCalls.push(keyLocal); + }, + }), + ); await expect( - commitTelegramMessageDispatchReplay({ guard, keys, requirePersistent: true }), + commitTelegramMessageDispatchReplay({ guard, claims, requirePersistent: true }), ).rejects.toBe(diskError); expect(commitCalls).toEqual(["first", "second"]); @@ -268,7 +305,7 @@ describe("Telegram message dispatch replay guard", () => { await commitTelegramMessageDispatchReplay({ guard: writer, - keys: [first.key, second.key], + claims: [first.handle, second.handle], }); const reader = createTelegramMessageDispatchReplayGuard(); @@ -287,31 +324,28 @@ describe("Telegram message dispatch replay guard", () => { throw new Error("expected initial claim"); } - await expect( - claimTelegramMessageDispatchReplay({ - guard, - accountId: "work", - msg: message(), - }), - ).resolves.toEqual({ - kind: "claimed", - key: storedReplayKey("work", message()), + const work = await claimTelegramMessageDispatchReplay({ + guard, + accountId: "work", + msg: message(), }); + expect(work.kind).toBe("claimed"); + if (work.kind === "claimed") { + expect(work.handle.keys).toEqual([storedReplayKey("work", message())]); + } releaseTelegramMessageDispatchReplay({ + claims: [first.handle], + }); + const retry = await claimTelegramMessageDispatchReplay({ guard, - keys: [first.key], - }); - await expect( - claimTelegramMessageDispatchReplay({ - guard, - accountId: "default", - msg: message(), - }), - ).resolves.toEqual({ - kind: "claimed", - key: first.key, + accountId: "default", + msg: message(), }); + expect(retry.kind).toBe("claimed"); + if (retry.kind === "claimed") { + expect(retry.handle.keys).toEqual(first.handle.keys); + } }); it("lets an in-flight duplicate retry after the first claim is released", async () => { @@ -331,14 +365,14 @@ describe("Telegram message dispatch replay guard", () => { msg: message(), }); releaseTelegramMessageDispatchReplay({ - guard, - keys: [first.key], + claims: [first.handle], error: new Error("retry"), }); - await expect(duplicate).resolves.toEqual({ - kind: "claimed", - key: first.key, - }); + const retry = await duplicate; + expect(retry.kind).toBe("claimed"); + if (retry.kind === "claimed") { + expect(retry.handle.keys).toEqual(first.handle.keys); + } }); }); diff --git a/extensions/telegram/src/message-dispatch-dedupe.ts b/extensions/telegram/src/message-dispatch-dedupe.ts index d165da125ef..cfcb0892ff3 100644 --- a/extensions/telegram/src/message-dispatch-dedupe.ts +++ b/extensions/telegram/src/message-dispatch-dedupe.ts @@ -2,8 +2,10 @@ import path from "node:path"; import type { Message } from "grammy/types"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; -import { createClaimableDedupe, type ClaimableDedupe } from "openclaw/plugin-sdk/persistent-dedupe"; -import { normalizeStringEntries, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { + createChannelReplayGuard, + type ChannelReplayClaimHandle, +} from "openclaw/plugin-sdk/persistent-dedupe"; export const TELEGRAM_MESSAGE_DISPATCH_DEDUPE_TTL_MS = 7 * 24 * 60 * 60 * 1000; export const TELEGRAM_MESSAGE_DISPATCH_DEDUPE_NAMESPACE = "global"; @@ -12,14 +14,13 @@ export const TELEGRAM_MESSAGE_DISPATCH_DEDUPE_STATE_PLUGIN_ID = "telegram-messag const TELEGRAM_MESSAGE_DISPATCH_DEDUPE_MEMORY_MAX_ENTRIES = 50_000; export const TELEGRAM_MESSAGE_DISPATCH_DEDUPE_STATE_MAX_ENTRIES = 50_000; -type TelegramMessageDispatchReplayGuard = ClaimableDedupe & - Required>; - type TelegramMessageDispatchClaim = - | { kind: "claimed"; key: string } + | { kind: "claimed"; handle: ChannelReplayClaimHandle } | { kind: "duplicate" } | { kind: "invalid" }; +export type TelegramMessageDispatchReplayClaim = ChannelReplayClaimHandle; + type TelegramMessageDispatchReplayForgetFailure = { key: string; error?: unknown; @@ -92,44 +93,51 @@ function buildTelegramMessageDispatchStoredReplayKey(params: { : null; } +type TelegramMessageDispatchReplayEvent = + | { accountId: string; msg: Message } + | { keys?: readonly string[] }; + export function createTelegramMessageDispatchReplayGuard( params: { onDiskError?: (error: unknown) => void; } = {}, -): TelegramMessageDispatchReplayGuard { - return createClaimableDedupe({ - ttlMs: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_TTL_MS, - memoryMaxSize: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_MEMORY_MAX_ENTRIES, - pluginId: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_STATE_PLUGIN_ID, - namespacePrefix: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_NAMESPACE_PREFIX, - stateMaxEntries: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_STATE_MAX_ENTRIES, - ...(params.onDiskError ? { onDiskError: params.onDiskError } : {}), +) { + return createChannelReplayGuard({ + dedupe: { + ttlMs: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_TTL_MS, + memoryMaxSize: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_MEMORY_MAX_ENTRIES, + pluginId: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_STATE_PLUGIN_ID, + namespacePrefix: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_NAMESPACE_PREFIX, + stateMaxEntries: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_STATE_MAX_ENTRIES, + ...(params.onDiskError ? { onDiskError: params.onDiskError } : {}), + }, + buildReplayKey: (event) => + "msg" in event ? buildTelegramMessageDispatchStoredReplayKey(event) : (event.keys ?? []), + namespace: () => TELEGRAM_MESSAGE_DISPATCH_DEDUPE_NAMESPACE, }); } +type TelegramMessageDispatchReplayGuard = Pick< + ReturnType, + "claim" | "forget" | "warmup" +>; + export async function claimTelegramMessageDispatchReplay(params: { guard: TelegramMessageDispatchReplayGuard; accountId: string; msg: Message; }): Promise { - const key = buildTelegramMessageDispatchStoredReplayKey({ - accountId: params.accountId, - msg: params.msg, - }); - if (!key) { - return { kind: "invalid" }; - } - let releaseRetries = 0; while (true) { - const claim = await params.guard.claim(key, { - namespace: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_NAMESPACE, + const claim = await params.guard.claim({ + accountId: params.accountId, + msg: params.msg, }); if (claim.kind === "claimed") { - return { kind: "claimed", key }; + return { kind: "claimed", handle: claim.handle }; } - if (claim.kind === "duplicate") { - return { kind: "duplicate" }; + if (claim.kind === "duplicate" || claim.kind === "invalid") { + return claim; } try { await claim.pending; @@ -143,33 +151,27 @@ export async function claimTelegramMessageDispatchReplay(params: { } } -function normalizeReplayKeys(keys?: readonly string[]): string[] { - return uniqueStrings(normalizeStringEntries(keys ?? [])); -} - export async function commitTelegramMessageDispatchReplay(params: { guard: TelegramMessageDispatchReplayGuard; - keys?: readonly string[]; + claims?: readonly TelegramMessageDispatchReplayClaim[]; /** Require every claim to reach SQLite before the caller acknowledges durable adoption. */ requirePersistent?: boolean; }): Promise { - const keys = normalizeReplayKeys(params.keys); + const claims = [...new Set(params.claims ?? [])]; const committedKeys: string[] = []; // Commit serially so a later failure has no still-running sibling write that // can race rollback and recreate a key after it was forgotten. - for (const [index, key] of keys.entries()) { + for (const [index, claim] of claims.entries()) { let diskError: unknown; try { - const recorded = await params.guard.commit( - key, + const recorded = await claim.commit( params.requirePersistent === true ? { - namespace: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_NAMESPACE, onDiskError: (error) => { diskError = error; }, } - : { namespace: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_NAMESPACE }, + : undefined, ); if (params.requirePersistent === true && diskError !== undefined) { throw diskError instanceof Error @@ -177,22 +179,17 @@ export async function commitTelegramMessageDispatchReplay(params: { : new Error(formatErrorMessage(diskError), { cause: diskError }); } if (recorded) { - committedKeys.push(key); + committedKeys.push(...claim.keys); } } catch (error) { - for (const pendingKey of keys.slice(index + 1)) { - params.guard.release(pendingKey, { - namespace: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_NAMESPACE, - error, - }); + for (const pendingClaim of claims.slice(index + 1)) { + pendingClaim.release({ error }); } const failures: TelegramMessageDispatchReplayForgetFailure[] = []; for (const committedKey of committedKeys) { try { - const forgotten = await params.guard.forget(committedKey, { - namespace: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_NAMESPACE, - }); + const forgotten = await params.guard.forget({ keys: [committedKey] }); if (!forgotten) { failures.push({ key: committedKey }); } @@ -203,17 +200,19 @@ export async function commitTelegramMessageDispatchReplay(params: { let failedKeyCleanupError: unknown; try { - await params.guard.forget(key, { - namespace: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_NAMESPACE, - onDiskError: (rollbackError) => { - failedKeyCleanupError = rollbackError; + await params.guard.forget( + { keys: claim.keys }, + { + onDiskError: (rollbackError) => { + failedKeyCleanupError = rollbackError; + }, }, - }); + ); } catch (rollbackError) { failedKeyCleanupError = rollbackError; } if (failedKeyCleanupError !== undefined) { - failures.push({ key, error: failedKeyCleanupError }); + failures.push(...claim.keys.map((key) => ({ key, error: failedKeyCleanupError }))); } if (failures.length > 0) { throw new TelegramMessageDispatchReplayForgetError(failures); @@ -224,15 +223,10 @@ export async function commitTelegramMessageDispatchReplay(params: { } export function releaseTelegramMessageDispatchReplay(params: { - guard: TelegramMessageDispatchReplayGuard; - keys?: readonly string[]; + claims?: readonly TelegramMessageDispatchReplayClaim[]; error?: unknown; }): void { - const keys = normalizeReplayKeys(params.keys); - for (const key of keys) { - params.guard.release(key, { - namespace: TELEGRAM_MESSAGE_DISPATCH_DEDUPE_NAMESPACE, - error: params.error, - }); + for (const claim of new Set(params.claims ?? [])) { + claim.release({ error: params.error }); } } diff --git a/extensions/telegram/src/polling-session.test.ts b/extensions/telegram/src/polling-session.test.ts index 9d415f967a3..61d26a4cbc1 100644 --- a/extensions/telegram/src/polling-session.test.ts +++ b/extensions/telegram/src/polling-session.test.ts @@ -125,26 +125,27 @@ async function claimSpooledUpdate(update: TelegramSpooledUpdate) { async function createTelegramMessageDispatchReplayForgetError(): Promise { type ReplayGuard = Parameters[0]["guard"]; + type ReplayClaim = import("openclaw/plugin-sdk/persistent-dedupe").ChannelReplayClaimHandle; const diskError = new Error("dedupe disk write failed"); const guard: ReplayGuard = { - claim: async () => ({ kind: "claimed" }), - commit: async (key, options) => { + claim: async () => ({ kind: "invalid" }), + forget: async (event) => !("keys" in event && event.keys?.[0] === "first"), + warmup: async () => 0, + }; + const claims: ReplayClaim[] = ["first", "second"].map((key) => ({ + keys: [key], + commit: async (options) => { if (key === "second") { options?.onDiskError?.(diskError); } return true; }, - forget: async (key) => key !== "first", - hasRecent: async () => false, - warmup: async () => 0, - clearMemory: () => undefined, - memorySize: () => 0, release: () => undefined, - }; + })); try { await commitTelegramMessageDispatchReplay({ guard, - keys: ["first", "second"], + claims, requirePersistent: true, }); } catch (error) { diff --git a/extensions/whatsapp/src/inbound/dedupe.ts b/extensions/whatsapp/src/inbound/dedupe.ts index d99e4de6625..48c3e12e41e 100644 --- a/extensions/whatsapp/src/inbound/dedupe.ts +++ b/extensions/whatsapp/src/inbound/dedupe.ts @@ -1,15 +1,20 @@ // Whatsapp plugin module implements dedupe behavior. import { createDedupeCache } from "openclaw/plugin-sdk/dedupe-runtime"; -import { createClaimableDedupe } from "openclaw/plugin-sdk/persistent-dedupe"; +import { createChannelReplayGuard } from "openclaw/plugin-sdk/persistent-dedupe"; export const WHATSAPP_INBOUND_DEDUPE_TTL_MS = 20 * 60_000; const RECENT_WEB_MESSAGE_MAX = 5000; const RECENT_OUTBOUND_MESSAGE_TTL_MS = 20 * 60_000; const RECENT_OUTBOUND_MESSAGE_MAX = 5000; -const claimableInboundMessages = createClaimableDedupe({ - ttlMs: WHATSAPP_INBOUND_DEDUPE_TTL_MS, - memoryMaxSize: RECENT_WEB_MESSAGE_MAX, +type WhatsAppInboundReplayKeys = string | readonly string[]; + +export const whatsAppInboundReplayGuard = createChannelReplayGuard({ + dedupe: { + ttlMs: WHATSAPP_INBOUND_DEDUPE_TTL_MS, + memoryMaxSize: RECENT_WEB_MESSAGE_MAX, + }, + buildReplayKey: (keys) => keys, }); const recentOutboundMessages = createDedupeCache({ ttlMs: RECENT_OUTBOUND_MESSAGE_TTL_MS, @@ -38,27 +43,10 @@ function buildMessageKey(params: { } export function resetWebInboundDedupe(): void { - claimableInboundMessages.clearMemory(); + whatsAppInboundReplayGuard.clearMemory(); recentOutboundMessages.clear(); } -type RecentInboundMessageClaimKind = "claimed" | "duplicate" | "inflight"; - -export async function claimRecentInboundMessageDelivery( - key: string, -): Promise { - const claim = await claimableInboundMessages.claim(key); - return claim.kind; -} - -export async function commitRecentInboundMessage(key: string): Promise { - await claimableInboundMessages.commit(key); -} - -export function releaseRecentInboundMessage(key: string, error?: unknown): void { - claimableInboundMessages.release(key, { error }); -} - export function rememberRecentOutboundMessage(params: { accountId: string; remoteJid: string; diff --git a/extensions/whatsapp/src/inbound/monitor.ts b/extensions/whatsapp/src/inbound/monitor.ts index 1d6f002670b..0974d251878 100644 --- a/extensions/whatsapp/src/inbound/monitor.ts +++ b/extensions/whatsapp/src/inbound/monitor.ts @@ -22,9 +22,9 @@ import { parseStrictFiniteNumber, resolveExpiresAtMsFromDurationMs, } from "openclaw/plugin-sdk/number-runtime"; +import type { ChannelReplayClaimHandle } from "openclaw/plugin-sdk/persistent-dedupe"; import { defaultRuntime } from "openclaw/plugin-sdk/runtime-env"; import { createSubsystemLogger } from "openclaw/plugin-sdk/runtime-env"; -import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; import { maybeResolveWhatsAppApprovalReaction } from "../approval-reactions.js"; import { readWebSelfIdentityForDecision, WhatsAppAuthUnstableError } from "../auth-store.js"; import { getWhatsAppConnectionController } from "../connection-controller-runtime-context.js"; @@ -58,12 +58,10 @@ import { requireWhatsAppInboundAdmission, } from "./admission.js"; import { - claimRecentInboundMessageDelivery, - commitRecentInboundMessage, isRecentOutboundMessage, - releaseRecentInboundMessage, rememberRecentOutboundMessage, WhatsAppRetryableInboundError, + whatsAppInboundReplayGuard, } from "./dedupe.js"; import { createWhatsAppDurableInboundMessageId, @@ -453,7 +451,7 @@ export async function attachWebInboxToSocket( }; type QueuedInboundMessageMetadata = { admission: AdmittedWebInboundCallbackMessage["admission"]; - dedupeKey?: string; + replayClaim?: ChannelReplayClaimHandle; debounceKey?: string; durableId?: string; readReceipt?: WhatsAppReadReceiptTarget; @@ -503,9 +501,9 @@ export async function attachWebInboxToSocket( entries: QueuedInboundMessage[], error?: unknown, ): Promise => { - const dedupeKeys = uniqueStrings( - entries.map((entry) => entry.dedupeKey).filter(isNonEmptyString), - ); + const replayClaims = entries + .map((entry) => entry.replayClaim) + .filter((claim): claim is ChannelReplayClaimHandle => claim !== undefined); const durableEntries = entries.filter( (entry): entry is QueuedInboundMessage & { durableId: string } => isNonEmptyString(entry.durableId), @@ -516,7 +514,9 @@ export async function attachWebInboxToSocket( ); const retryableError = resolveRetryableWhatsAppInboundError(error); if (retryableError) { - dedupeKeys.forEach((dedupeKey) => releaseRecentInboundMessage(dedupeKey, retryableError)); + for (const claim of replayClaims) { + claim.release({ error: retryableError }); + } await Promise.all( durableEntries.map((entry) => durableInboundJournal.release(entry.durableId, { @@ -527,7 +527,7 @@ export async function attachWebInboxToSocket( return; } await Promise.all([ - ...dedupeKeys.map((dedupeKey) => commitRecentInboundMessage(dedupeKey)), + ...replayClaims.map((claim) => claim.commit()), ...durableEntries.map((entry) => durableInboundJournal.complete( entry.durableId, @@ -1235,9 +1235,11 @@ export async function attachWebInboxToSocket( } const dedupeKey = inbound.id ? `${options.accountId}:${inbound.remoteJid}:${inbound.id}` : ""; - const dedupeClaim = dedupeKey ? await claimRecentInboundMessageDelivery(dedupeKey) : "claimed"; - if (dedupeClaim !== "claimed") { - if (dedupeClaim === "duplicate") { + const dedupeClaim = dedupeKey + ? await whatsAppInboundReplayGuard.claim(dedupeKey) + : ({ kind: "invalid" } as const); + if (dedupeClaim.kind === "duplicate" || dedupeClaim.kind === "inflight") { + if (dedupeClaim.kind === "duplicate") { await completeUndeliverableDurableInbound(durableId, durableMetadata); await maybeMarkNonSelfChatReadReceipt(inbound, deliveryReadReceipt); } @@ -1249,6 +1251,7 @@ export async function attachWebInboxToSocket( durableId, readReceipt: deliveryReadReceipt, receiveOrder, + ...(dedupeClaim.kind === "claimed" ? { replayClaim: dedupeClaim.handle } : {}), }); }; @@ -1361,6 +1364,7 @@ export async function attachWebInboxToSocket( durableId?: string; readReceipt?: WhatsAppReadReceiptTarget; receiveOrder?: number; + replayClaim?: ChannelReplayClaimHandle; }, ) => { const chatJid = inbound.remoteJid; @@ -1500,7 +1504,7 @@ export async function attachWebInboxToSocket( } : undefined, group, - dedupeKey: inbound.id ? `${options.accountId}:${inbound.remoteJid}:${inbound.id}` : undefined, + replayClaim: durable.replayClaim, durableId: durable.durableId, readReceipt: durable.readReceipt, receiveOrder: durable.receiveOrder, diff --git a/extensions/zalo/src/monitor.webhook.ts b/extensions/zalo/src/monitor.webhook.ts index ed8bba19b27..d090ac41b79 100644 --- a/extensions/zalo/src/monitor.webhook.ts +++ b/extensions/zalo/src/monitor.webhook.ts @@ -1,6 +1,6 @@ // Zalo plugin module implements monitor.webhook behavior. import type { IncomingMessage, ServerResponse } from "node:http"; -import { createClaimableDedupe } from "openclaw/plugin-sdk/persistent-dedupe"; +import { createChannelReplayGuard } from "openclaw/plugin-sdk/persistent-dedupe"; import { safeEqualSecret } from "openclaw/plugin-sdk/security-runtime"; import { runDetachedWebhookWork } from "openclaw/plugin-sdk/webhook-request-guards"; import type { ResolvedZaloAccount } from "./accounts.js"; @@ -52,10 +52,6 @@ const webhookRateLimiter = createFixedWindowRateLimiter({ maxRequests: WEBHOOK_RATE_LIMIT_DEFAULTS.maxRequests, maxTrackedKeys: WEBHOOK_RATE_LIMIT_DEFAULTS.maxTrackedKeys, }); -const recentWebhookEvents = createClaimableDedupe({ - ttlMs: ZALO_WEBHOOK_REPLAY_WINDOW_MS, - memoryMaxSize: 5000, -}); const webhookAnomalyTracker = createWebhookAnomalyTracker({ maxTrackedKeys: WEBHOOK_ANOMALY_COUNTER_DEFAULTS.maxTrackedKeys, ttlMs: WEBHOOK_ANOMALY_COUNTER_DEFAULTS.ttlMs, @@ -93,6 +89,17 @@ function buildReplayEventCacheKey(target: ZaloWebhookTarget, update: ZaloUpdate) ]); } +const recentWebhookEvents = createChannelReplayGuard<{ + target: ZaloWebhookTarget; + update: ZaloUpdate; +}>({ + dedupe: { + ttlMs: ZALO_WEBHOOK_REPLAY_WINDOW_MS, + memoryMaxSize: 5000, + }, + buildReplayKey: ({ target, update }) => buildReplayEventCacheKey(target, update), +}); + export class ZaloRetryableWebhookError extends Error { constructor(message: string, options?: ErrorOptions) { super(message, options); @@ -106,31 +113,19 @@ async function processZaloReplayGuardedUpdate(params: { processUpdate: ZaloWebhookProcessUpdate; nowMs?: number; }): Promise<"processed" | "duplicate"> { - const replayEventKey = buildReplayEventCacheKey(params.target, params.update); - if (replayEventKey) { - const replayClaim = await recentWebhookEvents.claim(replayEventKey, { now: params.nowMs }); - if (replayClaim.kind !== "claimed") { - return "duplicate"; - } - } - - params.target.statusSink?.({ lastInboundAt: Date.now() }); - try { - await params.processUpdate({ update: params.update, target: params.target }); - if (replayEventKey) { - await recentWebhookEvents.commit(replayEventKey); - } - return "processed"; - } catch (error) { - if (replayEventKey) { - if (error instanceof ZaloRetryableWebhookError) { - recentWebhookEvents.release(replayEventKey, { error }); - } else { - await recentWebhookEvents.commit(replayEventKey); - } - } - throw error; - } + const event = { target: params.target, update: params.update }; + const result = await recentWebhookEvents.processGuarded( + event, + async () => { + params.target.statusSink?.({ lastInboundAt: Date.now() }); + await params.processUpdate(event); + }, + { + dedupe: { now: params.nowMs }, + onError: (error) => (error instanceof ZaloRetryableWebhookError ? "release" : "commit"), + }, + ); + return result.kind === "processed" ? "processed" : "duplicate"; } function recordWebhookStatus( diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index 1b4e906181f..4454dcf18d0 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -241,7 +241,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +2: generic channel retry runner and Retry-After parser. // +1: shared speech-provider API key resolver. // +32: shared channel setup, config-schema, policy, and status helpers. - 7984, + // +2: shared channel replay-guard factory and claim handle. + 7986, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( @@ -261,7 +262,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +2: generic channel retry runner and Retry-After parser. // +1: shared speech-provider API key resolver. // +24: shared channel setup, config-schema, policy, and status helpers. - 4464, + // +1: shared channel replay-guard factory. + 4465, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( diff --git a/src/plugin-sdk/channel-replay-guard.ts b/src/plugin-sdk/channel-replay-guard.ts new file mode 100644 index 00000000000..babfb33fabe --- /dev/null +++ b/src/plugin-sdk/channel-replay-guard.ts @@ -0,0 +1,275 @@ +import type { + ClaimableDedupe, + ClaimableDedupeOptions, + PersistentDedupeCheckOptions, +} from "./persistent-dedupe.types.js"; + +type ReplayKeys = string | readonly (string | null | undefined)[] | null | undefined; + +type ChannelReplayCommitOptions = Omit; + +export type ChannelReplayClaimHandle = { + readonly keys: readonly [string, ...string[]]; + commit: (options?: ChannelReplayCommitOptions) => Promise; + release: (options?: { error?: unknown }) => void; +}; + +type ChannelReplayClaimResult = + | { kind: "claimed"; handle: ChannelReplayClaimHandle } + | { kind: "duplicate" } + | { kind: "inflight"; pending: Promise } + | { kind: "invalid" }; + +type ChannelReplayProcessResult = + | { kind: "processed"; value: T } + | { kind: "duplicate" } + | { kind: "inflight"; pending: Promise }; + +type ChannelReplayErrorMode = "commit" | "release"; + +type ChannelReplayProcessOptions = { + dedupe?: PersistentDedupeCheckOptions; + onError?: ChannelReplayErrorMode | ((error: unknown) => ChannelReplayErrorMode); +}; + +export type ChannelReplayGuardParams = { + dedupe: ClaimableDedupeOptions; + buildReplayKey: (event: TEvent) => ReplayKeys; + namespace?: (event: TEvent) => string | undefined; +}; + +export type ChannelReplayGuard = { + claim: ( + event: TEvent, + options?: PersistentDedupeCheckOptions, + ) => Promise; + shouldProcess: (event: TEvent, options?: PersistentDedupeCheckOptions) => Promise; + processGuarded: ( + event: TEvent, + process: () => Promise, + options?: ChannelReplayProcessOptions, + ) => Promise>; + hasRecent: (event: TEvent, options?: PersistentDedupeCheckOptions) => Promise; + forget: (event: TEvent, options?: PersistentDedupeCheckOptions) => Promise; + warmup: (namespace?: string, onError?: (error: unknown) => void) => Promise; + clearMemory: () => void; +}; + +function normalizeReplayKeys(value: ReplayKeys): string[] { + const values = Array.isArray(value) ? value : [value]; + return [ + ...new Set(values.map((key) => key?.trim()).filter((key): key is string => Boolean(key))), + ]; +} + +export function createChannelReplayGuardWithDedupe( + params: Omit, "dedupe">, + dedupe: ClaimableDedupe & Required>, +): ChannelReplayGuard { + const claimOwners = new Map(); + const resolveKeys = (event: TEvent) => normalizeReplayKeys(params.buildReplayKey(event)); + const resolveOwnerKey = (key: string, options?: PersistentDedupeCheckOptions) => + `${options?.namespace?.trim() || "global"}\0${key}`; + const resolveOptions = ( + event: TEvent, + options?: PersistentDedupeCheckOptions, + ): PersistentDedupeCheckOptions | undefined => { + if (options?.namespace !== undefined) { + return options; + } + const namespace = params.namespace?.(event); + return namespace === undefined ? options : { ...options, namespace }; + }; + const releaseKeys = ( + keys: readonly string[], + options?: { namespace?: string; error?: unknown }, + ) => { + for (const key of keys) { + dedupe.release(key, options); + } + }; + + const commitKeys = async ( + keys: readonly string[], + options?: PersistentDedupeCheckOptions, + ): Promise => { + const results = await Promise.all(keys.map((key) => dedupe.commit(key, options))); + return results.some(Boolean); + }; + + const createClaimHandle = ( + keys: [string, ...string[]], + claimId: symbol, + dedupeOptions?: PersistentDedupeCheckOptions, + ): ChannelReplayClaimHandle => { + const ownedKeys = Object.freeze([...keys]) as readonly [string, ...string[]]; + let settlement: + | { kind: "claimed" } + | { kind: "committing"; pending: Promise } + | { kind: "released" } = { kind: "claimed" }; + return { + keys: ownedKeys, + commit: (options) => { + if (settlement.kind === "committing") { + return settlement.pending; + } + if (settlement.kind === "released") { + return Promise.resolve(false); + } + const settlingKeys = ownedKeys.filter((key) => { + const owner = claimOwners.get(resolveOwnerKey(key, dedupeOptions)); + if (owner?.claimId !== claimId || owner.state !== "claimed") { + return false; + } + owner.state = "committing"; + return true; + }); + if (settlingKeys.length === 0) { + settlement = { kind: "released" }; + return Promise.resolve(false); + } + const pending = commitKeys( + settlingKeys, + options + ? { ...dedupeOptions, ...options, namespace: dedupeOptions?.namespace } + : dedupeOptions, + ).finally(() => { + for (const key of settlingKeys) { + const ownerKey = resolveOwnerKey(key, dedupeOptions); + if (claimOwners.get(ownerKey)?.claimId === claimId) { + claimOwners.delete(ownerKey); + } + } + }); + settlement = { kind: "committing", pending }; + return pending; + }, + release: (options) => { + if (settlement.kind !== "claimed") { + return; + } + settlement = { kind: "released" }; + const releasingKeys = ownedKeys.filter( + (key) => claimOwners.get(resolveOwnerKey(key, dedupeOptions))?.claimId === claimId, + ); + releaseKeys(releasingKeys, { namespace: dedupeOptions?.namespace, error: options?.error }); + for (const key of releasingKeys) { + const ownerKey = resolveOwnerKey(key, dedupeOptions); + if (claimOwners.get(ownerKey)?.claimId === claimId) { + claimOwners.delete(ownerKey); + } + } + }, + }; + }; + + const claim: ChannelReplayGuard["claim"] = async (event, options) => { + const keys = resolveKeys(event); + if (keys.length === 0) { + return { kind: "invalid" }; + } + const dedupeOptions = resolveOptions(event, options); + const claimId = Symbol("channel-replay-claim"); + const claimedKeys: string[] = []; + const pending: Promise[] = []; + try { + for (const key of keys) { + const result = await dedupe.claim(key, dedupeOptions); + if (result.kind === "claimed") { + claimedKeys.push(key); + claimOwners.set(resolveOwnerKey(key, dedupeOptions), { claimId, state: "claimed" }); + } else if (result.kind === "inflight") { + pending.push(result.pending); + } + } + } catch (error) { + releaseKeys(claimedKeys, { namespace: dedupeOptions?.namespace, error }); + for (const key of claimedKeys) { + const ownerKey = resolveOwnerKey(key, dedupeOptions); + if (claimOwners.get(ownerKey)?.claimId === claimId) { + claimOwners.delete(ownerKey); + } + } + throw error; + } + if (claimedKeys.length > 0) { + return { + kind: "claimed", + handle: createClaimHandle(claimedKeys as [string, ...string[]], claimId, dedupeOptions), + }; + } + if (pending.length > 0) { + const aggregate = Promise.all(pending).then((results) => results.some(Boolean)); + void aggregate.catch(() => {}); + return { + kind: "inflight", + pending: aggregate, + }; + } + return { kind: "duplicate" }; + }; + + return { + claim, + shouldProcess: async (event, options) => { + const result = await claim(event, options); + if (result.kind === "invalid") { + return true; + } + if (result.kind !== "claimed") { + return false; + } + return await result.handle.commit(); + }, + processGuarded: async (event, process, options) => { + const dedupeOptions = resolveOptions(event, options?.dedupe); + const result = await claim(event, dedupeOptions); + if (result.kind === "duplicate" || result.kind === "inflight") { + return result; + } + if (result.kind === "invalid") { + return { kind: "processed", value: await process() }; + } + let value: Awaited>; + try { + value = await process(); + } catch (error) { + const errorMode = + typeof options?.onError === "function" + ? options.onError(error) + : (options?.onError ?? "release"); + if (errorMode === "commit") { + await result.handle.commit(); + } else { + result.handle.release({ error }); + } + throw error; + } + await result.handle.commit(); + return { kind: "processed", value }; + }, + hasRecent: async (event, options) => { + const keys = resolveKeys(event); + if (keys.length === 0) { + return false; + } + const dedupeOptions = resolveOptions(event, options); + const results = await Promise.all(keys.map((key) => dedupe.hasRecent(key, dedupeOptions))); + return results.some(Boolean); + }, + forget: async (event, options) => { + const dedupeOptions = resolveOptions(event, options); + // Active handles alone own settlement; forget only removes committed rows. + const keys = resolveKeys(event).filter( + (key) => !claimOwners.has(resolveOwnerKey(key, dedupeOptions)), + ); + if (keys.length === 0) { + return false; + } + const results = await Promise.all(keys.map((key) => dedupe.forget(key, dedupeOptions))); + return results.some(Boolean); + }, + warmup: dedupe.warmup, + clearMemory: dedupe.clearMemory, + }; +} diff --git a/src/plugin-sdk/persistent-dedupe.test.ts b/src/plugin-sdk/persistent-dedupe.test.ts new file mode 100644 index 00000000000..b116ef44154 --- /dev/null +++ b/src/plugin-sdk/persistent-dedupe.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it, vi } from "vitest"; +import { createChannelReplayGuard } from "./persistent-dedupe.js"; + +type ReplayEvent = { + accountId: string; + keys: readonly (string | null | undefined)[]; +}; + +function createGuard() { + return createChannelReplayGuard({ + dedupe: { ttlMs: 10_000, memoryMaxSize: 100 }, + buildReplayKey: (event) => event.keys, + namespace: (event) => event.accountId, + }); +} + +async function expectClaimed(claim: Awaited["claim"]>>) { + expect(claim.kind).toBe("claimed"); + if (claim.kind !== "claimed") { + throw new Error(`expected claimed result, received ${claim.kind}`); + } + return claim.handle; +} + +describe("createChannelReplayGuard", () => { + it("normalizes multi-key claims and mirrors commit state to in-flight waiters", async () => { + const guard = createGuard(); + const event = { accountId: "work", keys: [" message-1 ", "message-1", "message-2"] }; + + const handle = await expectClaimed(await guard.claim(event)); + expect(handle.keys).toEqual(["message-1", "message-2"]); + const inflight = await guard.claim(event); + expect(inflight.kind).toBe("inflight"); + await expect(handle.commit()).resolves.toBe(true); + if (inflight.kind === "inflight") { + await expect(inflight.pending).resolves.toBe(true); + } + await expect(guard.claim(event)).resolves.toEqual({ kind: "duplicate" }); + }); + + it("fails open for invalid keys without recording them", async () => { + const guard = createGuard(); + const event = { accountId: "work", keys: [" ", null, undefined] }; + const process = vi.fn(async () => "handled"); + + await expect(guard.claim(event)).resolves.toEqual({ kind: "invalid" }); + await expect(guard.shouldProcess(event)).resolves.toBe(true); + await expect(guard.processGuarded(event, process)).resolves.toEqual({ + kind: "processed", + value: "handled", + }); + expect("commit" in guard).toBe(false); + expect("release" in guard).toBe(false); + expect(process).toHaveBeenCalledOnce(); + }); + + it("releases failed claims and rejects their in-flight waiters", async () => { + const guard = createGuard(); + const event = { accountId: "work", keys: ["message-3"] }; + + const handle = await expectClaimed(await guard.claim(event)); + const inflight = await guard.claim(event); + const failure = new Error("retry me"); + handle.release({ error: failure }); + if (inflight.kind === "inflight") { + await expect(inflight.pending).rejects.toThrow("retry me"); + } + await expect(guard.claim(event)).resolves.toMatchObject({ kind: "claimed" }); + }); + + it("does not let a mixed claim commit another claim's in-flight key", async () => { + const guard = createGuard(); + const sharedOwner = await expectClaimed( + await guard.claim({ accountId: "work", keys: ["shared", "first-only"] }), + ); + const mixedOwner = await expectClaimed( + await guard.claim({ accountId: "work", keys: ["shared", "second-only"] }), + ); + expect(mixedOwner.keys).toEqual(["second-only"]); + const sharedWaiter = await guard.claim({ accountId: "work", keys: ["shared"] }); + + await expect(mixedOwner.commit()).resolves.toBe(true); + sharedOwner.release({ error: new Error("first handler failed") }); + + if (sharedWaiter.kind === "inflight") { + await expect(sharedWaiter.pending).rejects.toThrow("first handler failed"); + } + await expect(guard.claim({ accountId: "work", keys: ["shared"] })).resolves.toMatchObject({ + kind: "claimed", + }); + await expect(guard.claim({ accountId: "work", keys: ["second-only"] })).resolves.toEqual({ + kind: "duplicate", + }); + }); + + it("does not let the first claim commit keys owned by a mixed second claim", async () => { + const guard = createGuard(); + const firstOwner = await expectClaimed( + await guard.claim({ accountId: "work", keys: ["shared", "first-only"] }), + ); + const secondOwner = await expectClaimed( + await guard.claim({ accountId: "work", keys: ["shared", "second-only"] }), + ); + const sharedWaiter = await guard.claim({ accountId: "work", keys: ["shared"] }); + const secondWaiter = await guard.claim({ accountId: "work", keys: ["second-only"] }); + + secondOwner.release({ error: new Error("second handler failed") }); + await expect(firstOwner.commit()).resolves.toBe(true); + + if (sharedWaiter.kind === "inflight") { + await expect(sharedWaiter.pending).resolves.toBe(true); + } + if (secondWaiter.kind === "inflight") { + await expect(secondWaiter.pending).rejects.toThrow("second handler failed"); + } + await expect( + guard.claim({ accountId: "work", keys: ["shared", "first-only"] }), + ).resolves.toEqual({ kind: "duplicate" }); + await expect(guard.claim({ accountId: "work", keys: ["second-only"] })).resolves.toMatchObject({ + kind: "claimed", + }); + }); + + it.each([ + { errorMode: "release" as const, nextKind: "claimed" }, + { errorMode: "commit" as const, nextKind: "duplicate" }, + ])("uses $errorMode error settlement in processGuarded", async ({ errorMode, nextKind }) => { + const guard = createGuard(); + const event = { accountId: "work", keys: [`message-${errorMode}`] }; + + await expect( + guard.processGuarded( + event, + async () => { + throw new Error("handler failed"); + }, + { onError: errorMode }, + ), + ).rejects.toThrow("handler failed"); + await expect(guard.claim(event)).resolves.toMatchObject({ kind: nextKind }); + }); + + it("scopes keys by namespace and supports recency cleanup", async () => { + const guard = createGuard(); + const work = { accountId: "work", keys: ["message-4"] }; + const home = { accountId: "home", keys: ["message-4"] }; + + await expect(guard.shouldProcess(work)).resolves.toBe(true); + await expect(guard.shouldProcess(work)).resolves.toBe(false); + await expect(guard.shouldProcess(home)).resolves.toBe(true); + await expect(guard.hasRecent(work)).resolves.toBe(true); + await expect(guard.forget(work)).resolves.toBe(true); + await expect(guard.hasRecent(work)).resolves.toBe(false); + }); +}); diff --git a/src/plugin-sdk/persistent-dedupe.ts b/src/plugin-sdk/persistent-dedupe.ts index d2c3793c050..2334cbf9f3a 100644 --- a/src/plugin-sdk/persistent-dedupe.ts +++ b/src/plugin-sdk/persistent-dedupe.ts @@ -8,59 +8,43 @@ import { createPluginStateSyncKeyedStore, } from "../plugin-state/plugin-state-store.js"; import type { PluginStateSyncKeyedStore } from "../plugin-state/plugin-state-store.types.js"; -import type { FileLockOptions } from "./file-lock.js"; +import { + createChannelReplayGuardWithDedupe, + type ChannelReplayGuard, + type ChannelReplayClaimHandle, + type ChannelReplayGuardParams, +} from "./channel-replay-guard.js"; +import type { + ClaimableDedupe, + ClaimableDedupeClaimResult, + ClaimableDedupeOptions, + PersistentDedupe, + PersistentDedupeCheckOptions, + PersistentDedupeLegacyPathOptions, + PersistentDedupeOptions, + PersistentDedupePluginStateOptions, +} from "./persistent-dedupe.types.js"; const LEGACY_PATH_OWNER_ID = "core:persistent-dedupe"; const DEFAULT_NAMESPACE_PREFIX = "persistent-dedupe"; +export type { ChannelReplayClaimHandle }; +export type { + ClaimableDedupe, + ClaimableDedupeClaimResult, + ClaimableDedupeOptions, + PersistentDedupe, + PersistentDedupeCheckOptions, + PersistentDedupeLegacyPathOptions, + PersistentDedupeOptions, + PersistentDedupePluginStateOptions, +} from "./persistent-dedupe.types.js"; + export type PersistentDedupeEntry = { key: string; seenAt: number; }; -type PersistentDedupeBaseOptions = { - /** Milliseconds a recorded key remains recent; `0` keeps keys until cache pruning. */ - ttlMs: number; - /** Maximum process-local cache entries used before consulting SQLite. */ - memoryMaxSize: number; - onDiskError?: (error: unknown) => void; -}; - -/** Configuration for a SQLite plugin-state dedupe namespace cache. */ -export type PersistentDedupePluginStateOptions = PersistentDedupeBaseOptions & { - /** Plugin id that owns the persisted dedupe namespace. */ - pluginId: string; - /** Prefix for persisted plugin-state namespaces; defaults to `persistent-dedupe`. */ - namespacePrefix?: string; - /** Maximum persisted entries retained per namespace. */ - stateMaxEntries: number; - /** Test/runtime env used to resolve the shared OpenClaw state database. */ - env?: NodeJS.ProcessEnv; - resolveFilePath?: undefined; - fileMaxEntries?: undefined; - lockOptions?: undefined; -}; - -/** Legacy path-shaped configuration. Paths now name SQLite namespaces, not JSON files. */ -export type PersistentDedupeLegacyPathOptions = PersistentDedupeBaseOptions & { - pluginId?: undefined; - stateMaxEntries?: undefined; - namespacePrefix?: undefined; - /** Maximum persisted entries retained per legacy namespace. */ - fileMaxEntries: number; - /** Maps a namespace to the retired JSON path; used only to derive a stable SQLite namespace. */ - resolveFilePath: (namespace: string) => string; - /** Test/runtime env used to resolve the shared OpenClaw state database. */ - env?: NodeJS.ProcessEnv; - /** @deprecated File locks are ignored because persistence is SQLite-backed. */ - lockOptions?: Partial; -}; - -/** Configuration for a persisted dedupe namespace cache. */ -export type PersistentDedupeOptions = - | PersistentDedupePluginStateOptions - | PersistentDedupeLegacyPathOptions; - export type PersistentDedupeLegacyJsonMigrationResult = { imported: number; skippedExpired: number; @@ -88,84 +72,6 @@ type PersistentDedupeLegacyJsonEntriesResult = { skippedInvalid: number; }; -/** Per-call options used when checking or recording a dedupe key. */ -export type PersistentDedupeCheckOptions = { - /** Logical bucket for the key; omitted/blank values use `global`. */ - namespace?: string; - /** Test or replay timestamp override used for TTL checks and writes. */ - now?: number; - /** Per-call disk error hook, overriding the helper-level hook. */ - onDiskError?: (error: unknown) => void; -}; - -/** Disk-backed dedupe guard that records recently seen keys per namespace. */ -export type PersistentDedupe = { - /** Returns true only when the key was not recently seen and was recorded for future checks. */ - checkAndRecord: (key: string, options?: PersistentDedupeCheckOptions) => Promise; - /** Checks memory/disk recency without recording a new timestamp. */ - hasRecent: (key: string, options?: PersistentDedupeCheckOptions) => Promise; - /** Removes a recorded key from process memory and persisted storage. */ - forget: (key: string, options?: PersistentDedupeCheckOptions) => Promise; - /** Loads recent disk entries into memory for one namespace and returns the loaded count. */ - warmup: (namespace?: string, onError?: (error: unknown) => void) => Promise; - /** Clears only process-local memory; persisted namespace files are left intact. */ - clearMemory: () => void; - /** Returns the current process-local cache size. */ - memorySize: () => number; -}; - -/** Claim attempt result for dedupe flows that need in-flight ownership. */ -export type ClaimableDedupeClaimResult = - | { kind: "claimed" } - | { kind: "duplicate" } - | { kind: "inflight"; pending: Promise }; - -/** Options for a claimable dedupe guard, either persistent or memory-only. */ -export type ClaimableDedupeOptions = - | PersistentDedupePluginStateOptions - | PersistentDedupeLegacyPathOptions - | { - ttlMs: number; - memoryMaxSize: number; - pluginId?: undefined; - stateMaxEntries?: undefined; - namespacePrefix?: undefined; - env?: undefined; - resolveFilePath?: undefined; - fileMaxEntries?: undefined; - lockOptions?: undefined; - onDiskError?: undefined; - }; - -/** Dedupe guard that lets one caller own a key while others wait or detect duplicates. */ -export type ClaimableDedupe = { - /** Starts ownership of a key, reports duplicates, or returns the active claim's pending result. */ - claim: ( - key: string, - options?: PersistentDedupeCheckOptions, - ) => Promise; - /** Records a claimed key as handled and resolves any waiters with the recorded result. */ - commit: (key: string, options?: PersistentDedupeCheckOptions) => Promise; - /** Releases an active claim without recording it, rejecting waiters with the supplied error. */ - release: ( - key: string, - options?: { - namespace?: string; - error?: unknown; - }, - ) => void; - /** Checks whether the key is recent without claiming or committing it. */ - hasRecent: (key: string, options?: PersistentDedupeCheckOptions) => Promise; - /** Removes an active or committed key from memory and persisted storage when supported. */ - forget?: (key: string, options?: PersistentDedupeCheckOptions) => Promise; - /** Warms persistent storage into memory when configured; memory-only guards return zero. */ - warmup: (namespace?: string, onError?: (error: unknown) => void) => Promise; - /** Clears process-local caches and in-memory persistent state. */ - clearMemory: () => void; - /** Returns the current process-local cache size. */ - memorySize: () => number; -}; - function resolveNamespace(namespace?: string): string { return namespace?.trim() || "global"; } @@ -771,3 +677,10 @@ export function createClaimableDedupe( memorySize: () => persistent?.memorySize() ?? memory.size(), }; } + +/** Create an event-keyed replay guard whose claims own their settlement handles. */ +export function createChannelReplayGuard( + params: ChannelReplayGuardParams, +): ChannelReplayGuard { + return createChannelReplayGuardWithDedupe(params, createClaimableDedupe(params.dedupe)); +} diff --git a/src/plugin-sdk/persistent-dedupe.types.ts b/src/plugin-sdk/persistent-dedupe.types.ts new file mode 100644 index 00000000000..20b520a9b93 --- /dev/null +++ b/src/plugin-sdk/persistent-dedupe.types.ts @@ -0,0 +1,122 @@ +import type { FileLockOptions } from "./file-lock.js"; + +type PersistentDedupeBaseOptions = { + /** Milliseconds a recorded key remains recent; `0` keeps keys until cache pruning. */ + ttlMs: number; + /** Maximum process-local cache entries used before consulting SQLite. */ + memoryMaxSize: number; + onDiskError?: (error: unknown) => void; +}; + +/** Configuration for a SQLite plugin-state dedupe namespace cache. */ +export type PersistentDedupePluginStateOptions = PersistentDedupeBaseOptions & { + /** Plugin id that owns the persisted dedupe namespace. */ + pluginId: string; + /** Prefix for persisted plugin-state namespaces; defaults to `persistent-dedupe`. */ + namespacePrefix?: string; + /** Maximum persisted entries retained per namespace. */ + stateMaxEntries: number; + /** Test/runtime env used to resolve the shared OpenClaw state database. */ + env?: NodeJS.ProcessEnv; + resolveFilePath?: undefined; + fileMaxEntries?: undefined; + lockOptions?: undefined; +}; + +/** Legacy path-shaped configuration. Paths now name SQLite namespaces, not JSON files. */ +export type PersistentDedupeLegacyPathOptions = PersistentDedupeBaseOptions & { + pluginId?: undefined; + stateMaxEntries?: undefined; + namespacePrefix?: undefined; + /** Maximum persisted entries retained per legacy namespace. */ + fileMaxEntries: number; + /** Maps a namespace to the retired JSON path; used only to derive a stable SQLite namespace. */ + resolveFilePath: (namespace: string) => string; + /** Test/runtime env used to resolve the shared OpenClaw state database. */ + env?: NodeJS.ProcessEnv; + /** @deprecated File locks are ignored because persistence is SQLite-backed. */ + lockOptions?: Partial; +}; + +/** Configuration for a persisted dedupe namespace cache. */ +export type PersistentDedupeOptions = + | PersistentDedupePluginStateOptions + | PersistentDedupeLegacyPathOptions; + +/** Per-call options used when checking or recording a dedupe key. */ +export type PersistentDedupeCheckOptions = { + /** Logical bucket for the key; omitted/blank values use `global`. */ + namespace?: string; + /** Test or replay timestamp override used for TTL checks and writes. */ + now?: number; + /** Per-call disk error hook, overriding the helper-level hook. */ + onDiskError?: (error: unknown) => void; +}; + +/** Disk-backed dedupe guard that records recently seen keys per namespace. */ +export type PersistentDedupe = { + /** Returns true only when the key was not recently seen and was recorded for future checks. */ + checkAndRecord: (key: string, options?: PersistentDedupeCheckOptions) => Promise; + /** Checks memory/disk recency without recording a new timestamp. */ + hasRecent: (key: string, options?: PersistentDedupeCheckOptions) => Promise; + /** Removes a recorded key from process memory and persisted storage. */ + forget: (key: string, options?: PersistentDedupeCheckOptions) => Promise; + /** Loads recent disk entries into memory for one namespace and returns the loaded count. */ + warmup: (namespace?: string, onError?: (error: unknown) => void) => Promise; + /** Clears only process-local memory; persisted namespace files are left intact. */ + clearMemory: () => void; + /** Returns the current process-local cache size. */ + memorySize: () => number; +}; + +/** Claim attempt result for dedupe flows that need in-flight ownership. */ +export type ClaimableDedupeClaimResult = + | { kind: "claimed" } + | { kind: "duplicate" } + | { kind: "inflight"; pending: Promise }; + +/** Options for a claimable dedupe guard, either persistent or memory-only. */ +export type ClaimableDedupeOptions = + | PersistentDedupePluginStateOptions + | PersistentDedupeLegacyPathOptions + | { + ttlMs: number; + memoryMaxSize: number; + pluginId?: undefined; + stateMaxEntries?: undefined; + namespacePrefix?: undefined; + env?: undefined; + resolveFilePath?: undefined; + fileMaxEntries?: undefined; + lockOptions?: undefined; + onDiskError?: undefined; + }; + +/** Dedupe guard that lets one caller own a key while others wait or detect duplicates. */ +export type ClaimableDedupe = { + /** Starts ownership of a key, reports duplicates, or returns the active claim's pending result. */ + claim: ( + key: string, + options?: PersistentDedupeCheckOptions, + ) => Promise; + /** Records a claimed key as handled and resolves any waiters with the recorded result. */ + commit: (key: string, options?: PersistentDedupeCheckOptions) => Promise; + /** Releases an active claim without recording it, rejecting waiters with the supplied error. */ + release: ( + key: string, + options?: { + namespace?: string; + error?: unknown; + }, + ) => void; + /** Checks whether the key is recent without claiming or committing it. */ + hasRecent: (key: string, options?: PersistentDedupeCheckOptions) => Promise; + /** Removes an active or committed key from memory and persisted storage when supported. */ + forget?: (key: string, options?: PersistentDedupeCheckOptions) => Promise; + /** Warms persistent storage into memory when configured; memory-only guards return zero. */ + warmup: (namespace?: string, onError?: (error: unknown) => void) => Promise; + /** Clears process-local caches and in-memory persistent state. */ + clearMemory: () => void; + /** Returns the current process-local cache size. */ + memorySize: () => number; +};