diff --git a/packages/normalization-core/src/node-crypto.ts b/packages/normalization-core/src/node-crypto.ts index 5a8feba198ab..d8b6e213970a 100644 --- a/packages/normalization-core/src/node-crypto.ts +++ b/packages/normalization-core/src/node-crypto.ts @@ -1,10 +1,41 @@ -import { hash } from "node:crypto"; +import { Buffer } from "node:buffer"; +import { createHash, hash, type Hash } from "node:crypto"; +import { writeStableStringify } from "./stable-stringify.js"; import { normalizeOptionalString } from "./string-coerce.js"; export function sha256Hex(input: string | Uint8Array): string { return hash("sha256", input, "hex"); } +export function sha256StableValue(value: unknown): { digest: string; byteWeight: number } { + let digest: Hash | undefined; + let chunks: string[] = []; + let characterCount = 0; + let byteWeight = 0; + writeStableStringify(value, (chunk) => { + if (chunk.length === 0) { + return; + } + chunks.push(chunk); + characterCount += chunk.length; + if (characterCount >= 16_384) { + const text = chunks.join(""); + digest ??= createHash("sha256"); + digest.update(text); + byteWeight += Buffer.byteLength(text); + chunks = []; + characterCount = 0; + } + }); + // Whole JSON string tokens keep surrogate pairs together at every UTF-8 boundary. + const text = chunks.join(""); + byteWeight += Buffer.byteLength(text); + return { + digest: digest ? digest.update(text).digest("hex") : sha256Hex(text), + byteWeight, + }; +} + export function sha256HexPrefixCore(input: string | Uint8Array, length: number): string { return sha256Hex(input).slice(0, length); } diff --git a/packages/normalization-core/src/stable-stringify.test.ts b/packages/normalization-core/src/stable-stringify.test.ts index 50bdc5452943..479a3772461c 100644 --- a/packages/normalization-core/src/stable-stringify.test.ts +++ b/packages/normalization-core/src/stable-stringify.test.ts @@ -3,12 +3,22 @@ * Verifies sorted keys, repeated references, cycles, binary data, and errors. */ import { describe, expect, it } from "vitest"; -import { stableStringify } from "./stable-stringify.js"; +import { sha256Hex, sha256StableValue } from "./node-crypto.js"; +import { stableStringify, writeStableStringify } from "./stable-stringify.js"; const sanitizeSurrogates = (text: string) => text.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { +const serializers: Record = { + stableStringify, + writeStableStringify: (value, normalizeString) => { + const chunks: string[] = []; + writeStableStringify(value, (chunk) => chunks.push(chunk), normalizeString); + return chunks.join(""); + }, +}; + +describe.each(Object.entries(serializers))("%s", (_name, serialize) => { it.each([ ['{"z":1,"a":2}', '{"a":2,"z":1}'], [ @@ -17,11 +27,11 @@ describe("stableStringify", () => { ], ['["text",0,-2.5,null,false]', '["text",0,-2.5,null,false]'], ])("preserves deterministic bytes for parsed JSON %#", (json, expected) => { - expect(stableStringify(JSON.parse(json))).toBe(expected); + expect(serialize(JSON.parse(json))).toBe(expected); }); it("sorts object keys recursively", () => { - expect(stableStringify({ b: { d: 4, c: 3 }, a: 1 })).toBe('{"a":1,"b":{"c":3,"d":4}}'); + expect(serialize({ b: { d: 4, c: 3 }, a: 1 })).toBe('{"a":1,"b":{"c":3,"d":4}}'); }); it("marks true circular references without collapsing repeated references", () => { @@ -29,9 +39,7 @@ describe("stableStringify", () => { const root: Record = { first: shared, second: shared }; root.self = root; - expect(stableStringify(root)).toBe( - '{"first":{"value":1},"second":{"value":1},"self":"[Circular]"}', - ); + expect(serialize(root)).toBe('{"first":{"value":1},"second":{"value":1},"self":"[Circular]"}'); }); it("handles circular arrays without treating later siblings as circular", () => { @@ -39,7 +47,7 @@ describe("stableStringify", () => { const items: unknown[] = [shared, shared]; items.push(items); - expect(stableStringify(items)).toBe('[{"value":"same"},{"value":"same"},"[Circular]"]'); + expect(serialize(items)).toBe('[{"value":"same"},{"value":"same"},"[Circular]"]'); }); it("opts into string normalization without changing the lossless default", () => { @@ -52,8 +60,8 @@ describe("stableStringify", () => { valid: "emoji 🙈 ok", }; - expect(stableStringify(value)).toContain("\\ud83d"); - expect(stableStringify(value, sanitizeSurrogates)).toBe( + expect(serialize(value)).toContain("\\ud83d"); + expect(serialize(value, sanitizeSurrogates)).toBe( '{"high":"leftright","key":"name","low":"leftright","valid":"emoji 🙈 ok"}', ); }); @@ -63,10 +71,10 @@ describe("stableStringify", () => { const malformed = { ba: 2, [`b${high}`]: 1 }; const normalized = { ba: 2, b: 1 }; - expect(stableStringify(malformed, sanitizeSurrogates)).toBe( - stableStringify(normalized, sanitizeSurrogates), + expect(serialize(malformed, sanitizeSurrogates)).toBe( + serialize(normalized, sanitizeSurrogates), ); - expect(stableStringify(malformed, sanitizeSurrogates)).toBe('{"b":1,"ba":2}'); + expect(serialize(malformed, sanitizeSurrogates)).toBe('{"b":1,"ba":2}'); }); it("serializes cache-trace edge types deterministically", () => { @@ -74,7 +82,7 @@ describe("stableStringify", () => { error.stack = "Error: boom\n at test"; expect( - stableStringify({ + serialize({ bytes: new Uint8Array([1, 2, 3]), error, finite: 1, @@ -88,4 +96,43 @@ describe("stableStringify", () => { '{"bytes":{"data":"AQID","type":"Uint8Array"},"error":{"message":"boom","name":"Error","stack":"Error: boom\\n at test"},"finite":1,"infinity":"Infinity","nan":"NaN","nil":null,"token":"123","undef":undefined}', ); }); + + it("preserves colliding normalized keys and reads getters in deterministic order", () => { + const observations: string[] = []; + const value = { + get b() { + observations.push("get:b"); + return "lower"; + }, + get B() { + observations.push("get:B"); + return "upper"; + }, + }; + expect( + serialize(value, (text) => { + observations.push(`normalize:${text}`); + return text.toLowerCase(); + }), + ).toBe('{"b":"upper","b":"lower"}'); + expect(observations).toEqual([ + "normalize:b", + "normalize:B", + "get:B", + "normalize:upper", + "get:b", + "normalize:lower", + ]); + }); +}); + +it.each([1, 20_000])("hashes and counts complete Unicode text with %i repetitions", (count) => { + const text = "🦞日本語\ud800".repeat(count); + const value = { z: [text, undefined], a: text }; + const quoted = JSON.stringify(text); + const expected = `{"a":${quoted},"z":[${quoted},undefined]}`; + expect(sha256StableValue(value)).toEqual({ + digest: sha256Hex(expected), + byteWeight: Buffer.byteLength(expected), + }); }); diff --git a/packages/normalization-core/src/stable-stringify.ts b/packages/normalization-core/src/stable-stringify.ts index e428cb973989..faf32d467721 100644 --- a/packages/normalization-core/src/stable-stringify.ts +++ b/packages/normalization-core/src/stable-stringify.ts @@ -5,6 +5,7 @@ */ type StableStringNormalizer = (value: string) => string; +type StableStringWriter = (chunk: string) => void; const preserveString = (value: string) => value; @@ -16,10 +17,20 @@ export function stableStringify( return stringifyStableValue(value, new WeakSet(), normalizeString); } +/** Writes the same deterministic text without retaining completed container strings. */ +export function writeStableStringify( + value: unknown, + write: StableStringWriter, + normalizeString: StableStringNormalizer = preserveString, +): void { + write(stringifyStableValue(value, new WeakSet(), normalizeString, write)); +} + function stringifyStableValue( value: unknown, stack: WeakSet, normalizeString: StableStringNormalizer, + write?: StableStringWriter, ): string { if (value === null || value === undefined) { return String(value); @@ -42,7 +53,7 @@ function stringifyStableValue( stack.add(value); try { - return stringifyObjectValue(value, stack, normalizeString); + return stringifyObjectValue(value, stack, normalizeString, write); } finally { stack.delete(value); } @@ -52,6 +63,7 @@ function stringifyObjectValue( value: object, stack: WeakSet, normalizeString: StableStringNormalizer, + write?: StableStringWriter, ): string { if (value instanceof Error) { return stringifyStableValue( @@ -62,6 +74,7 @@ function stringifyObjectValue( }, stack, normalizeString, + write, ); } if (value instanceof Uint8Array) { @@ -72,9 +85,21 @@ function stringifyObjectValue( }, stack, normalizeString, + write, ); } if (Array.isArray(value)) { + if (write) { + write("["); + let separator = ""; + for (const entry of value) { + write(separator); + write(stringifyStableValue(entry, stack, normalizeString, write)); + separator = ","; + } + write("]"); + return ""; + } const serializedEntries: string[] = []; for (const entry of value) { serializedEntries.push(stringifyStableValue(entry, stack, normalizeString)); @@ -85,6 +110,17 @@ function stringifyObjectValue( if (normalizeString === preserveString) { // oxlint-disable-next-line unicorn/no-array-sort -- Object.keys creates a private array. const fields = Object.keys(record).sort(); + if (write) { + write("{"); + let separator = ""; + for (const key of fields) { + write(`${separator}${JSON.stringify(key)}:`); + write(stringifyStableValue(record[key], stack, normalizeString, write)); + separator = ","; + } + write("}"); + return ""; + } let fieldIndex = 0; for (const key of fields) { fields[fieldIndex++] = @@ -101,6 +137,17 @@ function stringifyObjectValue( return normalizedOrder || compareStableStrings(left.key, right.key); }); const serializedFields: string[] = []; + if (write) { + write("{"); + let separator = ""; + for (const { key, normalizedKey } of entries) { + write(`${separator}${JSON.stringify(normalizedKey)}:`); + write(stringifyStableValue(record[key], stack, normalizeString, write)); + separator = ","; + } + write("}"); + return ""; + } for (const { key, normalizedKey } of entries) { serializedFields.push( `${JSON.stringify(normalizedKey)}:${stringifyStableValue(record[key], stack, normalizeString)}`, diff --git a/src/agents/embedded-agent-runner/provider-prompt-state.test.ts b/src/agents/embedded-agent-runner/provider-prompt-state.test.ts index 20291b195310..0a16e9eab655 100644 --- a/src/agents/embedded-agent-runner/provider-prompt-state.test.ts +++ b/src/agents/embedded-agent-runner/provider-prompt-state.test.ts @@ -111,7 +111,7 @@ describe("provider prompt state", () => { sentPayloads.push(replacement === undefined ? rawPayload : replacement); return createResultStream("error"); }); - const finalPayload = { input: "final", model: model.id }; + const finalPayload = { input: "final 🦞 日本語".repeat(3_000), model: model.id }; const wrapped = wrapStreamFnWithProviderPromptState({ streamFn: transport, state, diff --git a/src/agents/embedded-agent-runner/provider-prompt-state.ts b/src/agents/embedded-agent-runner/provider-prompt-state.ts index 67258fc7753d..98013cadd063 100644 --- a/src/agents/embedded-agent-runner/provider-prompt-state.ts +++ b/src/agents/embedded-agent-runner/provider-prompt-state.ts @@ -1,7 +1,6 @@ -import { Buffer } from "node:buffer"; import { responsesPromptObserver } from "@openclaw/ai/internal/openai"; import { stableStringify } from "@openclaw/normalization-core"; -import { sha256Hex } from "@openclaw/normalization-core/node-crypto"; +import { sha256Hex, sha256StableValue } from "@openclaw/normalization-core/node-crypto"; import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; import type { Model } from "openclaw/plugin-sdk/llm"; import { resolveGlobalSingleton } from "../../shared/global-singleton.js"; @@ -47,11 +46,11 @@ function snapshotProviderPrompt(params: { baseUrl: params.model.baseUrl, effectiveContextTokenBudget: params.effectiveContextTokenBudget, }); - const serialized = stableStringify(params.payload); + const payload = sha256StableValue(params.payload); return { scopeDigest: sha256Hex(scope), - digest: sha256Hex(serialized), - byteWeight: Buffer.byteLength(serialized), + digest: payload.digest, + byteWeight: payload.byteWeight, }; }