From d6ec1c6ceab37950ad3265f529b7aa2e03aa5e0a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 9 Jul 2026 05:50:28 +0100 Subject: [PATCH] fix(secrets): prevent capture and sentinel credential retention (#102420) * fix(secrets): harden sentinel capture lifecycle * chore: leave release notes to release automation --- packages/ai/src/providers/anthropic.test.ts | 2 +- .../src/providers/google-client-auth.test.ts | 2 +- .../openai-chatgpt-responses.test.ts | 2 +- .../ai/src/providers/openai-responses.test.ts | 6 +- src/agents/provider-local-service.test.ts | 2 +- src/agents/provider-secret-egress.test.ts | 2 +- src/agents/provider-transport-fetch.test.ts | 2 +- src/agents/provider-transport-fetch.ts | 7 +- src/logging/redact.test.ts | 10 ++ src/logging/secret-redaction-registry.ts | 14 +- src/media-understanding/image.test.ts | 4 +- src/proxy-capture/runtime.test.ts | 133 +++++++++++++++++- src/proxy-capture/runtime.ts | 117 ++++++++++----- src/secrets/sentinel.test.ts | 23 ++- src/secrets/sentinel.ts | 88 ++++++++---- 15 files changed, 330 insertions(+), 84 deletions(-) diff --git a/packages/ai/src/providers/anthropic.test.ts b/packages/ai/src/providers/anthropic.test.ts index 1c0f63776b7..f9451763c39 100644 --- a/packages/ai/src/providers/anthropic.test.ts +++ b/packages/ai/src/providers/anthropic.test.ts @@ -152,7 +152,7 @@ describe("Anthropic provider", () => { }); it("keeps sentinel-backed Foundry Authorization headers on bearer routing", async () => { - const sentinel = "oc-sent-v1-0123456789abcdef01234567"; + const sentinel = "oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end"; configureAiTransportHost({ buildModelFetch: () => async () => new Response(null, { status: 500 }), resolveSecretSentinel: (value) => value.replaceAll(sentinel, "Bearer entra-access-token"), diff --git a/packages/ai/src/providers/google-client-auth.test.ts b/packages/ai/src/providers/google-client-auth.test.ts index 31eb8c43820..a376c7fdbff 100644 --- a/packages/ai/src/providers/google-client-auth.test.ts +++ b/packages/ai/src/providers/google-client-auth.test.ts @@ -32,7 +32,7 @@ import { streamGoogle } from "./google.js"; const context = { messages: [{ role: "user", content: "hello", timestamp: 0 }], } satisfies Context; -const sentinel = "oc-sent-v1-0123456789abcdef01234567"; +const sentinel = "oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end"; function googleModel(): Model<"google-generative-ai"> { return { diff --git a/packages/ai/src/providers/openai-chatgpt-responses.test.ts b/packages/ai/src/providers/openai-chatgpt-responses.test.ts index c0fc9fd1f71..2c9fe444232 100644 --- a/packages/ai/src/providers/openai-chatgpt-responses.test.ts +++ b/packages/ai/src/providers/openai-chatgpt-responses.test.ts @@ -129,7 +129,7 @@ describe("streamOpenAICodexResponses transport", () => { const realToken = createJwt({ "https://api.openai.com/auth": { chatgpt_account_id: "acct-sentinel" }, }); - const sentinel = "oc-sent-v1-0123456789abcdef01234567"; + const sentinel = "oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end"; configureAiTransportHost({ resolveSecretSentinel: (value) => value.replaceAll(sentinel, realToken), }); diff --git a/packages/ai/src/providers/openai-responses.test.ts b/packages/ai/src/providers/openai-responses.test.ts index 92f6b235f4b..620565adff5 100644 --- a/packages/ai/src/providers/openai-responses.test.ts +++ b/packages/ai/src/providers/openai-responses.test.ts @@ -69,7 +69,7 @@ describe("OpenAI Responses provider", () => { baseUrl: "https://gateway.ai.cloudflare.com/v1/account/gateway/openai", }), context, - { apiKey: "oc-sent-v1-0123456789abcdef01234567" }, + { apiKey: "oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end" }, ).result(); const config = openAiMockState.configs[0] as { @@ -77,9 +77,9 @@ describe("OpenAI Responses provider", () => { defaultHeaders?: Record; fetch?: unknown; }; - expect(config.apiKey).toBe("oc-sent-v1-0123456789abcdef01234567"); + expect(config.apiKey).toBe("oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end"); expect(config.defaultHeaders?.["cf-aig-authorization"]).toBe( - "Bearer oc-sent-v1-0123456789abcdef01234567", + "Bearer oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end", ); expect(config.fetch).toBe(hostFetch); }); diff --git a/src/agents/provider-local-service.test.ts b/src/agents/provider-local-service.test.ts index dcd9c7accdf..c02f534ed60 100644 --- a/src/agents/provider-local-service.test.ts +++ b/src/agents/provider-local-service.test.ts @@ -191,7 +191,7 @@ describe("provider local service", () => { it("rejects unknown sentinels before starting a local service", async () => { const port = await freePort(); - const unknown = "oc-sent-v1-fedcba987654321001234567"; + const unknown = "oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end"; const model = attachModelProviderLocalService( { id: "demo", diff --git a/src/agents/provider-secret-egress.test.ts b/src/agents/provider-secret-egress.test.ts index 598fb430245..36add06902e 100644 --- a/src/agents/provider-secret-egress.test.ts +++ b/src/agents/provider-secret-egress.test.ts @@ -79,7 +79,7 @@ describe("unwrapModelHeaderSentinelsForProviderEgress", () => { { auth: { mode: "authorization-bearer", - token: "oc-sent-v1-00000000000000000000dead", + token: "oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end", }, }, ); diff --git a/src/agents/provider-transport-fetch.test.ts b/src/agents/provider-transport-fetch.test.ts index 2126936ab67..7a7950a9a0c 100644 --- a/src/agents/provider-transport-fetch.test.ts +++ b/src/agents/provider-transport-fetch.test.ts @@ -261,7 +261,7 @@ describe("buildGuardedModelFetch", () => { }); it("rejects unknown sentinel-shaped values before guarded fetch", async () => { - const unknown = "oc-sent-v1-fedcba987654321001234567"; + const unknown = "oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end"; await expect( buildGuardedModelFetch(sentinelModel())("https://api.openai.com/v1/responses", { headers: { Authorization: `Bearer ${unknown}` }, diff --git a/src/agents/provider-transport-fetch.ts b/src/agents/provider-transport-fetch.ts index 1fd85ef8ef1..8a666afbfb7 100644 --- a/src/agents/provider-transport-fetch.ts +++ b/src/agents/provider-transport-fetch.ts @@ -30,6 +30,7 @@ import type { Model } from "../llm/types.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { resolveDebugProxySettings } from "../proxy-capture/env.js"; import { + containsSecretSentinel, resolveSecretSentinel, SECRET_SENTINEL_PATTERN, swapSecretSentinelsInText, @@ -770,7 +771,7 @@ function headersContainSecretSentinel(headers: HeadersInit | undefined): boolean return false; } for (const value of new Headers(headers).values()) { - if (value.includes("oc-sent-v1-")) { + if (containsSecretSentinel(value)) { return true; } } @@ -778,7 +779,7 @@ function headersContainSecretSentinel(headers: HeadersInit | undefined): boolean } function swapSecretSentinelsInUrl(url: string): { text: string; unknown: string[] } { - if (!url.includes("oc-sent-v1-")) { + if (!containsSecretSentinel(url)) { return { text: url, unknown: [] }; } const unknown = new Set(); @@ -798,7 +799,7 @@ function swapSecretSentinelsForEgress(params: { url: string; headers?: HeadersIn url: string; headers?: Headers; } { - if (!params.url.includes("oc-sent-v1-") && !headersContainSecretSentinel(params.headers)) { + if (!containsSecretSentinel(params.url) && !headersContainSecretSentinel(params.headers)) { return { url: params.url }; } const urlSwap = swapSecretSentinelsInUrl(params.url); diff --git a/src/logging/redact.test.ts b/src/logging/redact.test.ts index 240bce658d3..456025fc464 100644 --- a/src/logging/redact.test.ts +++ b/src/logging/redact.test.ts @@ -73,6 +73,16 @@ describe("registered exact secret values", () => { expect(redactSensitiveText(`raw ${secret}`, { mode: "off" })).not.toContain(secret); }); + it("masks JSON-escaped registered values", () => { + const secret = 'quoted-"secret\\line\nvalue'; + registerSecretValueForRedaction(secret); + + const json = JSON.stringify({ credential: secret }); + expect(redactSensitiveText(json, { mode: "off" })).not.toContain( + JSON.stringify(secret).slice(1, -1), + ); + }); + it("evicts the oldest value after 512 registrations", () => { const first = "exact-registry-value-000"; registerSecretValueForRedaction(first); diff --git a/src/logging/secret-redaction-registry.ts b/src/logging/secret-redaction-registry.ts index 6611caad838..8aff64adb56 100644 --- a/src/logging/secret-redaction-registry.ts +++ b/src/logging/secret-redaction-registry.ts @@ -34,12 +34,20 @@ export function registerSecretValueForRedaction(value: string): void { if (value.length < MIN_SECRET_VALUE_LENGTH) { return; } - registerOneSecretValue(value); // URL egress percent-encodes injected values; redact that surface form too. const encoded = encodeURIComponent(value); if (encoded !== value) { registerOneSecretValue(encoded); } + // Captured structured payloads are serialized before persistence, so retain + // the JSON string-content form for credentials with escaped characters. + const jsonEscaped = JSON.stringify(value).slice(1, -1); + if (jsonEscaped !== value) { + registerOneSecretValue(jsonEscaped); + } + // Keep the raw value newest so bounded-registry eviction cannot drop the + // active credential while retaining only a transformed representation. + registerOneSecretValue(value); } /** Returns whether a value has SecretRef provenance in the process registry. */ @@ -47,6 +55,10 @@ export function isSecretValueRegisteredForRedaction(value: string): boolean { return registeredValues.has(value); } +export function hasRegisteredSecretValuesForRedaction(): boolean { + return registeredValues.size > 0; +} + /** Replaces registered exact values while preserving the caller's mask convention. */ export function redactRegisteredSecretValues( text: string, diff --git a/src/media-understanding/image.test.ts b/src/media-understanding/image.test.ts index 885bf3557bf..09d91689497 100644 --- a/src/media-understanding/image.test.ts +++ b/src/media-understanding/image.test.ts @@ -265,7 +265,7 @@ describe("describeImageWithModel", () => { it("unwraps a sentinel only at the direct MiniMax VLM handoff", async () => { getApiKeyForModelMock.mockResolvedValueOnce({ - apiKey: "oc-sent-v1-0123456789abcdef01234567", + apiKey: "oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end", source: "test", mode: "api-key", }); @@ -283,7 +283,7 @@ describe("describeImageWithModel", () => { }); expect(unwrapSecretSentinelsForProviderEgressMock).toHaveBeenCalledWith( - "oc-sent-v1-0123456789abcdef01234567", + "oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end", "MiniMax VLM request", ); const [, fetchOptionsValue] = requireFirstMockCall(fetchMock, "fetch"); diff --git a/src/proxy-capture/runtime.test.ts b/src/proxy-capture/runtime.test.ts index f4ed59385b3..ad90a45308a 100644 --- a/src/proxy-capture/runtime.test.ts +++ b/src/proxy-capture/runtime.test.ts @@ -7,6 +7,7 @@ import { import type { DebugProxySettings } from "./env.js"; import { captureHttpExchange, + captureWsEvent, finalizeDebugProxyCapture, initializeDebugProxyCapture, type DebugProxyCaptureRuntimeDeps, @@ -54,7 +55,11 @@ const deps: DebugProxyCaptureRuntimeDeps = { payload: { data?: Buffer | string | null; contentType?: string }, ) => ({ contentType: payload.contentType, - ...(typeof payload.data === "string" ? { dataText: payload.data } : {}), + ...(typeof payload.data === "string" + ? { dataText: payload.data } + : Buffer.isBuffer(payload.data) + ? { dataText: payload.data.toString("utf8") } + : {}), }), safeJsonString: (value: unknown) => (value == null ? undefined : JSON.stringify(value)), }; @@ -227,6 +232,107 @@ describe("debug proxy runtime", () => { }); }); + it("redacts registered values from every persisted WebSocket field", () => { + const secret = 'mattermost-"capture\\secret\nline'; + registerSecretValueForRedaction(secret); + + captureWsEvent( + { + url: `wss://chat.example.test/api/v4/websocket?token=${encodeURIComponent(secret)}`, + direction: "outbound", + kind: "ws-frame", + flowId: "mattermost-auth", + payload: JSON.stringify({ action: "authentication_challenge", data: { token: secret } }), + errorText: `failed with ${secret}`, + meta: { subsystem: "mattermost-websocket", detail: secret }, + }, + settings, + deps, + ); + captureWsEvent( + { + url: "wss://chat.example.test/api/v4/websocket", + direction: "inbound", + kind: "ws-frame", + flowId: "mattermost-auth", + payload: Buffer.from(JSON.stringify({ echoedToken: secret })), + }, + settings, + deps, + ); + + const [outbound, inbound] = events; + expect(outbound?.path).toBe("/api/v4/websocket?token=%5BREDACTED%5D"); + expect(outbound?.dataText).toContain('"token":"[REDACTED]"'); + expect(outbound?.errorText).toBe("failed with [REDACTED]"); + expect(JSON.parse(String(outbound?.metaJson))).toStrictEqual({ + subsystem: "mattermost-websocket", + detail: "[REDACTED]", + }); + expect(inbound?.dataText).toContain('"echoedToken":"[REDACTED]"'); + expect(JSON.stringify(events)).not.toContain(secret); + expect(JSON.stringify(events)).not.toContain(JSON.stringify(secret).slice(1, -1)); + }); + + it("redacts registered credential bytes from otherwise non-UTF-8 frames", () => { + const secret = "binary-frame-capture-secret"; + registerSecretValueForRedaction(secret); + const payload = Buffer.concat([ + Buffer.from([0xff, 0x00]), + Buffer.from(secret, "utf8"), + Buffer.from([0xfe]), + ]); + + captureWsEvent( + { + url: "wss://chat.example.test/api/v4/websocket", + direction: "outbound", + kind: "ws-frame", + flowId: "binary-auth", + payload, + }, + settings, + deps, + ); + + expect(events[0]?.dataText).toBe("[REDACTED BINARY PAYLOAD]"); + expect(events[0]?.dataText).not.toContain(secret); + }); + + it("redacts registered values from HTTP payloads and metadata", async () => { + const secret = 'http-"capture\\secret\nline'; + const contentTypeSecret = "http-content-type-secret"; + registerSecretValueForRedaction(secret); + registerSecretValueForRedaction(contentTypeSecret); + + captureHttpExchange( + { + url: "https://api.example.test/v1/messages", + method: "POST", + requestHeaders: { "content-type": `application/json; token=${contentTypeSecret}` }, + requestBody: JSON.stringify({ credential: secret }), + response: new Response(JSON.stringify({ echoedCredential: secret }), { + status: 200, + headers: { "content-type": `application/json; token=${contentTypeSecret}` }, + }), + meta: { credential: secret }, + }, + settings, + deps, + ); + await waitForResponseSettled(); + + const request = events.find((event) => event.kind === "request"); + const response = events.find((event) => event.kind === "response"); + expect(request?.dataText).toContain('"credential":"[REDACTED]"'); + expect(request?.metaJson).toContain('"credential":"[REDACTED]"'); + expect(request?.contentType).toBe("application/json; token=[REDACTED]"); + expect(response?.dataText).toContain('"echoedCredential":"[REDACTED]"'); + expect(response?.metaJson).toContain('"credential":"[REDACTED]"'); + expect(response?.contentType).toBe("application/json; token=[REDACTED]"); + expect(JSON.stringify(events)).not.toContain(secret); + }); + it("redacts registered values from failed global-fetch capture events", async () => { const secret = "capture-failure/secret"; registerSecretValueForRedaction(secret); @@ -376,7 +482,9 @@ describe("debug proxy runtime", () => { // Some seams hand capture a Response-like object that cannot be cloned. It // must still be observable (status/headers) via the shared metadata path, // tagged bodyCapture: "unavailable" (distinct from the "too-large" cap path). - const headers = new Headers({ "content-type": "application/json" }); + const secret = "metadata-only-content-type-secret"; + registerSecretValueForRedaction(secret); + const headers = new Headers({ "content-type": `application/json; token=${secret}` }); captureHttpExchange( { url: "https://api.openai.com/v1/uncloneable", @@ -392,8 +500,29 @@ describe("debug proxy runtime", () => { const response = events.find((event) => event.kind === "response"); expect(response).toBeDefined(); expect(response?.status).toBe(503); + expect(response?.contentType).toBe("application/json; token=[REDACTED]"); expect(JSON.parse(String(response?.metaJson))).toMatchObject({ bodyCapture: "unavailable" }); expect(response).not.toHaveProperty("dataText"); expect(events.some((event) => event.kind === "error")).toBe(false); }); + + it("records Response-like status metadata when the Headers API is absent", async () => { + initializeDebugProxyCapture("test", settings, deps); + captureHttpExchange( + { + url: "https://api.openai.com/v1/no-headers-api", + method: "GET", + response: { status: 204 } as unknown as Response, + }, + settings, + deps, + ); + await waitForResponseSettled(); + finalizeDebugProxyCapture(settings, deps); + + const response = events.find((event) => event.kind === "response"); + expect(response?.status).toBe(204); + expect(response?.contentType).toBeUndefined(); + expect(JSON.parse(String(response?.metaJson))).toMatchObject({ bodyCapture: "unavailable" }); + }); }); diff --git a/src/proxy-capture/runtime.ts b/src/proxy-capture/runtime.ts index e4fb9083af0..13cfc26de72 100644 --- a/src/proxy-capture/runtime.ts +++ b/src/proxy-capture/runtime.ts @@ -1,8 +1,12 @@ // Proxy capture runtime coordinates capture sessions, proxy startup, and storage. +import { isUtf8 } from "node:buffer"; import { randomUUID } from "node:crypto"; import { URL } from "node:url"; import { normalizeRequestInitHeadersForFetch } from "../infra/fetch-headers.js"; -import { redactRegisteredSecretValues } from "../logging/secret-redaction-registry.js"; +import { + hasRegisteredSecretValuesForRedaction, + redactRegisteredSecretValues, +} from "../logging/secret-redaction-registry.js"; import { resolveDebugProxySettings, type DebugProxySettings } from "./env.js"; import { closeDebugProxyCaptureStore, @@ -19,6 +23,7 @@ import type { const DEBUG_PROXY_FETCH_PATCH_KEY = Symbol.for("openclaw.debugProxy.fetchPatch"); const REDACTED_CAPTURE_HEADER_VALUE = "[REDACTED]"; +const REDACTED_CAPTURE_BINARY_PAYLOAD = Buffer.from("[REDACTED BINARY PAYLOAD]", "utf8"); // Cap captured response bodies so debug proxy capture cannot be turned into an // out-of-memory vector. The patched global fetch tees every outbound response // through clone(), so a single large (or hostile, effectively endless) provider @@ -271,6 +276,31 @@ function redactCaptureText(value: string): string { return redactRegisteredSecretValues(value, () => REDACTED_CAPTURE_HEADER_VALUE); } +function redactCapturePayload(value: string | Buffer | null | undefined): string | Buffer | null { + if (typeof value === "string") { + return redactCaptureText(value); + } + if (!Buffer.isBuffer(value)) { + return value ?? null; + } + if (!isUtf8(value)) { + // Binary frames can mix arbitrary bytes with credential text. Once any + // resolved secret exists, omit their contents instead of guessing safely. + return hasRegisteredSecretValuesForRedaction() ? REDACTED_CAPTURE_BINARY_PAYLOAD : value; + } + const text = value.toString("utf8"); + const redacted = redactCaptureText(text); + return redacted === text ? value : Buffer.from(redacted, "utf8"); +} + +function redactedCaptureJson( + value: unknown, + stringify: typeof safeJsonString = safeJsonString, +): string | undefined { + const serialized = stringify(value); + return serialized === undefined ? undefined : redactCaptureText(serialized); +} + function createHttpCaptureEventBase(params: { settings: DebugProxySettings; rawUrl: string; @@ -374,7 +404,7 @@ function installDebugProxyGlobalFetchPatch( host: parsed.host, path: `${parsed.pathname}${parsed.search}`, errorText: redactCaptureText(error instanceof Error ? error.message : String(error)), - metaJson: runtime.safeJsonString({ captureOrigin: "global-fetch" }), + metaJson: redactedCaptureJson({ captureOrigin: "global-fetch" }, runtime.safeJsonString), }); } throw error; @@ -465,12 +495,21 @@ export function captureHttpExchange( typeof params.requestBody === "string" || Buffer.isBuffer(params.requestBody) ? params.requestBody : null; + const rawRequestContentType = + params.requestHeaders instanceof Headers + ? (params.requestHeaders.get("content-type") ?? undefined) + : params.requestHeaders?.["content-type"]; + const requestContentType = + rawRequestContentType === undefined ? undefined : redactCaptureText(rawRequestContentType); + const rawResponseContentType = + typeof params.response.headers?.get === "function" + ? (params.response.headers.get("content-type") ?? undefined) + : undefined; + const responseContentType = + rawResponseContentType === undefined ? undefined : redactCaptureText(rawResponseContentType); const requestPayload = runtime.persistEventPayload(store, { - data: requestBody, - contentType: - params.requestHeaders instanceof Headers - ? (params.requestHeaders.get("content-type") ?? undefined) - : params.requestHeaders?.["content-type"], + data: redactCapturePayload(requestBody), + contentType: requestContentType, }); store.recordEvent({ ...createHttpCaptureEventBase({ @@ -483,12 +522,9 @@ export function captureHttpExchange( flowId, method: params.method, }), - contentType: - params.requestHeaders instanceof Headers - ? (params.requestHeaders.get("content-type") ?? undefined) - : params.requestHeaders?.["content-type"], + contentType: requestContentType, headersJson: runtime.safeJsonString(redactedCaptureHeaders(params.requestHeaders)), - metaJson: runtime.safeJsonString(params.meta), + metaJson: redactedCaptureJson(params.meta, runtime.safeJsonString), ...requestPayload, }); // Records the response status/headers without a body. Used both when a @@ -507,15 +543,12 @@ export function captureHttpExchange( method: params.method, }), status: params.response.status, - contentType: - typeof params.response.headers?.get === "function" - ? (params.response.headers.get("content-type") ?? undefined) - : undefined, + contentType: responseContentType, headersJson: params.response.headers && typeof params.response.headers.entries === "function" ? runtime.safeJsonString(redactedCaptureHeaders(params.response.headers)) : undefined, - metaJson: runtime.safeJsonString({ ...params.meta, bodyCapture }), + metaJson: redactedCaptureJson({ ...params.meta, bodyCapture }, runtime.safeJsonString), }); }; const cloneable = @@ -550,8 +583,8 @@ export function captureHttpExchange( return; } const responsePayload = runtime.persistEventPayload(store, { - data: buffer, - contentType: params.response.headers.get("content-type") ?? undefined, + data: redactCapturePayload(buffer), + contentType: responseContentType, }); store.recordEvent({ ...createHttpCaptureEventBase({ @@ -565,9 +598,9 @@ export function captureHttpExchange( method: params.method, }), status: params.response.status, - contentType: params.response.headers.get("content-type") ?? undefined, + contentType: responseContentType, headersJson: runtime.safeJsonString(redactedCaptureHeaders(params.response.headers)), - metaJson: runtime.safeJsonString(params.meta), + metaJson: redactedCaptureJson(params.meta, runtime.safeJsonString), ...responsePayload, }); }) @@ -590,24 +623,30 @@ export function captureHttpExchange( // Websocket seams call this directly because Node fetch patching cannot observe // frame traffic. -export function captureWsEvent(params: { - url: string; - direction: "outbound" | "inbound" | "local"; - kind: "ws-open" | "ws-frame" | "ws-close" | "error"; - flowId: string; - payload?: string | Buffer; - closeCode?: number; - errorText?: string; - meta?: Record; -}): void { - const settings = resolveDebugProxySettings(); +export function captureWsEvent( + params: { + url: string; + direction: "outbound" | "inbound" | "local"; + kind: "ws-open" | "ws-frame" | "ws-close" | "error"; + flowId: string; + payload?: string | Buffer; + closeCode?: number; + errorText?: string; + meta?: Record; + }, + resolved?: DebugProxySettings, + deps: DebugProxyCaptureRuntimeDeps = {}, +): void { + const settings = resolved ?? resolveDebugProxySettings(); if (!settings.enabled) { return; } - const store = getDebugProxyCaptureStore(); - const url = new URL(params.url); - const payload = persistEventPayload(store, { - data: params.payload, + const runtime = resolveRuntimeDeps(deps); + const store = runtime.getStore(); + const captureUrl = redactCaptureUrl(params.url); + const url = new URL(captureUrl); + const payload = runtime.persistEventPayload(store, { + data: redactCapturePayload(params.payload), contentType: "application/json", }); store.recordEvent({ @@ -615,15 +654,15 @@ export function captureWsEvent(params: { ts: Date.now(), sourceScope: "openclaw", sourceProcess: settings.sourceProcess, - protocol: protocolFromUrl(params.url), + protocol: protocolFromUrl(captureUrl), direction: params.direction, kind: params.kind, flowId: params.flowId, host: url.host, path: `${url.pathname}${url.search}`, closeCode: params.closeCode, - errorText: params.errorText, - metaJson: safeJsonString(params.meta), + errorText: params.errorText === undefined ? undefined : redactCaptureText(params.errorText), + metaJson: redactedCaptureJson(params.meta, runtime.safeJsonString), ...payload, }); } diff --git a/src/secrets/sentinel.test.ts b/src/secrets/sentinel.test.ts index 2c7b21284f4..20d4687ae77 100644 --- a/src/secrets/sentinel.test.ts +++ b/src/secrets/sentinel.test.ts @@ -15,15 +15,16 @@ describe("secret sentinels", () => { resetSecretRedactionRegistryForTest(); }); - it("mints, recognizes, resolves, and reuses sentinels by value and label", () => { + it("mints, recognizes, and resolves authenticated process-local sentinels", () => { const first = mintSecretSentinel("provider-secret-value", { label: "model-auth:openai" }); const repeated = mintSecretSentinel("provider-secret-value", { label: "model-auth:openai" }); const otherLabel = mintSecretSentinel("provider-secret-value", { label: "model-auth:other" }); - expect(first).toMatch(/^oc-sent-v1-[0-9a-f]{24}$/); + expect(first).toMatch(/^oc-sent-v2\.[A-Za-z0-9_-]+\.end$/); expect(first.match(SECRET_SENTINEL_PATTERN)).toEqual([first]); expect(looksLikeSecretSentinel(first)).toBe(true); expect(resolveSecretSentinel(first)).toBe("provider-secret-value"); + expect(resolveSecretSentinel(repeated)).toBe("provider-secret-value"); expect(repeated).toBe(first); expect(otherLabel).not.toBe(first); }); @@ -41,15 +42,29 @@ describe("secret sentinels", () => { }); it("reports unknown sentinel-shaped values without replacing them", () => { - const unknown = "oc-sent-v1-0123456789abcdef01234567"; + const unknown = "oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end"; expect(swapSecretSentinelsInText(`Bearer ${unknown}`)).toEqual({ text: `Bearer ${unknown}`, unknown: [unknown], }); }); + it("rejects tampered sentinel ciphertext", () => { + const sentinel = mintSecretSentinel("tamper-resistant-secret", { label: "model-auth:test" }); + const payloadStart = "oc-sent-v2.".length; + const replacement = sentinel[payloadStart] === "A" ? "B" : "A"; + const tampered = `${sentinel.slice(0, payloadStart)}${replacement}${sentinel.slice(payloadStart + 1)}`; + + expect(looksLikeSecretSentinel(tampered)).toBe(true); + expect(resolveSecretSentinel(tampered)).toBeUndefined(); + expect(swapSecretSentinelsInText(`Bearer ${tampered}`)).toEqual({ + text: `Bearer ${tampered}`, + unknown: [tampered], + }); + }); + it("treats sentinel-shaped bytes inside resolved values as opaque", () => { - const secret = "prefix-oc-sent-v1-0123456789abcdef01234567"; + const secret = "prefix-oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end"; const sentinel = mintSecretSentinel(secret, { label: "nested-shape" }); expect(swapSecretSentinelsInText(`Bearer ${sentinel}`)).toEqual({ diff --git a/src/secrets/sentinel.ts b/src/secrets/sentinel.ts index fc16581d494..504a2a58817 100644 --- a/src/secrets/sentinel.ts +++ b/src/secrets/sentinel.ts @@ -1,13 +1,23 @@ -import { randomBytes } from "node:crypto"; +import { createCipheriv, createDecipheriv, createHash, createHmac, randomBytes } from "node:crypto"; import { registerSecretValueForRedaction } from "../logging/secret-redaction-registry.js"; -const SECRET_SENTINEL_PREFIX = "oc-sent-v1-"; -const SECRET_SENTINEL_SOURCE = `${SECRET_SENTINEL_PREFIX}[0-9a-f]{24}`; +const SECRET_SENTINEL_PREFIX = "oc-sent-v2."; +const SECRET_SENTINEL_SUFFIX = ".end"; +const SECRET_SENTINEL_SOURCE = "oc-sent-v2\\.[A-Za-z0-9_-]+\\.end"; +const SECRET_SENTINEL_CIPHER = "aes-256-gcm"; +const SECRET_SENTINEL_NONCE_BYTES = 12; +const SECRET_SENTINEL_SCOPE_BYTES = 8; +const SECRET_SENTINEL_TAG_BYTES = 16; +const SECRET_SENTINEL_HEADER_BYTES = + SECRET_SENTINEL_SCOPE_BYTES + SECRET_SENTINEL_NONCE_BYTES + SECRET_SENTINEL_TAG_BYTES; export const SECRET_SENTINEL_PATTERN = new RegExp(SECRET_SENTINEL_SOURCE, "g"); -const valuesBySentinel = new Map(); -const sentinelsByValueAndLabel = new Map>(); +// One process key keeps sentinels resolvable for in-flight requests without a +// plaintext registry that retains every historical credential until exit. +const secretSentinelKeys = randomBytes(64); +const secretSentinelCipherKey = secretSentinelKeys.subarray(0, 32); +const secretSentinelNonceKey = secretSentinelKeys.subarray(32); function secretSentinelsEnabled(env: NodeJS.ProcessEnv = process.env): boolean { const configured = env.OPENCLAW_SECRET_SENTINELS?.trim().toLowerCase(); @@ -18,40 +28,70 @@ export function looksLikeSecretSentinel(value: string): boolean { return new RegExp(`^${SECRET_SENTINEL_SOURCE}$`).test(value); } -/** Mints one stable process-local sentinel for a secret value and label. */ +export function containsSecretSentinel(value: string): boolean { + return value.includes(SECRET_SENTINEL_PREFIX); +} + +function secretSentinelScope(label: string): Buffer { + return createHash("sha256").update(label).digest().subarray(0, SECRET_SENTINEL_SCOPE_BYTES); +} + +/** Seals a secret into authenticated ciphertext that only this process can resolve. */ export function mintSecretSentinel(value: string, meta: { label: string }): string { registerSecretValueForRedaction(value); if (!secretSentinelsEnabled()) { return value; } - const byLabel = sentinelsByValueAndLabel.get(value) ?? new Map(); - const existing = byLabel.get(meta.label); - if (existing) { - return existing; - } - let sentinel: string; - do { - sentinel = `${SECRET_SENTINEL_PREFIX}${randomBytes(12).toString("hex")}`; - } while (valuesBySentinel.has(sentinel)); - byLabel.set(meta.label, sentinel); - sentinelsByValueAndLabel.set(value, byLabel); - valuesBySentinel.set(sentinel, value); - return sentinel; + const scope = secretSentinelScope(meta.label); + // A keyed nonce preserves the old stable-by-value-and-label behavior without + // retaining a reverse plaintext map. Different plaintexts collide only at + // the 96-bit HMAC truncation boundary. + const nonce = createHmac("sha256", secretSentinelNonceKey) + .update(scope) + .update(value) + .digest() + .subarray(0, SECRET_SENTINEL_NONCE_BYTES); + const cipher = createCipheriv(SECRET_SENTINEL_CIPHER, secretSentinelCipherKey, nonce); + cipher.setAAD(scope); + const ciphertext = Buffer.concat([cipher.update(value, "utf8"), cipher.final()]); + const sealed = Buffer.concat([scope, nonce, cipher.getAuthTag(), ciphertext]); + return `${SECRET_SENTINEL_PREFIX}${sealed.toString("base64url")}${SECRET_SENTINEL_SUFFIX}`; } -/** Resolves a process-local sentinel without exposing the registry itself. */ +/** Opens a process-local sentinel and rejects malformed or tampered values. */ export function resolveSecretSentinel(sentinel: string): string | undefined { - const value = valuesBySentinel.get(sentinel); - if (value !== undefined) { + if (!looksLikeSecretSentinel(sentinel)) { + return undefined; + } + try { + const encoded = sentinel.slice(SECRET_SENTINEL_PREFIX.length, -SECRET_SENTINEL_SUFFIX.length); + const sealed = Buffer.from(encoded, "base64url"); + if (sealed.length < SECRET_SENTINEL_HEADER_BYTES) { + return undefined; + } + const scope = sealed.subarray(0, SECRET_SENTINEL_SCOPE_BYTES); + const nonce = sealed.subarray( + SECRET_SENTINEL_SCOPE_BYTES, + SECRET_SENTINEL_SCOPE_BYTES + SECRET_SENTINEL_NONCE_BYTES, + ); + const tagStart = SECRET_SENTINEL_SCOPE_BYTES + SECRET_SENTINEL_NONCE_BYTES; + const tag = sealed.subarray(tagStart, tagStart + SECRET_SENTINEL_TAG_BYTES); + const ciphertext = sealed.subarray(SECRET_SENTINEL_HEADER_BYTES); + const decipher = createDecipheriv(SECRET_SENTINEL_CIPHER, secretSentinelCipherKey, nonce); + decipher.setAAD(scope); + decipher.setAuthTag(tag); + const value = Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString("utf8"); // Refresh the bounded redaction registry whenever a live sentinel is used. registerSecretValueForRedaction(value); + return value; + } catch { + return undefined; } - return value; } /** Swaps every known sentinel substring and reports unknown sentinel-shaped values. */ export function swapSecretSentinelsInText(text: string): { text: string; unknown: string[] } { - if (!text.includes(SECRET_SENTINEL_PREFIX)) { + if (!containsSecretSentinel(text)) { return { text, unknown: [] }; } const unknown = new Set();