From 3faf669801d0d3597e2b6d51f15452fe91033889 Mon Sep 17 00:00:00 2001 From: Alex Knight Date: Wed, 10 Jun 2026 14:34:50 -0700 Subject: [PATCH] Redact tool output secrets (#85196) * redact tool output secrets * Expand tool-output secret redaction * fix(security): keep redaction prefilter in sync with expanded defaults - build DEFAULT_REDACT_PREFILTER_RE from sources covering every default pattern family: new vendor prefixes, webhook hosts, bare query/form keys, userinfo/connection-string passwords, and percent/plus/invisible obfuscated keys (including trailing separator splices) - run default-pattern redaction tests through the default options path and redact the vendor corpus per token so prefilter gaps fail tests - fix quoted standalone assignment values containing the other quote char or an unterminated quote; never re-mask *** placeholders - align net-policy URL query-name separator stripping with logging key normalization (Hangul fillers) * fix(security): keep base64-prefix redaction out of media payloads - pure-base64-alphabet token prefixes (gAAAA, AKIA, ASIA, dapi, ATCTT3xFfG, ATATT, ATBB) now require a non-alphanumeric left boundary, skip explicit ;base64, payload spans, and run unchunked so chunk starts cannot fake the boundary or hide the container from the lookbehind - tokens after URL/path delimiters or assignments still mask; data-URL media survives redaction byte-identical (fixes chat media mirror CI) - regression tests: tiny-PNG data URL, in-blob plus boundary, chunk-aligned large data URL, reset-path Fernet token, path AWS key --------- Co-authored-by: Alex Knight <15041791+amknight@users.noreply.github.com> --- .../src/redact-sensitive-url.test.ts | 49 ++ .../net-policy/src/redact-sensitive-url.ts | 26 +- .../gateway-cli/run.option-collisions.test.ts | 2 +- .../exec-approval-command-display.test.ts | 50 ++ src/infra/exec-approval-command-display.ts | 52 +- src/logging/redact.test.ts | 774 +++++++++++++---- src/logging/redact.ts | 779 +++++++++++++++++- 7 files changed, 1512 insertions(+), 220 deletions(-) diff --git a/packages/net-policy/src/redact-sensitive-url.test.ts b/packages/net-policy/src/redact-sensitive-url.test.ts index 02e9bce039d..f99d83b7e41 100644 --- a/packages/net-policy/src/redact-sensitive-url.test.ts +++ b/packages/net-policy/src/redact-sensitive-url.test.ts @@ -22,6 +22,24 @@ describe("redactSensitiveUrl", () => { ); }); + it("redacts encoded and invisible-spliced sensitive query param names", () => { + expect( + redactSensitiveUrl("https://example.com/mcp?client%5Fse%E2%80%8Bcret=secret&safe=value"), + ).toBe("https://example.com/mcp?client_se%E2%80%8Bcret=***&safe=value"); + }); + + it("redacts encoded sensitive query names with decoded whitespace and control separators", () => { + expect( + redactSensitiveUrl("https://example.com/mcp?client%5Fse%20cret=space&client%5Fse%00cret=nul"), + ).toBe("https://example.com/mcp?client_se+cret=***&client_se%00cret=***"); + }); + + it("redacts query names with plus-encoded separators", () => { + expect(redactSensitiveUrl("https://example.com/mcp?client_se+cret=secret&safe=value")).toBe( + "https://example.com/mcp?client_se+cret=***&safe=value", + ); + }); + it("keeps non-sensitive URLs unchanged", () => { expect(redactSensitiveUrl("https://example.com/mcp?safe=value")).toBe( "https://example.com/mcp?safe=value", @@ -36,6 +54,26 @@ describe("redactSensitiveUrlLikeString", () => { ); }); + it("redacts encoded and invisible-spliced query names in invalid URL-like strings", () => { + expect( + redactSensitiveUrlLikeString("//example.com/mcp?client%5Fse%E2%80%8Bcret=secret&safe=value"), + ).toBe("//example.com/mcp?client%5Fse%E2%80%8Bcret=***&safe=value"); + }); + + it("redacts encoded query names with decoded whitespace and control separators in invalid URL-like strings", () => { + expect( + redactSensitiveUrlLikeString( + "//example.com/mcp?client%5Fse%20cret=space&client%5Fse%00cret=nul", + ), + ).toBe("//example.com/mcp?client%5Fse%20cret=***&client%5Fse%00cret=***"); + }); + + it("redacts plus-spliced query names in invalid URL-like strings", () => { + expect(redactSensitiveUrlLikeString("//example.com/mcp?client_se+cret=secret&safe=value")).toBe( + "//example.com/mcp?client_se+cret=***&safe=value", + ); + }); + it("redacts every URL-like userinfo occurrence in arbitrary text", () => { expect( redactSensitiveUrlLikeString( @@ -61,6 +99,17 @@ describe("isSensitiveUrlQueryParamName", () => { expect(isSensitiveUrlQueryParamName("hook-token")).toBe(true); expect(isSensitiveUrlQueryParamName("passwd")).toBe(true); expect(isSensitiveUrlQueryParamName("signature")).toBe(true); + expect(isSensitiveUrlQueryParamName("code")).toBe(true); + expect(isSensitiveUrlQueryParamName("x-amz-signature")).toBe(true); + expect(isSensitiveUrlQueryParamName("X-Amz-Security-Token")).toBe(true); + expect(isSensitiveUrlQueryParamName("id_token")).toBe(true); + expect(isSensitiveUrlQueryParamName("app_secret")).toBe(true); + expect(isSensitiveUrlQueryParamName("client%5Fse\u200Bcret")).toBe(true); + expect(isSensitiveUrlQueryParamName("client%5Fse%20cret")).toBe(true); + expect(isSensitiveUrlQueryParamName("client%5Fse%00cret")).toBe(true); + expect(isSensitiveUrlQueryParamName("client_se+cret")).toBe(true); + expect(isSensitiveUrlQueryParamName("client_se\u3164cret")).toBe(true); + expect(isSensitiveUrlQueryParamName("credential")).toBe(true); expect(isSensitiveUrlQueryParamName("safe")).toBe(false); }); }); diff --git a/packages/net-policy/src/redact-sensitive-url.ts b/packages/net-policy/src/redact-sensitive-url.ts index 76d86312aaa..b13e1780617 100644 --- a/packages/net-policy/src/redact-sensitive-url.ts +++ b/packages/net-policy/src/redact-sensitive-url.ts @@ -22,15 +22,39 @@ const SENSITIVE_URL_QUERY_PARAM_NAMES = new Set([ "pass", "passwd", "auth", + "jwt", + "session", + "id_token", + "code", "client_secret", + "app_secret", "hook_token", "refresh_token", "signature", + "x_amz_signature", + "x_amz_security_token", + "private_key", + "credential", + "authorization", ]); +// Keep in sync with FORM_BODY_KEY_SEPARATOR_RE in src/logging/redact.ts: Hangul fillers are +// category Lo, so \p{C}\p{Z} alone would let them splice sensitive key names. +const URL_QUERY_NAME_SEPARATOR_RE = /[\p{C}\p{Z}\u115F\u1160\u3164\uFFA0+]/gu; + +function normalizeUrlQueryParamName(name: string): string { + const stripped = name.replace(URL_QUERY_NAME_SEPARATOR_RE, ""); + try { + return normalizeLowercaseStringOrEmpty( + decodeURIComponent(stripped).replace(URL_QUERY_NAME_SEPARATOR_RE, ""), + ).replaceAll("-", "_"); + } catch { + return normalizeLowercaseStringOrEmpty(stripped).replaceAll("-", "_"); + } +} /** True for auth-like URL query parameter names that should be redacted. */ export function isSensitiveUrlQueryParamName(name: string): boolean { - const normalized = normalizeLowercaseStringOrEmpty(name).replaceAll("-", "_"); + const normalized = normalizeUrlQueryParamName(name); return SENSITIVE_URL_QUERY_PARAM_NAMES.has(normalized); } diff --git a/src/cli/gateway-cli/run.option-collisions.test.ts b/src/cli/gateway-cli/run.option-collisions.test.ts index 36e577861df..1f9488b9501 100644 --- a/src/cli/gateway-cli/run.option-collisions.test.ts +++ b/src/cli/gateway-cli/run.option-collisions.test.ts @@ -648,6 +648,6 @@ describe("gateway run option collisions", () => { ).rejects.toThrow("__exit__:1"); }, ); - expect(runtimeErrors[0]).toContain("Use either --passw***d or --password-file."); + expect(runtimeErrors[0]).toContain("Use either --password or --password-file."); }); }); diff --git a/src/infra/exec-approval-command-display.test.ts b/src/infra/exec-approval-command-display.test.ts index 496f0b754d7..bf99f51a5c1 100644 --- a/src/infra/exec-approval-command-display.test.ts +++ b/src/infra/exec-approval-command-display.test.ts @@ -104,6 +104,21 @@ describe("sanitizeExecApprovalDisplayText", () => { expect(result).toContain("https://api.example.com"); }); + it("masks newly added vendor token prefixes through the default redaction path", () => { + const token = "glpat-abcdefghijklmnopqrstuv"; + const result = sanitizeExecApprovalDisplayText(`deploy --with ${token}`); + expect(result).not.toContain(token); + }); + + it("does not let contextual secret matches hide split-token bypass detection", () => { + const discordToken = `${"A".repeat(24)}.${"B".repeat(6)}.${"C".repeat(27)}`; + const cmd = `discord sk-abc123\u200B456789012345678 ${discordToken}`; + const result = sanitizeExecApprovalDisplayText(cmd); + expect(result).not.toContain("sk-abc123"); + expect(result).not.toContain("456789012345678"); + expect(result).not.toContain(discordToken); + }); + it("keeps PEM private-key context visible when raw redaction already covers the key (not a bypass)", () => { const cmd = "echo -----BEGIN RSA PRIVATE KEY-----\nABCDEF0123456789abcdef\n-----END RSA PRIVATE KEY----- > key.pem"; @@ -157,6 +172,41 @@ describe("sanitizeExecApprovalDisplayText", () => { expect(result).not.toContain("456789012345678"); expect(result).toContain("remainder"); }); + + it("masks form body values whose sensitive key is spliced with an invisible character", () => { + const cmd = "client_id=visible&app_se\u200Bcret=opaque-app-secret&safe=value"; + const result = sanitizeExecApprovalDisplayText(cmd); + expect(result).not.toContain("opaque-app-secret"); + expect(result).toContain("client_id=visible"); + expect(result).toContain("safe=value"); + }); + + it("masks form body values whose encoded sensitive key is spliced with an invisible character", () => { + const cmd = "client_id=visible&client%5Fse\u200Bcret=oauth-secret&safe=value"; + const result = sanitizeExecApprovalDisplayText(cmd); + expect(result).not.toContain("oauth-secret"); + expect(result).toContain("client_id=visible"); + expect(result).toContain("safe=value"); + }); + + it("masks form body values whose sensitive key is spliced with a plus separator", () => { + const cmd = "client_id=visible&client_se+cret=oauth-secret&safe=value"; + const result = sanitizeExecApprovalDisplayText(cmd); + expect(result).not.toContain("oauth-secret"); + expect(result).toContain("client_id=visible"); + expect(result).toContain("safe=value"); + }); + + it("keeps parsed form-body secrets masked when a separate spliced token triggers bypass rendering", () => { + const cmd = + "client_id=visible&client%5Fsecret=oauth,secret&safe=1 echo sk-abc123\u200B456789012345678"; + const result = sanitizeExecApprovalDisplayText(cmd); + expect(result).not.toContain("oauth,secret"); + expect(result).not.toContain(",secret"); + expect(result).not.toContain("456789012345678"); + expect(result).toContain("client_id=visible"); + expect(result).toContain("safe=1"); + }); }); describe("sanitizeExecApprovalWarningText", () => { diff --git a/src/infra/exec-approval-command-display.ts b/src/infra/exec-approval-command-display.ts index 361753ce114..966bce1b3d2 100644 --- a/src/infra/exec-approval-command-display.ts +++ b/src/infra/exec-approval-command-display.ts @@ -1,5 +1,9 @@ // Sanitizes command text before it is displayed in approval prompts. -import { redactSensitiveText, resolveRedactOptions } from "../logging/redact.js"; +import { + computeSensitiveRedactionBitmap, + redactSensitiveText, + resolveRedactOptions, +} from "../logging/redact.js"; import type { ExecApprovalRequestPayload } from "./exec-approvals.js"; // Escape control characters, Unicode format/line/paragraph separators, and non-ASCII space @@ -60,31 +64,6 @@ function truncateForDisplay(text: string): SanitizedExecApprovalDisplayText { }; } -// Build a boolean bitmap of positions in `text` that ANY redaction pattern would match. -// Patterns are applied independently to the raw text (not sequentially against a -// progressively-redacted view) so later patterns can still find matches that the in-place -// redaction would have replaced first. That is conservative — it may over-count overlapping -// matches — but that is acceptable for a coverage check. Indices are UTF-16 code-unit -// offsets, matching what `matchAll` returns and aligning with `String#length`. -function computeRedactionBitmap(text: string, patterns: RegExp[]): boolean[] { - const bitmap: boolean[] = Array.from({ length: text.length }, () => false); - for (const pattern of patterns) { - const iter = pattern.flags.includes("g") - ? new RegExp(pattern.source, pattern.flags) - : new RegExp(pattern.source, `${pattern.flags}g`); - for (const match of text.matchAll(iter)) { - if (match.index === undefined) { - continue; - } - const end = match.index + match[0].length; - for (let i = match.index; i < end; i++) { - bitmap[i] = true; - } - } - } - return bitmap; -} - // Iterate by full Unicode code point so astral-plane invisibles (e.g. U+E0061 TAG LATIN // SMALL LETTER A, category Cf) are matched as single characters instead of being seen as a // surrogate pair whose halves are category Cs and would escape the invisible-char regex. @@ -126,17 +105,16 @@ function sanitizeExecApprovalDisplayTextInternal( if (strippedRedacted === stripped) { return truncateForDisplay(escapeInvisibles(rawRedacted, options)); } - // Detect bypass by position-bitmap coverage. Run each redaction pattern independently on - // both views and map stripped-view match positions back to original coordinates. If every - // position the stripped view would mask is also masked by the raw view, the raw view - // already covered everything — for example, an ordinary multi-line PEM private key where - // raw produces `BEGIN/…redacted…/END` while stripped collapses to `***`. A real bypass - // exists only when the stripped view masks at least one original position raw missed (e.g. - // the tail of an `sk-` token whose prefix-boundary was broken by a spliced zero-width or - // NBSP character). - const { patterns } = resolveRedactOptions({ mode: "tools" }); - const rawMask = computeRedactionBitmap(commandText, patterns); - const strippedMask = computeRedactionBitmap(stripped, patterns); + // Detect bypass by position-bitmap coverage. Run the redaction matchers on both views and + // map stripped-view match positions back to original coordinates. If every position the + // stripped view would mask is also masked by the raw view, the raw view already covered + // everything — for example, an ordinary multi-line PEM private key where raw produces + // `BEGIN/…redacted…/END` while stripped collapses to `***`. A real bypass exists only when + // the stripped view masks at least one original position raw missed (e.g. the tail of an + // `sk-` token whose prefix-boundary was broken by a spliced zero-width or NBSP character). + const redaction = resolveRedactOptions({ mode: "tools" }); + const rawMask = computeSensitiveRedactionBitmap(commandText, redaction); + const strippedMask = computeSensitiveRedactionBitmap(stripped, redaction); let bypassDetected = false; for (let i = 0; i < strippedMask.length; i++) { if (strippedMask[i] && !rawMask[strippedToOrig[i]]) { diff --git a/src/logging/redact.test.ts b/src/logging/redact.test.ts index 825cdc2d426..0162c3ac001 100644 --- a/src/logging/redact.test.ts +++ b/src/logging/redact.test.ts @@ -10,6 +10,7 @@ import { redactSensitiveFieldValue, redactSensitiveLines, redactSensitiveText, + redactToolDetail, resolveRedactOptions, } from "./redact.js"; @@ -34,10 +35,7 @@ afterEach(() => { describe("redactSensitiveText", () => { it("masks env assignments while keeping the key", () => { const input = "OPENAI_API_KEY=sk-1234567890abcdef"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("OPENAI_API_KEY=sk-123…cdef"); }); @@ -50,28 +48,19 @@ describe("redactSensitiveText", () => { "PASSWORD=${PASSWORD:-}", "GITHUB_TOKEN=${GITHUB_TOKEN}", ].join("\n"); - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe(input); }); it("masks shell env references that do not match the assignment key", () => { - const output = redactSensitiveText("DISCORD_BOT_TOKEN=$SUPERSECRET123", { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText("DISCORD_BOT_TOKEN=$SUPERSECRET123", { mode: "tools" }); expect(output).toBe("DISCORD_BOT_TOKEN=***"); }); it("masks literal shell env expansion defaults in assignments", () => { const fallback = "discordliteral1234567890"; const input = `DISCORD_BOT_TOKEN="\${DISCORD_BOT_TOKEN:-${fallback}}"`; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).not.toContain(fallback); expect(output).toBe('DISCORD_BOT_TOKEN="${DISC…890}"'); }); @@ -88,10 +77,7 @@ describe("redactSensitiveText", () => { const xai = "issue85049-xai-cleartext-token-ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; const brave = "issue85049-brave-cleartext-token-ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; const input = String.raw`raw_params={"command":"export XAI_API_KEY=\"${xai}\" && export BRAVE_API_KEY=\\\"${brave}\\\" && echo blocked"}`; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toContain("XAI_API_KEY="); expect(output).toContain("BRAVE_API_KEY="); expect(output).not.toContain(xai); @@ -101,65 +87,50 @@ describe("redactSensitiveText", () => { it("masks CLI flags", () => { const input = "curl --token abcdef1234567890ghij https://api.test"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("curl --token abcdef…ghij https://api.test"); }); it("masks hook token CLI flags", () => { const input = "gog gmail watch serve --hook-token abcdef1234567890ghij"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("gog gmail watch serve --hook-token abcdef…ghij"); }); + it("does not treat option-alternative prose as a CLI flag secret", () => { + const input = "Use either --password or --password-file."; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe(input); + }); + it("masks sensitive URL query parameters", () => { const input = "connect https://user.example/sync?access_token=abcdef1234567890ghij&safe=value"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("connect https://user.example/sync?access_token=abcdef…ghij&safe=value"); }); it("masks short URL query tokens fully", () => { const input = "cdp=https://browserless.example.com/?token=supersecret123"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("cdp=https://browserless.example.com/?token=***"); }); it("masks standalone lowercase token assignments in diagnostic output", () => { const input = "matrix access_token=abcdef1234567890ghij next"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("matrix access_token=abcdef…ghij next"); }); it("masks JSON fields", () => { const input = '{"token":"abcdef1234567890ghij"}'; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe('{"token":"abcdef…ghij"}'); }); it("masks payment credential JSON fields without redacting unrelated amounts", () => { const input = '{"card_number":"4242424242424242","cvc":"123","sharedPaymentToken":"spt_abcdefghijklmnopqrstuvwxyz","payment_credential":"paycred_abcdefghijklmnopqrstuvwxyz","amount":"4200"}'; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe( '{"card_number":"***","cvc":"***","sharedPaymentToken":"spt_ab…wxyz","payment_credential":"paycre…wxyz","amount":"4200"}', ); @@ -168,19 +139,22 @@ describe("redactSensitiveText", () => { it("masks HTTP client config secrets in JSON and object-inspection fields", () => { const appSecret = "feishu_app_secret_1234567890"; const clientSecret = "oauth_client_secret_1234567890"; + const credential = "opaque_credential_1234567890"; const input = [ `body: {"app_secret":"${appSecret}"}`, `config: { appSecret: '${appSecret}', client_secret: '${clientSecret}' }`, + `payload: {"credential":"${credential}"}`, + `details: { credential: '${credential}' }`, ].join("\n"); - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toContain('"app_secret":"feishu…7890"'); expect(output).toContain("appSecret: 'feishu…7890'"); expect(output).toContain("client_secret: 'oauth_…7890'"); + expect(output).toContain('"credential":"***"'); + expect(output).toContain("credential: 'opaque…7890'"); expect(output).not.toContain(appSecret); expect(output).not.toContain(clientSecret); + expect(output).not.toContain(credential); }); it("masks payment credential assignments and flags", () => { @@ -191,10 +165,7 @@ describe("redactSensitiveText", () => { "--payment-credential paycred_abcdefghijklmnopqrstuvwxyz", "--card-number 4000056655665556", ].join(" "); - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).not.toContain("4242424242424242"); expect(output).not.toContain("4000056655665556"); expect(output).not.toContain("spt_abcdefghijklmnopqrstuvwxyz"); @@ -210,11 +181,8 @@ describe("redactSensitiveText", () => { const bearer = "feishu_tenant_access_abcdef123456"; const cookie = "session_cookie_value_abcdef123456"; const input = `headers: { authorization: 'Bearer ${bearer}', cookie: '${cookie}' }`; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); - expect(output).toContain("authorization: 'Bearer…3456'"); + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toContain("authorization: '***'"); expect(output).toContain("cookie: 'sessio…3456'"); expect(output).not.toContain(bearer); expect(output).not.toContain(cookie); @@ -223,10 +191,7 @@ describe("redactSensitiveText", () => { it("masks payment credential URL query parameters", () => { const input = "POST /authorize?shared_payment_token=spt_abcdefghijklmnopqrstuvwxyz&card_number=4242424242424242&amount=4200"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe( "POST /authorize?shared_payment_token=spt_ab…wxyz&card_number=***&amount=4200", ); @@ -267,24 +232,26 @@ describe("redactSensitiveText", () => { it("masks bearer tokens", () => { const input = "Authorization: Bearer abcdef1234567890ghij"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("Authorization: Bearer abcdef…ghij"); }); it("masks Basic authorization header tokens", () => { const secret = "c2VjcmV0OnBhc3M="; - const output = redactSensitiveText(`Authorization: Basic ${secret}`, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(`Authorization: Basic ${secret}`, { mode: "tools" }); expect(output).toBe("Authorization: Basic ***"); expect(output).not.toContain(secret); }); + it("masks Bot authorization header tokens", () => { + const secret = `${"A".repeat(24)}.${"B".repeat(6)}.${"C".repeat(27)}`; + const output = redactSensitiveText(`Authorization: Bot ${secret}`, { mode: "tools" }); + + expect(output).toBe("Authorization: Bot AAAAAA…CCCC"); + expect(output).not.toContain(secret); + }); + it("masks named Gateway security headers", () => { const openClawToken = "supersecretgatewaytoken1234567890"; const pomeriumJwt = "eyJheaderabcd.eyJpayloadabcd.signatureabcd123456"; @@ -294,10 +261,7 @@ describe("redactSensitiveText", () => { `x-pomerium-jwt-assertion: ${pomeriumJwt}`, `X-Api-Key=${apiKey}`, ].join("\n"); - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toContain("X-OpenClaw-Token: supers…7890"); expect(output).toContain("x-pomerium-jwt-assertion: eyJhea…3456"); @@ -309,10 +273,7 @@ describe("redactSensitiveText", () => { it("masks token prefixes embedded after adjacent text", () => { const token = `ghp_${"a".repeat(5_000)}`; - const output = redactSensitiveText(`prefix-${token} suffix`, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(`prefix-${token} suffix`, { mode: "tools" }); expect(output).toBe("prefix-ghp_aa…aaaa suffix"); expect(output).not.toContain(token); expect(output).not.toContain("a".repeat(100)); @@ -320,66 +281,362 @@ describe("redactSensitiveText", () => { it("masks URL query tokens", () => { const input = "GET /_matrix/client/v3/sync?access_token=abcdef1234567890ghij"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("GET /_matrix/client/v3/sync?access_token=abcdef…ghij"); }); it("masks bot-style tokens", () => { const input = "123456:ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("123456…cdef"); }); it("masks bot API URL tokens", () => { const input = "GET https://api.example.test/bot123456:ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef/getMe HTTP/1.1"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("GET https://api.example.test/bot123456…cdef/getMe HTTP/1.1"); }); it("redacts short tokens fully", () => { const input = "TOKEN=shortvalue"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("TOKEN=***"); }); it("does not redact lowercase key diagnostics", () => { const input = 'agents.defaults: Unrecognized key: "llm"'; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe(input); }); + it("does not redact diagnostic code assignments outside URL or form bodies", () => { + const input = "Oops: failed: code=E1 status=500"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe(input); + }); + + it("masks standalone pass assignments", () => { + const output = redactSensitiveText("db pass=opaque-pass-secret-1234567890 next", { + mode: "tools", + }); + expect(output).toBe("db pass=opaque…7890 next"); + expect(output).not.toContain("opaque-pass-secret-1234567890"); + }); + + it("masks complete unquoted assignment values that contain delimiter-like punctuation", () => { + const input = "password=abc,def token=abc;def client_secret=abc]def pass=abc)def"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("password=*** token=*** client_secret=*** pass=***"); + expect(output).not.toContain("abc,def"); + expect(output).not.toContain("abc;def"); + expect(output).not.toContain("abc]def"); + expect(output).not.toContain("abc)def"); + }); + + it("masks quoted standalone assignments", () => { + const input = "password='abc;def' token=\"abc;def\" secret=`abc;def`"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("password='***' token=\"***\" secret=`***`"); + expect(output).not.toContain("abc;def"); + }); + it("masks sensitive URL query params while preserving non-sensitive params", () => { const input = "GET /_matrix/client/v3/sync?access_token=abcdef1234567890ghij&since=123"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("GET /_matrix/client/v3/sync?access_token=abcdef…ghij&since=123"); }); it("treats sensitive URL query param names case-insensitively", () => { const input = "connect https://gateway.example/ws?Access-Token=short-token&ok=1"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("connect https://gateway.example/ws?Access-Token=***&ok=1"); + }); + + it("masks opaque sensitive URL query params without known token prefixes", () => { + const input = + "callback https://example.test/oauth?code=oauth-code-abc123&state=visible&x-amz-signature=abc123xyz&x-amz-security-token=aws-session-token-123&authorization=authz-secret-123&private_key=pk-secret-123&app_secret=app-secret-123&credential=credential-secret-123"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe( + "callback https://example.test/oauth?code=***&state=visible&x-amz-signature=***&x-amz-security-token=aws-se…-123&authorization=***&private_key=***&app_secret=***&credential=creden…-123", + ); + }); + + it("masks URL userinfo and database connection-string passwords", () => { + const input = [ + "https://browser-user:browser-password-1234567890@api.example.test/v1", + "https://:empty-username-password-1234567890@api.example.test/v1", + "https://same:same@example.test/v1", + "postgres://dbuser:database-password-1234567890@db.example.test/openclaw", + "postgres://secret:secret@db.example.test/openclaw", + "mongodb+srv://mongo:mongodb-password-1234567890@cluster.example.test/app", + "redis://:redis-password-1234567890@cache.example.test/0", + "rediss://cache:redis-tls-password-1234567890@cache.example.test/0", + ].join(" "); + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).not.toContain("browser-password-1234567890"); + expect(output).not.toContain("empty-username-password-1234567890"); + expect(output).not.toContain("database-password-1234567890"); + expect(output).not.toContain("mongodb-password-1234567890"); + expect(output).not.toContain("redis-password-1234567890"); + expect(output).not.toContain("redis-tls-password-1234567890"); + expect(output).toContain("https://browser-user:browse…7890@api.example.test/v1"); + expect(output).toContain("https://:empty-…7890@api.example.test/v1"); + expect(output).toContain("https://same:***@example.test/v1"); + expect(output).toContain("postgres://dbuser:databa…7890@db.example.test/openclaw"); + expect(output).toContain("postgres://secret:***@db.example.test/openclaw"); + expect(output).toContain("mongodb+srv://mongo:mongod…7890@cluster.example.test/app"); + expect(output).toContain("redis://:redis-…7890@cache.example.test/0"); + expect(output).toContain("rediss://cache:redis-…7890@cache.example.test/0"); + }); + + it("masks sensitive form-urlencoded body fields by exact key", () => { + const input = + "code=oauth-code-123&hook_token=hook-token-123&jwt=jwt-secret-123&pass=form-pass-123&client_secret=oauth-client-secret-1234567890&refresh_token=refresh-token-1234567890&token_count=42&session_id=session-visible"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe( + "code=***&hook_token=***&jwt=***&pass=***&client_secret=***&refresh_token=***&token_count=42&session_id=session-visible", + ); + expect(output).not.toContain("oauth-code-123"); + expect(output).not.toContain("hook-token-123"); + expect(output).not.toContain("jwt-secret-123"); + expect(output).not.toContain("form-pass-123"); + expect(output).not.toContain("oauth-client-secret-1234567890"); + expect(output).not.toContain("refresh-token-1234567890"); + }); + + it("masks non-auth form body secret fields after a safe first key", () => { + const input = + "client_id=visible&app_secret=opaque-app-secret&credential=opaque-credential&shared_payment_token=spt_abcdefghijklmnopqrstuvwxyz&safe=value"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe( + "client_id=visible&app_secret=***&credential=***&shared_payment_token=***&safe=value", + ); + expect(output).not.toContain("opaque-app-secret"); + expect(output).not.toContain("opaque-credential"); + expect(output).not.toContain("spt_abcdefghijklmnopqrstuvwxyz"); + }); + + it("masks form body secret fields embedded in diagnostic prose", () => { + const input = + "body: client_id=visible&app_secret=opaque-app-secret&credential=opaque-credential&safe=value"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("body: client_id=visible&app_secret=***&credential=***&safe=value"); + expect(output).not.toContain("opaque-app-secret"); + expect(output).not.toContain("opaque-credential"); + }); + + it("masks form body secret fields in multiline tool output", () => { + const input = + "request start\nbody: client_id=visible&app_secret=opaque-app-secret&safe=value\nrequest end"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe( + "request start\nbody: client_id=visible&app_secret=***&safe=value\nrequest end", + ); + expect(output).not.toContain("opaque-app-secret"); + }); + + it("masks percent-encoded form body secret keys", () => { + const input = "body: client%5Fsecret=oauth-secret&app%2Dsecret=app-secret&safe=value"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("body: client%5Fsecret=***&app%2Dsecret=***&safe=value"); + expect(output).not.toContain("oauth-secret"); + expect(output).not.toContain("app-secret"); + }); + + it("masks quoted form body secret fields embedded in diagnostic prose", () => { + const input = + 'body: "client_secret=oauth-secret&safe=value" fallback: `safe=value&app_secret=app-secret`'; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe( + 'body: "client_secret=***&safe=value" fallback: `safe=value&app_secret=***`', + ); + expect(output).not.toContain("oauth-secret"); + expect(output).not.toContain("app-secret"); + }); + + it("masks percent-encoded form body keys spliced with invisible characters", () => { + const input = "body: client%5Fse\u200Bcret=oauth-secret&safe=value"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("body: client%5Fse\u200Bcret=***&safe=value"); + expect(output).not.toContain("oauth-secret"); + }); + + it("masks form body keys with leading invisible separators", () => { + const input = "body: \u200Bclient_secret=oauth-secret&safe=value"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("body: \u200Bclient_secret=***&safe=value"); + expect(output).not.toContain("oauth-secret"); + }); + + it("masks form body keys with plus-encoded separators", () => { + const input = "body: client_se+cret=oauth-secret&safe=value"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("body: client_se+cret=***&safe=value"); + expect(output).not.toContain("oauth-secret"); + }); + + it("masks form and query keys with raw control separators", () => { + const input = + "body: client_se\u0000cret=oauth-secret&safe=value GET /cb?client_se\u0001cret=query-secret&safe=1"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe( + "body: client_se\u0000cret=***&safe=value GET /cb?client_se\u0001cret=***&safe=1", + ); + expect(output).not.toContain("oauth-secret"); + expect(output).not.toContain("query-secret"); + }); + + it("masks quoted form body values after equals", () => { + const input = + 'body: password="opaque-password-secret" client_id=visible&app_secret="opaque-app-secret"&safe=1'; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("body: password=*** client_id=visible&app_secret=***&safe=1"); + expect(output).not.toContain("opaque-password-secret"); + expect(output).not.toContain("opaque-app-secret"); + }); + + it("masks form body keys with percent-encoded invisible separators", () => { + const input = "body: client%5Fse%E2%80%8Bcret=oauth-secret&safe=value"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("body: client%5Fse%E2%80%8Bcret=***&safe=value"); + expect(output).not.toContain("oauth-secret"); + }); + + it("masks form body keys with percent-encoded whitespace and control separators", () => { + const input = + "body: client%5Fse%20cret=space-secret&safe=value next: client%5Fse%00cret=nul-secret&safe=value"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe( + "body: client%5Fse%20cret=***&safe=value next: client%5Fse%00cret=***&safe=value", + ); + expect(output).not.toContain("space-secret"); + expect(output).not.toContain("nul-secret"); + }); + + it("masks URL query keys with percent-encoded invisible separators", () => { + const input = "GET https://example.test/cb?client%5Fse%E2%80%8Bcret=oauth-secret&safe=1"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("GET https://example.test/cb?client%5Fse%E2%80%8Bcret=***&safe=1"); + expect(output).not.toContain("oauth-secret"); + }); + + it("masks URL query keys with plus-encoded separators", () => { + const input = "GET https://example.test/cb?client_se+cret=oauth-secret&safe=1"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("GET https://example.test/cb?client_se+cret=***&safe=1"); + expect(output).not.toContain("oauth-secret"); + }); + + it("masks URL query keys with percent-encoded whitespace and control separators", () => { + const input = + "GET https://example.test/cb?client%5Fse%20cret=space-secret&safe=1&client%5Fse%00cret=nul-secret"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe( + "GET https://example.test/cb?client%5Fse%20cret=***&safe=1&client%5Fse%00cret=***", + ); + expect(output).not.toContain("space-secret"); + expect(output).not.toContain("nul-secret"); + }); + + it("masks encoded sensitive URL query keys after later separators", () => { + const input = "GET https://example.test/cb?scope=read,write&client%5Fsecret=oauth-secret"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("GET https://example.test/cb?scope=read,write&client%5Fsecret=***"); + expect(output).not.toContain("oauth-secret"); + }); + + it("masks complete encoded URL query values that contain commas", () => { + const input = "GET https://example.test/cb?client%5Fsecret=abc,def&safe=1"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("GET https://example.test/cb?client%5Fsecret=***&safe=1"); + expect(output).not.toContain("abc,def"); + expect(output).not.toContain(",def"); + }); + + it("masks complete URL query values that contain delimiter-like punctuation", () => { + const input = + "GET /cb?token=abc)def&safe=1 /cb?client%5Fsecret=abc]def&safe=1 /cb?code=short#frag"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe( + "GET /cb?token=***&safe=1 /cb?client%5Fsecret=***&safe=1 /cb?code=***#frag", + ); + expect(output).not.toContain("abc)def"); + expect(output).not.toContain("abc]def"); + expect(output).toContain("#frag"); + }); + + it("masks quoted URL query values after equals", () => { + const input = 'GET /cb?token="opaque-token-secret"&safe=1 /cb?client%5Fsecret="oauth-secret"'; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("GET /cb?token=opaque…cret&safe=1 /cb?client%5Fsecret=***"); + expect(output).not.toContain("opaque-token-secret"); + expect(output).not.toContain("oauth-secret"); + }); + + it("masks complete encoded form values that contain commas", () => { + const input = "body: client%5Fsecret=abc,def&safe=value"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("body: client%5Fsecret=***&safe=value"); + expect(output).not.toContain("abc,def"); + expect(output).not.toContain(",def"); + }); + + it("masks complete form values that contain delimiter-like punctuation", () => { + const input = "body: client_secret=abc)def&safe=1 next: client%5Fsecret=abc]def&safe=1"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("body: client_secret=***&safe=1 next: client%5Fsecret=***&safe=1"); + expect(output).not.toContain("abc)def"); + expect(output).not.toContain("abc]def"); + }); + + it("masks encoded sensitive form keys in single-pair and multiline diagnostics", () => { + const input = [ + "client%5Fsecret=single-secret", + "trace body: client%5Fsecret=multiline-secret&safe=1", + ].join("\n"); + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("client%5Fsecret=***\ntrace body: client%5Fsecret=***&safe=1"); + expect(output).not.toContain("single-secret"); + expect(output).not.toContain("multiline-secret"); + }); + + it("masks single-pair form fields in explicit body contexts", () => { + const input = "body: code=oauth-code-123 form_body=signature=aws-signature-123 Oops code=E1"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("body: code=*** form_body=signature=*** Oops code=E1"); + expect(output).not.toContain("oauth-code-123"); + expect(output).not.toContain("aws-signature-123"); + }); + + it("masks entire-line explicit body wrapper form payloads", () => { + const output = redactSensitiveText("body=client_secret=oauth-secret&safe=1", { mode: "tools" }); + expect(output).toBe("body=client_secret=***&safe=1"); + expect(output).not.toContain("oauth-secret"); + + const outputWithLaterSecret = redactSensitiveText( + "form_body=client_secret=oauth-secret&app_secret=app-secret", + { mode: "tools" }, + ); + expect(outputWithLaterSecret).toBe("form_body=client_secret=***&app_secret=***"); + expect(outputWithLaterSecret).not.toContain("oauth-secret"); + expect(outputWithLaterSecret).not.toContain("app-secret"); + }); + + it("masks first-position form-urlencoded fields embedded in larger log lines", () => { + const input = "manual callback code=oauth-code-123&state=visible"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe("manual callback code=***&state=visible"); + expect(output).not.toContain("oauth-code-123"); + }); + + it("does not apply built-in form-body redaction when custom patterns override defaults", () => { + const input = "password=value&safe=1"; const output = redactSensitiveText(input, { mode: "tools", - patterns: defaults, + patterns: [String.raw`custom-secret-([A-Za-z0-9]+)`], }); - expect(output).toBe("connect https://gateway.example/ws?Access-Token=***&ok=1"); + expect(output).toBe(input); }); it("redacts private key blocks", () => { @@ -389,10 +646,7 @@ describe("redactSensitiveText", () => { "ZYXWVUT987654321", "-----END PRIVATE KEY-----", ].join("\n"); - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe( ["-----BEGIN PRIVATE KEY-----", "…redacted…", "-----END PRIVATE KEY-----"].join("\n"), ); @@ -407,6 +661,24 @@ describe("redactSensitiveText", () => { expect(output).toBe("token=abcdef…ghij"); }); + it("keeps single-capture custom patterns focused on the captured occurrence", () => { + const input = "password=abc123456789012345&confirm=abc123456789012345"; + const output = redactSensitiveText(input, { + mode: "tools", + patterns: [String.raw`password=([^&]+)&confirm=\1`], + }); + expect(output).toBe("password=abc123…2345&confirm=abc123456789012345"); + }); + + it("masks captured custom-pattern values even when the value repeats later", () => { + const input = "password=abc123456789012345&confirm=abc123456789012345"; + const output = redactSensitiveText(input, { + mode: "tools", + patterns: [String.raw`password=([^&]+)&confirm=[^&]+`], + }); + expect(output).toBe("password=abc123…2345&confirm=abc123456789012345"); + }); + it("honors escaped character classes in custom patterns", () => { const input = "contact peter@dc.io"; const output = redactSensitiveText(input, { @@ -428,68 +700,250 @@ describe("redactSensitiveText", () => { it("redacts large payloads with bounded regex passes", () => { const input = `${"x".repeat(40_000)} OPENAI_API_KEY=sk-1234567890abcdef ${"y".repeat(40_000)}`; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toContain("OPENAI_API_KEY=sk-123…cdef"); }); it("masks Tencent Cloud SecretId (AKID prefix, uppercase-only)", () => { const input = "SecretId is AKIDZ8EXAMPLEFAKE01KEY99TEST"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("SecretId is AKIDZ8…TEST"); }); it("masks Tencent Cloud SecretId with mixed-case characters", () => { const input = "AKIDz8exampleFake01Key99Test"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("AKIDz8…Test"); }); it("masks Alibaba Cloud AccessKey ID (LTAI prefix)", () => { const input = "AccessKeyId=LTAI5tExampleFakeKeyXyz9"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("AccessKeyId=LTAI5t…Xyz9"); }); it("masks HuggingFace tokens (hf_ prefix)", () => { const input = "hf_ABCDEFghijklmnopqrstuv"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("hf_ABC…stuv"); }); it("masks Replicate tokens (r8_ prefix)", () => { const input = "r8_ABCDEFghijklmnopqrstuv"; - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).toBe("r8_ABC…stuv"); }); + it("masks expanded vendor-prefix token corpus", () => { + const tokens = [ + "sk-ant-abcdefghijklmnopqrstuvwxyz", + "gho_abcdefghijklmnopqrstuvwxyz", + "ghu_abcdefghijklmnopqrstuvwxyz", + "ghs_abcdefghijklmnopqrstuvwxyz", + "ghr_abcdefghijklmnopqrstuvwxyz", + "glpat-abcdefghijklmnopqrstuvwxyz12.ab.abcdefghi", + `gloas-${"a".repeat(64)}`, + ["xoxb", "1234567890", "abcdefghijklmnopqrstuvwxyz"].join("-"), + "https://hooks.slack.com/services/T1234567890/B1234567890/abcdefghijklmnopqrstuvwxy", + "https://discord.com/api/webhooks/123456789012345678/abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdef", + `discord bot token ${"A".repeat(24)}.${"B".repeat(6)}.${"C".repeat(27)}`, + "AIzaabcdefghijklmnopqrstuvwxyzABCDE", + "pplx-abcdefghijklmnopqrstuvwxyz", + "fal_abcdefghijklmnopqrstuvwxyz", + "fc-abcdefghijklmnopqrstuvwxyz", + "bb_live_abcdefghijklmnopqrstuvwxyz", + "gAAAAabcdefghijklmnopqrstuvwxyz123456", + "AKIAABCDEFGHIJKLMNOP", + "ASIAABCDEFGHIJKLMNOP", + "api_org_abcdefghijklmnopqrstuvwxyz12345678", + ["sk", "live", "abcdefghijklmnopqrstuvwxyz"].join("_"), + ["sk", "test", "abcdefghijklmnopqrstuvwxyz"].join("_"), + ["rk", "live", "abcdefghijklmnopqrstuvwxyz"].join("_"), + "SG.abcdefghijklmnopqrstuvwxyz.0123456789abcdefghijklmnopqrstuvwxyz", + "npm_abcdefghijklmnopqrstuvwxyz", + "pypi-abcdefghijklmnopqrstuvwxyz", + "dop_v1_abcdefghijklmnopqrstuvwxyz", + "doo_v1_abcdefghijklmnopqrstuvwxyz", + "dor_v1_abcdefghijklmnopqrstuvwxyz", + `dp.pt.${"A".repeat(43)}`, + `dckr_pat_${"A".repeat(27)}`, + `dckr_oat_${"B".repeat(32)}`, + `bkua_${"a".repeat(40)}`, + `CCIPAT_${"A".repeat(22)}_${"a".repeat(40)}`, + `sbp_${"a".repeat(40)}`, + `dapi${"a".repeat(32)}-1`, + `ddp_${"A".repeat(36)}`, + `glsa_${"A".repeat(41)}`, + `glc_eyJ${"A".repeat(80)}`, + `nfp_${"A".repeat(36)}`, + `CFPAT-${"A".repeat(43)}`, + `ATCTT3xFfG${"A".repeat(48)}=ABCDEF12`, + `ATATT${"A".repeat(48)}=ABCDEF12`, + `ATBB${"A".repeat(24)}`, + `BBDC-${"A".repeat(42)}`, + `HRKU-AA${"A".repeat(58)}`, + ["pat", "na1", "12345678", "1234", "1234", "1234", "123456789abc"].join("-"), + `apify_api_${"A".repeat(36)}`, + `FlyV1 fm123_${"A".repeat(120)}`, + `fio-u-${"A".repeat(64)}`, + "am_abcdefghijklmnopqrstuvwxyz", + "sk_abcdefghijklmnopqrstuvwxyz", + "tvly-abcdefghijklmnopqrstuvwxyz", + "exa_abcdefghijklmnopqrstuvwxyz", + "gsk_abcdefghijklmnopqrstuvwxyz", + "syt_abcdefghijklmnopqrstuvwxyz", + "retaindb_abcdefghijklmnopqrstuvwxyz", + "hsk-abcdefghijklmnopqrstuvwxyz", + "mem0_abcdefghijklmnopqrstuvwxyz", + "brv_abcdefghijklmnopqrstuvwxyz", + "xai-abcdefghijklmnopqrstuvwxyzABCDE", + ]; + // Redact each fixture alone so every vendor pattern proves it stays reachable through + // DEFAULT_REDACT_PREFILTER_RE; a joined corpus would let one trigger unlock all others. + for (const token of tokens) { + expect(redactSensitiveText(token, { mode: "tools" }), token).not.toContain(token); + } + expect(redactSensitiveText("AKIAABCDEFGHIJKLMNOP", { mode: "tools" })).toBe("AKIAAB…MNOP"); + expect( + redactSensitiveText(["sk", "live", "abcdefghijklmnopqrstuvwxyz"].join("_"), { + mode: "tools", + }), + ).toBe("sk_liv…wxyz"); + expect( + redactSensitiveText("SG.abcdefghijklmnopqrstuvwxyz.0123456789abcdefghijklmnopqrstuvwxyz", { + mode: "tools", + }), + ).toBe("SG.abc…wxyz"); + expect(redactSensitiveText("xai-abcdefghijklmnopqrstuvwxyzABCDE", { mode: "tools" })).toBe( + "xai-ab…BCDE", + ); + }); + + it("does not redact ordinary identifiers containing short token-prefix substrings", () => { + const input = + "npm_telegram_package_spec ask_openclaw_query_patterns team_management risk_assessment glpat-docs dapi-example sbp_short nfp_site CCIPAT_docs ATATT-example"; + const output = redactSensitiveText(input, { mode: "tools" }); + expect(output).toBe(input); + }); + + it("does not corrupt base64 blobs that embed token-prefix shapes", () => { + // Tiny-PNG base64 contains a gAAAA run from zero-filled IHDR bytes; pure-base64-alphabet + // prefixes must not fire mid-blob or media payloads get mangled. + const dataUrl = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNgYAAAAAMAASsJTYQAAAAASUVORK5CYII="; + expect(redactSensitiveText(dataUrl, { mode: "tools" })).toBe(dataUrl); + const blobWithUppercaseRun = `blob: ${"x".repeat(4)}AKIAABCDEFGHIJKLMNOPqrstuv`; + expect(redactSensitiveText(blobWithUppercaseRun, { mode: "tools" })).toBe(blobWithUppercaseRun); + const dataUrlWithPlusBoundary = `data:application/octet-stream;base64,AAAA+gAAAA${"B".repeat(24)}`; + expect(redactSensitiveText(dataUrlWithPlusBoundary, { mode: "tools" })).toBe( + dataUrlWithPlusBoundary, + ); + expect(redactSensitiveText("aws AKIA_ID=AKIAABCDEFGHIJKLMNOP", { mode: "tools" })).toBe( + "aws AKIA_ID=AKIAAB…MNOP", + ); + }); + + it("does not corrupt large data URLs across chunked replacement boundaries", () => { + // replacePatternBounded slices 32 KiB+ inputs into 16 KiB chunks; a chunk start must not + // satisfy the pure-base64 prefix boundary (`^`) or hide the `;base64,` container from its + // lookbehind, so the boundary patterns run unchunked. + const prefix = "data:application/octet-stream;base64,"; + const chunkSize = 16_384; + const pad = "A".repeat(chunkSize * 2 - prefix.length); + const dataUrl = `${prefix}${pad}gAAAA${"B".repeat(24)}${"C".repeat(chunkSize)}`; + expect(redactSensitiveText(dataUrl, { mode: "tools" })).toBe(dataUrl); + }); + + it("masks pure-base64-alphabet tokens after URL and path delimiters", () => { + const fernet = `gAAAA${"B".repeat(24)}`; + const reset = `https://app.example/reset/${fernet} visited`; + const output = redactSensitiveText(reset, { mode: "tools" }); + expect(output).not.toContain(fernet); + expect(output).toContain("https://app.example/reset/"); + const s3Path = "fetch /buckets/AKIAABCDEFGHIJKLMNOP/objects"; + expect(redactSensitiveText(s3Path, { mode: "tools" })).toBe( + "fetch /buckets/AKIAAB…MNOP/objects", + ); + }); + + it("masks bare sensitive query and form keys through the default options path", () => { + expect( + redactSensitiveText("GET https://example.test/oauth?code=opaque-grant-123", { + mode: "tools", + }), + ).toBe("GET https://example.test/oauth?code=***"); + expect(redactSensitiveText("jwt=opaque.jwt.value&safe=1", { mode: "tools" })).toBe( + "jwt=***&safe=1", + ); + expect(redactSensitiveText("db pass=opaquepass123 next", { mode: "tools" })).toBe( + "db pass=*** next", + ); + }); + + it("masks obfuscated form keys with opaque values through the default options path", () => { + // Values intentionally avoid literal prefilter trigger words so the obfuscated key alone + // must make the default fast path run. + expect( + redactSensitiveText("body: client%5Fse\u200Bcret=opaque-value-123&safe=1", { + mode: "tools", + }), + ).toBe("body: client%5Fse\u200Bcret=***&safe=1"); + expect( + redactSensitiveText("GET https://example.test/cb?client_se+cret=opaque-value-123&safe=1", { + mode: "tools", + }), + ).toBe("GET https://example.test/cb?client_se+cret=***&safe=1"); + expect( + redactSensitiveText("body: client_secre%74=opaque-value-123&safe=1", { mode: "tools" }), + ).toBe("body: client_secre%74=***&safe=1"); + expect( + redactSensitiveText("body: client_se\u3164cret\u3164=opaque-value-123&safe=1", { + mode: "tools", + }), + ).toBe("body: client_se\u3164cret\u3164=***&safe=1"); + }); + + it("masks connection-string passwords through the default options path", () => { + expect( + redactSensitiveText("postgres://dbuser:opaquepw12345@db.example.test/openclaw", { + mode: "tools", + }), + ).toBe("postgres://dbuser:***@db.example.test/openclaw"); + }); + + it("masks quoted standalone values containing the other quote character", () => { + const input = `password="it's-a-secret" next`; + expect(redactSensitiveText(input, { mode: "tools" })).toBe('password="***" next'); + }); + + it("masks unterminated quoted standalone values", () => { + const input = 'token="opaque-abc123 rest'; + expect(redactSensitiveText(input, { mode: "tools" })).toBe("token=*** rest"); + }); + + it("treats explicit default patterns like the built-in default path", () => { + const input = + 'GET /cb?client_secret=oauth-secret-123&safe=1 glpat-abcdefghijklmnopqrstuv password="it\'s"'; + expect(redactSensitiveText(input, { mode: "tools", patterns: defaults })).toBe( + redactSensitiveText(input, { mode: "tools" }), + ); + }); + + it("redacts raw secret values that contain an ellipsis", () => { + const input = "password=abcdef…1234567890"; + const output = redactSensitiveText(input, { mode: "tools" }); + + expect(output).toBe("password=***"); + expect(redactSensitiveFieldValue("password", "abcdef…1234567890")).toBe("***"); + }); + it("masks OAuth and JWT token shapes", () => { const input = [ "ya29.fake-access-token-with-enough-length", "1//0fake-refresh-token-with-enough-length", "eyJheaderabcd.eyJpayloadabcd.signatureabcd123456", ].join(" "); - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).not.toContain("ya29.fake-access-token"); expect(output).not.toContain("1//0fake-refresh-token"); expect(output).not.toContain("eyJheaderabcd.eyJpayloadabcd.signatureabcd123456"); @@ -502,10 +956,7 @@ describe("redactSensitiveText", () => { '{"password":"lmno-pqrs-tuvw-xyza"}', "main-test-case-name", ].join(" "); - const output = redactSensitiveText(input, { - mode: "tools", - patterns: defaults, - }); + const output = redactSensitiveText(input, { mode: "tools" }); expect(output).not.toContain("abcd-efgh-ijkl-mnop"); expect(output).not.toContain("qrst-uvwx-yzab-cdef"); expect(output).not.toContain("lmno-pqrs-tuvw-xyza"); @@ -535,6 +986,18 @@ describe("redactSensitiveText", () => { ); }); + it("forces redaction for tool details even when log redaction is disabled", () => { + writeConfig(`{ + logging: { + redactSensitive: "off", + }, + }`); + + expect(redactToolDetail("OPENAI_API_KEY=sk-1234567890abcdef")).toBe( + "OPENAI_API_KEY=sk-123…cdef", + ); + }); + it("does not resolve patterns when mode is off", () => { const options = { mode: "off" as const, @@ -546,6 +1009,7 @@ describe("redactSensitiveText", () => { expect(resolveRedactOptions(options)).toEqual({ mode: "off", patterns: [], + redactFormBodies: false, }); expect(redactSensitiveText("OPENAI_API_KEY=sk-1234567890abcdef", options)).toBe( "OPENAI_API_KEY=sk-1234567890abcdef", @@ -661,7 +1125,7 @@ describe("redactSecrets", () => { describe("redactSensitiveLines", () => { it("redacts matching content across all lines", () => { - const resolved = resolveRedactOptions({ mode: "tools", patterns: defaults }); + const resolved = resolveRedactOptions({ mode: "tools" }); const lines = ["curl --token abcdef1234567890ghij https://api.test", "normal log line"]; const result = redactSensitiveLines(lines, resolved); expect(result[0]).toBe("curl --token abcdef…ghij https://api.test"); @@ -677,18 +1141,18 @@ describe("redactSensitiveLines", () => { it("returns lines unmodified when resolved patterns is empty — does not fall back to defaults", () => { // Simulates the case where all user-configured patterns fail to compile. // The pre-resolved empty array must be honored, not silently replaced with defaults. - const resolved = { mode: "tools" as const, patterns: [] }; + const resolved = { mode: "tools" as const, patterns: [], redactFormBodies: false }; const lines = ["TOKEN=abcdef1234567890ghij"]; expect(redactSensitiveLines(lines, resolved)).toEqual(lines); }); it("returns empty array unchanged — does not produce a synthetic blank line", () => { - const resolved = resolveRedactOptions({ mode: "tools", patterns: defaults }); + const resolved = resolveRedactOptions({ mode: "tools" }); expect(redactSensitiveLines([], resolved)).toStrictEqual([]); }); it("redacts a PEM block spanning multiple lines in the array", () => { - const resolved = resolveRedactOptions({ mode: "tools", patterns: defaults }); + const resolved = resolveRedactOptions({ mode: "tools" }); const lines = [ "log: key follows", "-----BEGIN PRIVATE KEY-----", @@ -704,4 +1168,20 @@ describe("redactSensitiveLines", () => { expect(joined).toContain("…redacted…"); expect(joined).not.toContain("ABCDEF1234567890"); }); + + it("applies form-body redaction per line before joining for multiline patterns", () => { + const resolved = resolveRedactOptions({ mode: "tools" }); + const lines = [ + "jwt=opaque-jwt-secret-123&safe=1", + "key=opaque-key-secret-123&safe=1", + "https://example.test/cb?client%5Fsecret=oauth-secret&safe=1", + "normal log line", + ]; + expect(redactSensitiveLines(lines, resolved)).toEqual([ + "jwt=***&safe=1", + "key=***&safe=1", + "https://example.test/cb?client%5Fsecret=***&safe=1", + "normal log line", + ]); + }); }); diff --git a/src/logging/redact.ts b/src/logging/redact.ts index 5376a278ea6..b41198d43fc 100644 --- a/src/logging/redact.ts +++ b/src/logging/redact.ts @@ -15,9 +15,80 @@ const DEFAULT_REDACT_KEEP_END = 4; const PAYMENT_CREDENTIAL_ENV_KEYS = String.raw`CARD[_-]?NUMBER|CARD[_-]?CVC|CARD[_-]?CVV|CVC|CVV|SECURITY[_-]?CODE|PAYMENT[_-]?CREDENTIAL|SHARED[_-]?PAYMENT[_-]?TOKEN`; const PAYMENT_CREDENTIAL_QUERY_KEYS = String.raw`card[-_]?number|card[-_]?cvc|card[-_]?cvv|cvc|cvv|security[-_]?code|payment[-_]?credential|shared[-_]?payment[-_]?token`; +const AUTH_QUERY_KEYS = String.raw`access[-_]?token|auth[-_]?token|hook[-_]?token|refresh[-_]?token|id[-_]?token|api[-_]?key|apikey|client[-_]?secret|app[-_]?secret|private[-_]?key|credential|authorization|token|key|secret|password|pass|passwd|auth|jwt|session|code|signature|x[-_]?amz[-_]?(?:signature|security[-_]?token)`; +const FORM_BODY_FIRST_PAIR_KEYS = String.raw`${AUTH_QUERY_KEYS}|app[-_]?secret|credential|${PAYMENT_CREDENTIAL_QUERY_KEYS}`; +const STANDALONE_ASSIGNMENT_SECRET_KEYS = String.raw`access_token|refresh_token|id_token|auth[-_]?token|hook[-_]?token|api[-_]?key|client[-_]?secret|app[-_]?secret|private[-_]?key|authorization|jwt|token|secret|password|pass|passwd|credential|${PAYMENT_CREDENTIAL_QUERY_KEYS}`; +const BODY_SECRET_KEYS = new Set([ + "access_token", + "auth_token", + "hook_token", + "refresh_token", + "id_token", + "token", + "api_key", + "apikey", + "client_secret", + "app_secret", + "password", + "pass", + "passwd", + "auth", + "jwt", + "session", + "code", + "signature", + "x_amz_signature", + "x_amz_security_token", + "secret", + "credential", + "private_key", + "authorization", + "key", + "card_number", + "card_cvc", + "card_cvv", + "cvc", + "cvv", + "security_code", + "payment_credential", + "shared_payment_token", +]); +const FORM_BODY_KEY_INVISIBLE_CHARS = String.raw`\p{C}\u00A0\u1680\u2000-\u200A\u202F\u205F\u3000\u115F\u1160\u3164\uFFA0`; +const FORM_BODY_KEY_OBFUSCATION_RE = new RegExp( + String.raw`[${FORM_BODY_KEY_INVISIBLE_CHARS}+]`, + "gu", +); +const FORM_BODY_KEY_SEPARATOR_RE = /[\p{C}\p{Z}\u115F\u1160\u3164\uFFA0+]/gu; +const FORM_BODY_PERCENT_ESCAPE_RE = /%[0-9A-Fa-f]{2}/u; +const FORM_BODY_KEY = String.raw`[${FORM_BODY_KEY_INVISIBLE_CHARS}+]*(?:[A-Za-z_]|%[0-9A-Fa-f]{2})(?:[A-Za-z0-9_.-]|%[0-9A-Fa-f]{2}|[${FORM_BODY_KEY_INVISIBLE_CHARS}+])*`; +const FORM_BODY_VALUE = "[^&\\s<>]*"; +const URL_QUERY_VALUE = "[^&#\\s<>]*"; +const FORM_BODY_PAIR = String.raw`${FORM_BODY_KEY}=${FORM_BODY_VALUE}`; +const FORM_BODY_RE = new RegExp(String.raw`^${FORM_BODY_PAIR}(?:&${FORM_BODY_PAIR})+$`, "u"); +const FORM_BODY_SUBSTRING_RE = new RegExp( + String.raw`(^|[\s:({\[,="'` + "`" + String.raw`])(${FORM_BODY_PAIR}(?:&${FORM_BODY_PAIR})+)`, + "gu", +); +const ENCODED_FORM_PAIR_RE = new RegExp( + String.raw`(^|[\s:({\[,="'` + "`" + String.raw`&])(${FORM_BODY_KEY})=(${FORM_BODY_VALUE})`, + "gu", +); +const FORM_BODY_CONTEXT_SINGLE_PAIR_RE = new RegExp( + String.raw`(\b(?:body|form(?:[-_\s]?body)?)\s*[:=]\s*(["'\x60]?))(${FORM_BODY_KEY})=(${FORM_BODY_VALUE})(["'\x60]?)`, + "giu", +); +const URL_QUERY_PAIR_RE = new RegExp( + String.raw`([?&])(${FORM_BODY_KEY})=(${URL_QUERY_VALUE})`, + "gu", +); +const SECRET_VALUE_TRAILING_DELIMITER_RE = /(["'`,;)}\]]+)$/u; +const SECRET_VALUE_SUFFIX_RE = /^["'`,;)}\]]*$/u; +const SECRET_VALUE_QUOTE_CHARS = new Set(['"', "'", "`"]); +const FORM_BODY_LINE_BREAK_SPLIT_RE = /(\r\n|\r|\n)/u; +const FORM_BODY_LINE_BREAK_SEGMENT_RE = /^(?:\r\n|\r|\n)$/u; const PAYMENT_CREDENTIAL_JSON_KEYS = String.raw`cardNumber|card_number|cardCvc|card_cvc|cardCvv|card_cvv|cvc|cvv|securityCode|security_code|paymentCredential|payment_credential|sharedPaymentToken|shared_payment_token`; const STRUCTURED_SECRET_FIELD_RE = new RegExp( - String.raw`^(?:api[-_]?key|apiKey|token|secret|password|passwd|access[-_]?token|accessToken|refresh[-_]?token|refreshToken|id[-_]?token|idToken|auth[-_]?token|authToken|client[-_]?secret|clientSecret|app[-_]?secret|appSecret|${PAYMENT_CREDENTIAL_QUERY_KEYS}|${PAYMENT_CREDENTIAL_JSON_KEYS})$`, + String.raw`^(?:api[-_]?key|apiKey|token|secret|password|passwd|credential|authorization|private[-_]?key|privateKey|access[-_]?token|accessToken|refresh[-_]?token|refreshToken|id[-_]?token|idToken|auth[-_]?token|authToken|client[-_]?secret|clientSecret|app[-_]?secret|appSecret|secret[-_]?value|secretValue|raw[-_]?secret|rawSecret|secret[-_]?input|secretInput|key[-_]?material|keyMaterial|${PAYMENT_CREDENTIAL_QUERY_KEYS}|${PAYMENT_CREDENTIAL_JSON_KEYS})$`, "i", ); const STRUCTURED_APP_PASSWORD_FIELD_RE = @@ -42,13 +113,26 @@ const STRUCTURED_SECRET_ENV_FIELD_RE = new RegExp( const ENV_ASSIGNMENT_REDACT_PATTERN = String.raw`/\b[A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|${PAYMENT_CREDENTIAL_ENV_KEYS})\b\s*[=:]\s*(["']?)([^\s"'\\]+)\1/g`; const ESCAPED_ENV_ASSIGNMENT_REDACT_PATTERN = String.raw`/\b[A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|${PAYMENT_CREDENTIAL_ENV_KEYS})\b\s*[=:]\s*\\+(["'])([^\s"'\\]+)\\+\1/g`; -const STANDALONE_ASSIGNMENT_REDACT_PATTERN = String.raw`(^|[\s,;])(?:access_token|refresh_token|auth[-_]?token|api[-_]?key|client[-_]?secret|app[-_]?secret|token|secret|password|passwd|${PAYMENT_CREDENTIAL_QUERY_KEYS})=([^\s&#]+)`; +// Quoted values may contain the other quote characters (`password="it's"`); only the matching +// closing quote ends the value. The unquoted variant accepts one leading quote so unterminated +// quotes still mask like plain values instead of escaping both patterns. +const STANDALONE_ASSIGNMENT_QUOTED_REDACT_PATTERN = String.raw`(^|[\s,;])(?:${STANDALONE_ASSIGNMENT_SECRET_KEYS})=(["'\x60])((?:(?!\2)[^\r\n])+)\2`; +const STANDALONE_ASSIGNMENT_REDACT_PATTERN = String.raw`(^|[\s,;])(?:${STANDALONE_ASSIGNMENT_SECRET_KEYS})=(["'\x60]?[^\s&#"'\x60<>]+)`; +// Pure-base64-alphabet token prefixes: require a non-alphanumeric left boundary (URL/path +// delimiters like `/` and `=` still qualify) but skip explicit `;base64,` payload spans, so +// data-URL media is never corrupted while tokens in URL paths or assignments still redact. +const BASE64_SAFE_TOKEN_BOUNDARY = String.raw`(^|[^A-Za-z0-9])(?(); +// Patterns whose left-context assertions (BASE64_SAFE_TOKEN_BOUNDARY) break under chunked +// replacement: a chunk start satisfies `^` and hides the `;base64,` container from the +// lookbehind, so these must always run against the full string. +const chunkUnsafePatterns = new WeakSet(); const DEFAULT_REDACT_PATTERNS: string[] = [ // ENV-style assignments. Keep this case-sensitive so diagnostics like @@ -57,42 +141,107 @@ const DEFAULT_REDACT_PATTERNS: string[] = [ ESCAPED_ENV_ASSIGNMENT_REDACT_PATTERN, // URL query parameters. Keep this separate from ENV-style assignments so // lower-case URL secrets stay redacted without hiding config-key diagnostics. - String.raw`/[?&](?:access[-_]?token|auth[-_]?token|hook[-_]?token|refresh[-_]?token|api[-_]?key|client[-_]?secret|token|key|secret|password|pass|passwd|auth|signature|${PAYMENT_CREDENTIAL_QUERY_KEYS})=([^&\s"'<>]+)/gi`, + String.raw`/[?&](?:${AUTH_QUERY_KEYS}|${PAYMENT_CREDENTIAL_QUERY_KEYS})=([^&#\s<>]+)/gi`, // JSON fields. - String.raw`"(?:apiKey|token|secret|password|passwd|accessToken|refreshToken|${PAYMENT_CREDENTIAL_JSON_KEYS})"\s*:\s*"([^"]+)"`, + String.raw`"(?:apiKey|api_key|token|secret|password|passwd|credential|authorization|accessToken|access_token|refreshToken|refresh_token|idToken|id_token|authToken|auth_token|clientSecret|client_secret|privateKey|private_key|secret_value|raw_secret|secret_input|key_material|${PAYMENT_CREDENTIAL_JSON_KEYS})"\s*:\s*"([^"]+)"`, // HTTP client diagnostics often stringify request config objects using // JSON or util.inspect-style fields rather than env/CLI syntax. - String.raw`(^|[\s,{])["']?(?:api[-_]key|access[-_]token|refresh[-_]token|authToken|auth[-_]token|clientSecret|client[-_]secret|appSecret|app[-_]secret)["']?\s*[:=]\s*(["'])([^"'\r\n]+)\2`, + String.raw`(^|[\s,{])["']?(?:api[-_]key|access[-_]token|refresh[-_]token|id[-_]token|authToken|auth[-_]token|clientSecret|client[-_]secret|appSecret|app[-_]secret|private[-_]key|credential|authorization|secret[-_]value|raw[-_]secret|secret[-_]input|key[-_]material)["']?\s*[:=]\s*(["'])([^"'\r\n]+)\2`, String.raw`(^|[\s,{])["']?(?:authorization|proxy-authorization|cookie|set-cookie|x-api-key|x-auth-token)["']?\s*[:=]\s*(["'])([^"'\r\n]+)\2`, // CLI flags. - String.raw`--(?:api[-_]?key|hook[-_]?token|token|secret|password|passwd|${PAYMENT_CREDENTIAL_QUERY_KEYS})\s+(["']?)([^\s"']+)\1`, + String.raw`--(?:api[-_]?key|hook[-_]?token|access[-_]?token|refresh[-_]?token|id[-_]?token|token|secret|password|passwd|credential|private[-_]?key|client[-_]?secret|${PAYMENT_CREDENTIAL_QUERY_KEYS})\s+(?!(?:or|and)\b(?=\s+--))(["']?)([^\s"']+)\1`, // Authorization headers. String.raw`Authorization\s*[:=]\s*Bearer\s+([A-Za-z0-9._\-+=]+)`, String.raw`Authorization\s*[:=]\s*Basic\s+([A-Za-z0-9+/=]+)`, + String.raw`Authorization\s*[:=]\s*Bot\s+([A-Za-z0-9._\-+=]{18,})`, String.raw`(?:X-OpenClaw-Token|x-pomerium-jwt-assertion|X-Api-Key|X-Auth-Token)\s*[:=]\s*([^\s"',;]+)`, String.raw`\bBearer\s+([A-Za-z0-9._\-+=]{18,})\b`, + // URL userinfo and common connection-string password slots. + String.raw`\b(?:https?|wss?|ftp):\/\/[^\/\s:@]*:([^\/\s@]+)@`, + String.raw`\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|rediss?|amqps?):\/\/[^:\s/@]*:([^@\s]+)@`, + // First pair in form-urlencoded bodies embedded in larger log lines. + String.raw`(^|[\s,;])(?:${FORM_BODY_FIRST_PAIR_KEYS})=([^&\s]+)(?=&[A-Za-z_][A-Za-z0-9_.-]*=)`, // Standalone token assignments in CLI or HTTP diagnostics. URL query params // are handled above so non-secret params survive and long values stay hinted. + STANDALONE_ASSIGNMENT_QUOTED_REDACT_PATTERN, STANDALONE_ASSIGNMENT_REDACT_PATTERN, // PEM blocks. String.raw`-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]+?-----END [A-Z ]*PRIVATE KEY-----`, // Common token prefixes. String.raw`\b(sk-[A-Za-z0-9_-]{8,})\b`, - String.raw`(ghp_[A-Za-z0-9]{20,})`, - String.raw`(github_pat_[A-Za-z0-9_]{20,})`, + String.raw`(ghp_[A-Za-z0-9]{10,})`, + String.raw`(github_pat_[A-Za-z0-9_]{10,})`, + String.raw`(gho_[A-Za-z0-9]{10,})`, + String.raw`(ghu_[A-Za-z0-9]{10,})`, + String.raw`(ghs_[A-Za-z0-9]{10,})`, + String.raw`(ghr_[A-Za-z0-9]{10,})`, + String.raw`(glpat-[A-Za-z0-9._=\-]{20,})`, + String.raw`(gloas-[A-Fa-f0-9]{32,})`, String.raw`(xox[baprs]-[A-Za-z0-9-]{10,})`, String.raw`(xapp-[A-Za-z0-9-]{10,})`, + String.raw`(https:\/\/hooks\.slack\.com\/(?:services\/T[A-Z0-9]+\/B[A-Z0-9]+|workflows\/T[A-Z0-9]+\/A[A-Z0-9]+\/[0-9]{17,19})\/[A-Za-z0-9]{20,})`, + String.raw`(https:\/\/discord(?:app)?\.com\/api\/webhooks\/[0-9]{17,20}\/[A-Za-z0-9_-]{60,})`, + String.raw`discord(?:.|\n|\r){0,40}?\b([A-Za-z0-9_-]{24}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27})\b`, String.raw`(gsk_[A-Za-z0-9_-]{10,})`, String.raw`(AIza[0-9A-Za-z\-_]{20,})`, String.raw`(ya29\.[0-9A-Za-z_\-./+=]{10,})`, String.raw`(1//0[0-9A-Za-z_\-./+=]{10,})`, String.raw`(eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})`, String.raw`(pplx-[A-Za-z0-9_-]{10,})`, + String.raw`(fal_[A-Za-z0-9_-]{10,})`, + String.raw`(fc-[A-Za-z0-9]{10,})`, + String.raw`(bb_live_[A-Za-z0-9_-]{10,})`, + // Prefixes made only of standard-base64 characters need a non-base64 left boundary so they + // do not fire inside unrelated base64 blobs (e.g. data-URL media), corrupting the payload. + String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(gAAAA[A-Za-z0-9_=-]{20,})`, + String.raw`(sk_live_[A-Za-z0-9]{10,})`, + String.raw`(sk_test_[A-Za-z0-9]{10,})`, + String.raw`(rk_live_[A-Za-z0-9]{10,})`, + String.raw`(SG\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})`, String.raw`(npm_[A-Za-z0-9]{10,})`, + String.raw`(pypi-[A-Za-z0-9_-]{10,})`, + String.raw`(dop_v1_[A-Za-z0-9]{10,})`, + String.raw`(doo_v1_[A-Za-z0-9]{10,})`, + String.raw`(dor_v1_[A-Za-z0-9]{10,})`, + String.raw`(dp\.(?:ct|pt|sa|scim|audit)\.[A-Za-z0-9]{40,44})`, + String.raw`(dp\.st\.[A-Za-z0-9]{40,44})`, + String.raw`(dp\.st\.[a-z0-9_-]{2,35}\.[A-Za-z0-9]{40,44})`, + String.raw`(dckr_(?:pat|oat)_[A-Za-z0-9_-]{27,32})`, + String.raw`(bkua_[a-z0-9]{40})`, + String.raw`(CCIPAT_[A-Za-z0-9]{22}_[A-Fa-f0-9]{40})`, + String.raw`(sbp_[a-z0-9]{40})`, + String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(dapi[0-9a-f]{32}(?:-\d)?)`, + String.raw`(dd[pw]_[A-Za-z0-9]{36})`, + String.raw`(glsa_[A-Za-z0-9_]{41})`, + String.raw`(glc_eyJ[A-Za-z0-9+/=]{60,160})`, + String.raw`(nfp_[A-Za-z0-9_]{36})`, + String.raw`(CFPAT-[A-Za-z0-9_\-]{40,})`, + String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(ATCTT3xFfG[A-Za-z0-9+/=_-]+=[A-Za-z0-9]{8})`, + String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(ATATT[A-Za-z0-9+/=_-]+=[A-Za-z0-9]{8})`, + String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(ATBB[A-Za-z0-9_=.-]{16,})`, + String.raw`(BBDC-[A-Za-z0-9+/@_-]{40,50})`, + String.raw`(HRKU-AA[A-Za-z0-9_-]{20,})`, + String.raw`(pat-(?:eu|na)1-[A-Za-z0-9]{8}\-[A-Za-z0-9]{4}\-[A-Za-z0-9]{4}\-[A-Za-z0-9]{4}\-[A-Za-z0-9]{12})`, + String.raw`(apify_api_[A-Za-z0-9\-]{20,})`, + String.raw`(FlyV1 fm\d+_[A-Za-z0-9+/=,_-]{100,})`, + String.raw`(fio-u-[A-Za-z0-9_-]{40,})`, + String.raw`(^|[^A-Za-z0-9_])(am_[A-Za-z0-9_-]{10,})`, + String.raw`(^|[^A-Za-z0-9_])(sk_[A-Za-z0-9_]{10,})`, + String.raw`(tvly-[A-Za-z0-9]{10,})`, + String.raw`(exa_[A-Za-z0-9]{10,})`, + String.raw`(syt_[A-Za-z0-9]{10,})`, + String.raw`(retaindb_[A-Za-z0-9]{10,})`, + String.raw`(hsk-[A-Za-z0-9]{10,})`, + String.raw`(mem0_[A-Za-z0-9]{10,})`, + String.raw`(brv_[A-Za-z0-9]{10,})`, + String.raw`(xai-[A-Za-z0-9]{30,})`, // Additional access-key and token-style prefixes. + String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(AKIA[A-Z0-9]{16})`, + String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(ASIA[A-Z0-9]{16})`, String.raw`(AKID[A-Za-z0-9]{10,})`, String.raw`(LTAI[A-Za-z0-9]{10,})`, String.raw`(hf_[A-Za-z0-9]{10,})`, + String.raw`(api_org_[A-Za-z0-9]{20,})`, String.raw`(r8_[A-Za-z0-9]{10,})`, // Telegram Bot API URLs embed the token as `/bot/...` (no word-boundary before digits). String.raw`\bbot(\d{6,}:[A-Za-z0-9_-]{20,})\b`, @@ -100,8 +249,34 @@ const DEFAULT_REDACT_PATTERNS: string[] = [ ]; let defaultResolvedPatterns: RegExp[] | undefined; -const DEFAULT_REDACT_PREFILTER_RE = - /(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|AUTH|COOKIE|SIGNATURE|CARD|CVC|CVV|PAYMENT|PRIVATE KEY|[?&]pass=|security[-_]?code|securityCode|\bBearer\s+|sk-|ghp_|github_pat_|xox[baprs]-|xapp-|gsk_|AIza|ya29\.|1\/\/0|eyJ|pplx-|npm_|AKID|LTAI|hf_|r8_|\bbot\d{6,}:|\b\d{6,}:[A-Za-z0-9_-]{20,})/i; +// Fast-path gate: with no user-configured patterns, redactSensitiveText skips the full +// default-pattern walk unless one of these triggers matches. Every DEFAULT_REDACT_PATTERNS +// entry and sensitive form/URL key must stay reachable here — a missing trigger silently +// leaks that secret shape, so each family keeps a default-options fixture in redact.test.ts. +const DEFAULT_REDACT_PREFILTER_SOURCES: string[] = [ + // Sensitive key names shared by the env/JSON/query/form/header/assignment families. + String.raw`KEY|TOKEN|SECRET|PASSWORD|PASSWD|AUTH|COOKIE|SIGNATURE|CREDENTIAL|CARD|CVC|CVV|PAYMENT|PRIVATE KEY`, + String.raw`security[-_]?code|\bpass=|jwt=|session=|code=`, + String.raw`\bBearer\s+`, + // URL userinfo and connection-string password slots (`scheme://user:pass@host`). + String.raw`:\/\/[^\/\s:@]*:[^\/\s@]+@`, + // Vendor token prefixes and webhook hosts, ordered like DEFAULT_REDACT_PATTERNS. + String.raw`sk-|gh[opsur]_|github_pat_|glpat-|gloas-|xox[baprs]-|xapp-|hooks\.slack\.com|discord|gsk_|AIza|ya29\.|1\/\/0|eyJ|pplx-|fal_|fc-|bb_live_|gAAAA|[sr]k_(?:live|test)_|\bSG\.|npm_|pypi-|do[opr]_v1_|dp\.(?:ct|pt|sa|st|scim|audit)\.|dckr_|bkua_|CCIPAT_|sbp_|dapi[0-9a-f]|dd[pw]_|glsa_|nfp_|CFPAT-|ATCTT3|ATATT|ATBB|BBDC-|HRKU-|pat-(?:eu|na)1-|apify_api_|FlyV1|fio-u-|tvly-|exa_|syt_|retaindb_|mem0_|brv_|xai-`, + String.raw`(?:^|[^A-Za-z0-9_])(?:am_|sk_)`, + String.raw`A[KS]IA[A-Z0-9]|AKID|LTAI|hf_|api_org_|r8_`, + String.raw`\bbot\d{6,}:|\b\d{6,}:[A-Za-z0-9_-]{20,}`, + // Obfuscated form/URL keys: percent escapes can rewrite any key letter, while plus or + // invisible splices break the literal key-name triggers above mid-word. After a splice the + // tail may mix further splices with key characters (e.g. an interior plus a trailing + // filler), but at least one key character must follow a splice so bare `+=` or line-leading + // `===` separators do not trip the fast path. + String.raw`%[0-9A-Fa-f]{2}[A-Za-z0-9_%.-]*=`, + String.raw`(?:\+|[${FORM_BODY_KEY_INVISIBLE_CHARS}])(?:[${FORM_BODY_KEY_INVISIBLE_CHARS}+]*[A-Za-z0-9_%.-])+[${FORM_BODY_KEY_INVISIBLE_CHARS}+]*=`, +]; +const DEFAULT_REDACT_PREFILTER_RE = new RegExp( + `(?:${DEFAULT_REDACT_PREFILTER_SOURCES.join("|")})`, + "iu", +); export type RedactOptions = { mode?: RedactSensitiveMode; @@ -111,6 +286,7 @@ export type RedactOptions = { export type ResolvedRedactOptions = { mode: RedactSensitiveMode; patterns: RegExp[]; + redactFormBodies: boolean; }; function normalizeMode(value?: string): RedactSensitiveMode { @@ -137,6 +313,9 @@ function parsePattern(raw: RedactPattern): RegExp | null { if (pattern && typeof raw === "string" && SHELL_REFERENCE_PRESERVING_PATTERN_SOURCES.has(raw)) { shellReferencePreservingPatterns.add(pattern); } + if (pattern && typeof raw === "string" && raw.startsWith(BASE64_SAFE_TOKEN_BOUNDARY)) { + chunkUnsafePatterns.add(pattern); + } return pattern; } @@ -150,7 +329,18 @@ function resolvePatterns(value?: RedactPattern[]): RegExp[] { return value.map(parsePattern).filter((re): re is RegExp => Boolean(re)); } +function includesDefaultRedactPatterns(value?: RedactPattern[]): boolean { + if (!value?.length) { + return true; + } + const source = new Set(value.filter((pattern): pattern is string => typeof pattern === "string")); + return DEFAULT_REDACT_PATTERNS.every((pattern) => source.has(pattern)); +} + function maskToken(token: string): string { + if (token === "***") { + return token; + } if (token.length < DEFAULT_REDACT_MIN_LENGTH) { return "***"; } @@ -159,6 +349,337 @@ function maskToken(token: string): string { return `${start}…${end}`; } +function splitSecretValueForMask(token: string): { + maskable: string; + suffix: string; + maskStart: number; + maskEnd: number; +} { + const openingQuote = token[0] ?? ""; + if (SECRET_VALUE_QUOTE_CHARS.has(openingQuote)) { + const closingQuoteIndex = token.lastIndexOf(openingQuote); + if (closingQuoteIndex > 0) { + const suffix = token.slice(closingQuoteIndex + 1); + if (SECRET_VALUE_SUFFIX_RE.test(suffix)) { + return { + maskable: token.slice(1, closingQuoteIndex), + suffix, + maskStart: 0, + maskEnd: closingQuoteIndex + 1, + }; + } + } + + const tokenWithoutLeadingQuote = token.slice(1); + const trailingDelimiter = + tokenWithoutLeadingQuote.match(SECRET_VALUE_TRAILING_DELIMITER_RE)?.[1] ?? ""; + const maskable = + trailingDelimiter && trailingDelimiter.length < tokenWithoutLeadingQuote.length + ? tokenWithoutLeadingQuote.slice(0, -trailingDelimiter.length) + : tokenWithoutLeadingQuote; + return { + maskable, + suffix: + trailingDelimiter && trailingDelimiter.length < tokenWithoutLeadingQuote.length + ? trailingDelimiter + : "", + maskStart: 0, + maskEnd: 1 + maskable.length, + }; + } + + const trailingDelimiter = token.match(SECRET_VALUE_TRAILING_DELIMITER_RE)?.[1] ?? ""; + const maskable = + trailingDelimiter && trailingDelimiter.length < token.length + ? token.slice(0, -trailingDelimiter.length) + : token; + return { + maskable, + suffix: maskable === token ? "" : trailingDelimiter, + maskStart: 0, + maskEnd: maskable.length, + }; +} + +function maskSecretValue(token: string, options?: { hinted?: boolean }): string { + const { maskable, suffix } = splitSecretValueForMask(token); + return `${options?.hinted ? maskToken(maskable) : "***"}${suffix}`; +} + +function normalizeSensitiveKeyName(value: string): string { + const stripped = value.replace(FORM_BODY_KEY_SEPARATOR_RE, ""); + try { + return decodeURIComponent(stripped) + .replace(FORM_BODY_KEY_SEPARATOR_RE, "") + .toLowerCase() + .replaceAll("-", "_"); + } catch { + return stripped.toLowerCase().replaceAll("-", "_"); + } +} + +function isSensitiveBodyKey(key: string): boolean { + return BODY_SECRET_KEYS.has(normalizeSensitiveKeyName(key)); +} + +function hasEncodedOrInvisibleFormKey(key: string): boolean { + return ( + FORM_BODY_PERCENT_ESCAPE_RE.test(key) || key.replace(FORM_BODY_KEY_OBFUSCATION_RE, "") !== key + ); +} + +function redactFormEncodedPairs( + value: string, + options?: { maskValues?: "fixed" | "hinted"; onlyEncodedOrInvisibleKeys?: boolean }, +): string { + return value + .split("&") + .map((pair) => { + const equalsIndex = pair.indexOf("="); + if (equalsIndex < 0) { + return pair; + } + const key = pair.slice(0, equalsIndex); + if (options?.onlyEncodedOrInvisibleKeys && !hasEncodedOrInvisibleFormKey(key)) { + return pair; + } + if (!isSensitiveBodyKey(key)) { + return pair; + } + const token = pair.slice(equalsIndex + 1); + const masked = maskSecretValue(token, { hinted: options?.maskValues === "hinted" }); + return `${key}=${masked}`; + }) + .join("&"); +} + +function markBitmapRange(bitmap: boolean[], start: number, end: number): void { + const boundedStart = Math.max(0, start); + const boundedEnd = Math.min(bitmap.length, end); + for (let i = boundedStart; i < boundedEnd; i++) { + bitmap[i] = true; + } +} + +function markSensitiveFormEncodedPairValues( + bitmap: boolean[], + value: string, + offset: number, + options?: { onlyEncodedOrInvisibleKeys?: boolean }, +): void { + let cursor = 0; + for (const pair of value.split("&")) { + const pairStart = cursor; + const pairEnd = pairStart + pair.length; + cursor = pairEnd + 1; + + const equalsIndex = pair.indexOf("="); + if (equalsIndex < 0) { + continue; + } + const key = pair.slice(0, equalsIndex); + if (options?.onlyEncodedOrInvisibleKeys && !hasEncodedOrInvisibleFormKey(key)) { + continue; + } + if (!isSensitiveBodyKey(key)) { + continue; + } + + const token = pair.slice(equalsIndex + 1); + const secretValue = splitSecretValueForMask(token); + const valueStart = pairStart + equalsIndex + 1 + secretValue.maskStart; + const valueEnd = pairStart + equalsIndex + 1 + secretValue.maskEnd; + markBitmapRange(bitmap, offset + valueStart, offset + valueEnd); + } +} + +function redactUrlQueryPairs(text: string): string { + if (!text || !text.includes("?")) { + return text; + } + return text.replace(URL_QUERY_PAIR_RE, (match, prefix: string, key: string, token: string) => { + if (!hasEncodedOrInvisibleFormKey(key) || !isSensitiveBodyKey(key)) { + return match; + } + return `${prefix}${key}=${maskSecretValue(token, { hinted: true })}`; + }); +} + +function markUrlQueryPairRedactions(text: string, bitmap: boolean[]): void { + if (!text || !text.includes("?")) { + return; + } + for (const match of text.matchAll(URL_QUERY_PAIR_RE)) { + if (match.index === undefined) { + continue; + } + const prefix = match[1] ?? ""; + const key = match[2] ?? ""; + const token = match[3] ?? ""; + if (!hasEncodedOrInvisibleFormKey(key) || !isSensitiveBodyKey(key)) { + continue; + } + const secretValue = splitSecretValueForMask(token); + const valueOffset = match.index + prefix.length + key.length + 1; + markBitmapRange(bitmap, valueOffset + secretValue.maskStart, valueOffset + secretValue.maskEnd); + } +} + +function redactEncodedFormPairs(text: string): string { + if (!text || (!text.includes("%") && text.replace(FORM_BODY_KEY_OBFUSCATION_RE, "") === text)) { + return text; + } + return text.replace(ENCODED_FORM_PAIR_RE, (match, prefix: string, key: string, token: string) => { + if (!hasEncodedOrInvisibleFormKey(key) || !isSensitiveBodyKey(key)) { + return match; + } + return `${prefix}${key}=${maskSecretValue(token)}`; + }); +} + +function markEncodedFormPairRedactions(text: string, bitmap: boolean[], offset = 0): void { + if (!text || (!text.includes("%") && text.replace(FORM_BODY_KEY_OBFUSCATION_RE, "") === text)) { + return; + } + for (const match of text.matchAll(ENCODED_FORM_PAIR_RE)) { + if (match.index === undefined) { + continue; + } + const prefix = match[1] ?? ""; + const key = match[2] ?? ""; + const token = match[3] ?? ""; + if (!hasEncodedOrInvisibleFormKey(key) || !isSensitiveBodyKey(key)) { + continue; + } + const secretValue = splitSecretValueForMask(token); + const valueOffset = match.index + prefix.length + key.length + 1; + markBitmapRange( + bitmap, + offset + valueOffset + secretValue.maskStart, + offset + valueOffset + secretValue.maskEnd, + ); + } +} + +function redactFormBodyContextSinglePairs(text: string): string { + if (!text || !/[=:]/u.test(text)) { + return text; + } + return text.replace( + FORM_BODY_CONTEXT_SINGLE_PAIR_RE, + (match, prefix: string, _quote: string, key: string, token: string, suffix: string) => { + if (!isSensitiveBodyKey(key)) { + return match; + } + return `${prefix}${key}=${maskSecretValue(token)}${suffix}`; + }, + ); +} + +function markFormBodyContextSinglePairRedactions( + text: string, + bitmap: boolean[], + offset = 0, +): void { + if (!text || !/[=:]/u.test(text)) { + return; + } + for (const match of text.matchAll(FORM_BODY_CONTEXT_SINGLE_PAIR_RE)) { + if (match.index === undefined) { + continue; + } + const prefix = match[1] ?? ""; + const key = match[3] ?? ""; + const token = match[4] ?? ""; + if (!isSensitiveBodyKey(key)) { + continue; + } + const secretValue = splitSecretValueForMask(token); + const valueOffset = match.index + prefix.length + key.length + 1; + markBitmapRange( + bitmap, + offset + valueOffset + secretValue.maskStart, + offset + valueOffset + secretValue.maskEnd, + ); + } +} + +function redactFormBodyLine(text: string): string { + if (!text) { + return text; + } + const contextRedacted = redactFormBodyContextSinglePairs(redactEncodedFormPairs(text)); + if (!contextRedacted.includes("&")) { + return contextRedacted; + } + if (FORM_BODY_RE.test(contextRedacted)) { + return redactFormEncodedPairs(contextRedacted); + } + const redacted = contextRedacted.replace( + FORM_BODY_SUBSTRING_RE, + (match, prefix: string, body: string) => { + const redactedBody = redactFormEncodedPairs(body); + return redactedBody === body ? match : `${prefix}${redactedBody}`; + }, + ); + return redactFormBodyContextSinglePairs(redactEncodedFormPairs(redacted)); +} + +function redactFormBody(text: string): string { + if (!text) { + return text; + } + if (FORM_BODY_LINE_BREAK_SPLIT_RE.test(text)) { + return text + .split(FORM_BODY_LINE_BREAK_SPLIT_RE) + .map((segment) => + FORM_BODY_LINE_BREAK_SEGMENT_RE.test(segment) ? segment : redactFormBodyLine(segment), + ) + .join(""); + } + return redactFormBodyLine(text); +} + +function markFormBodyLineRedactions(text: string, bitmap: boolean[], offset: number): void { + if (!text) { + return; + } + markEncodedFormPairRedactions(text, bitmap, offset); + markFormBodyContextSinglePairRedactions(text, bitmap, offset); + if (!text.includes("&")) { + return; + } + if (FORM_BODY_RE.test(text)) { + markSensitiveFormEncodedPairValues(bitmap, text, offset); + return; + } + for (const match of text.matchAll(FORM_BODY_SUBSTRING_RE)) { + if (match.index === undefined) { + continue; + } + const prefix = match[1] ?? ""; + const body = match[2] ?? ""; + markSensitiveFormEncodedPairValues(bitmap, body, offset + match.index + prefix.length); + } +} + +function markFormBodyRedactions(text: string, bitmap: boolean[]): void { + if (!text) { + return; + } + if (!FORM_BODY_LINE_BREAK_SPLIT_RE.test(text)) { + markFormBodyLineRedactions(text, bitmap, 0); + return; + } + let offset = 0; + for (const segment of text.split(FORM_BODY_LINE_BREAK_SPLIT_RE)) { + if (!FORM_BODY_LINE_BREAK_SEGMENT_RE.test(segment)) { + markFormBodyLineRedactions(segment, bitmap, offset); + } + offset += segment.length; + } +} + function redactPemBlock(block: string): string { const lines = block.split(/\r?\n/).filter(Boolean); if (lines.length < 2) { @@ -192,36 +713,161 @@ function isEmptyShellParameterExpansionTail(token: string): boolean { return /^[-=?+]\}$/.test(token); } +function hasBackreferenceToGroup(pattern: RegExp, groupNumber: number): boolean { + return new RegExp(String.raw`\\${groupNumber}(?!\d)`).test(pattern.source); +} + +type SecretCaptureSelection = { + captureCount: number; + index: number; + value: string; +}; + +function selectSecretCapture(match: string, groups: string[]): SecretCaptureSelection { + const tokens = groups + .map((value, index) => ({ index, value })) + .filter(({ value }) => typeof value === "string" && value.length > 0); + const selected = (tokens.length > 1 ? tokens[tokens.length - 1] : tokens[0]) ?? { + index: -1, + value: match, + }; + return { + ...selected, + captureCount: tokens.length, + }; +} + +function getIndexedCaptureStart( + pattern: RegExp, + input: string, + match: string, + matchOffset: number, + captureIndex: number, +): number | null { + if (matchOffset < 0 || !input) { + return null; + } + try { + const flags = pattern.flags.includes("d") ? pattern.flags : `${pattern.flags}d`; + const indexedPattern = new RegExp(pattern.source, flags); + indexedPattern.lastIndex = matchOffset; + const indexedMatch = indexedPattern.exec(input) as + | (RegExpExecArray & { indices?: Array<[number, number] | undefined> }) + | null; + const captureIndices = indexedMatch?.indices?.[captureIndex + 1]; + if (!indexedMatch || indexedMatch.index !== matchOffset || indexedMatch[0] !== match) { + return null; + } + if (!captureIndices) { + return null; + } + return captureIndices[0] - matchOffset; + } catch { + return null; + } +} + +function getSecretCaptureStart( + pattern: RegExp, + input: string, + match: string, + matchOffset: number, + selected: SecretCaptureSelection, +): number { + const indexedTokenStart = getIndexedCaptureStart( + pattern, + input, + match, + matchOffset, + selected.index, + ); + const preferFirstCapture = + selected.captureCount === 1 && + selected.index >= 0 && + hasBackreferenceToGroup(pattern, selected.index + 1); + return ( + indexedTokenStart ?? + (preferFirstCapture ? match.indexOf(selected.value) : match.lastIndexOf(selected.value)) + ); +} + function redactMatch( match: string, groups: string[], - options: { preserveShellReferences?: boolean } = {}, + pattern: RegExp, + context?: { input?: string; offset?: number }, ): string { if (match.includes("PRIVATE KEY-----")) { return redactPemBlock(match); } - const token = groups.findLast((value) => typeof value === "string" && value.length > 0) ?? match; - const preserveShellReferences = - options.preserveShellReferences && - (shouldPreserveShellReferenceMatch(match, token) || isEmptyShellParameterExpansionTail(token)); - if (preserveShellReferences) { + const selected = selectSecretCapture(match, groups); + const token = selected.value; + // An earlier pass (form-body or quoted-assignment masking) may already have replaced this + // value with ***; re-masking would strip its quote wrapper around the placeholder. + if (splitSecretValueForMask(token).maskable === "***") { return match; } - const masked = maskToken(token); + const isShellReferencePattern = shellReferencePreservingPatterns.has(pattern); + // Preserve shell variable references (e.g. `MY_TOKEN=$MY_TOKEN`) for assignment patterns + // registered as shell-reference-preserving, so non-secret expansions that merely echo the + // assignment key are not masked. + if ( + isShellReferencePattern && + (shouldPreserveShellReferenceMatch(match, token) || isEmptyShellParameterExpansionTail(token)) + ) { + return match; + } + // Assignment values can legitimately include trailing shell/structural characters + // (e.g. `${VAR:-default}`); mask the captured token whole so those characters count toward the + // retained hint instead of being exposed by delimiter-aware masking. + const masked = isShellReferencePattern + ? maskToken(token) + : maskSecretValue(token, { hinted: true }); if (token === match) { return masked; } - return match.replace(token, masked); + const tokenIndex = getSecretCaptureStart( + pattern, + context?.input ?? "", + match, + context?.offset ?? -1, + selected, + ); + if (tokenIndex < 0) { + return match; + } + return `${match.slice(0, tokenIndex)}${masked}${match.slice(tokenIndex + token.length)}`; } -function redactText(text: string, patterns: RegExp[]): string { +function redactText( + text: string, + patterns: RegExp[], + options?: { redactFormBodies?: boolean }, +): string { let next = text; + if (options?.redactFormBodies) { + next = redactUrlQueryPairs(next); + next = redactFormBody(next); + } for (const pattern of patterns) { - next = replacePatternBounded(next, pattern, (...args: string[]) => - redactMatch(args[0], args.slice(1, -2), { - preserveShellReferences: shellReferencePreservingPatterns.has(pattern), - }), - ); + const replacer = (...args: unknown[]) => { + const hasNamedGroups = + args.length > 0 && + typeof args[args.length - 1] === "object" && + args[args.length - 1] !== null; + const inputIndex = hasNamedGroups ? args.length - 2 : args.length - 1; + const offsetIndex = inputIndex - 1; + const match = typeof args[0] === "string" ? args[0] : ""; + const groups = args + .slice(1, offsetIndex) + .map((value) => (typeof value === "string" ? value : "")); + const offset = typeof args[offsetIndex] === "number" ? args[offsetIndex] : -1; + const input = typeof args[inputIndex] === "string" ? args[inputIndex] : ""; + return redactMatch(match, groups, pattern, { input, offset }); + }; + next = chunkUnsafePatterns.has(pattern) + ? next.replace(pattern, replacer) + : replacePatternBounded(next, pattern, replacer); } return next; } @@ -230,13 +876,74 @@ function couldMatchDefaultRedactPatterns(text: string): boolean { return DEFAULT_REDACT_PREFILTER_RE.test(text); } +function cloneGlobalPattern(pattern: RegExp): RegExp { + return pattern.flags.includes("g") + ? new RegExp(pattern.source, pattern.flags) + : new RegExp(pattern.source, `${pattern.flags}g`); +} + +function markPatternMatchRedaction( + bitmap: boolean[], + input: string, + pattern: RegExp, + match: RegExpMatchArray, +): void { + if (match.index === undefined) { + return; + } + const fullMatch = match[0] ?? ""; + if (fullMatch.includes("PRIVATE KEY-----")) { + markBitmapRange(bitmap, match.index, match.index + fullMatch.length); + return; + } + const selected = selectSecretCapture( + fullMatch, + match.slice(1).map((value) => (typeof value === "string" ? value : "")), + ); + const tokenStart = + selected.value === fullMatch + ? 0 + : getSecretCaptureStart(pattern, input, fullMatch, match.index, selected); + if (tokenStart < 0) { + return; + } + const secretValue = splitSecretValueForMask(selected.value); + markBitmapRange( + bitmap, + match.index + tokenStart + secretValue.maskStart, + match.index + tokenStart + secretValue.maskEnd, + ); +} + +export function computeSensitiveRedactionBitmap( + text: string, + resolved: ResolvedRedactOptions, +): boolean[] { + const bitmap: boolean[] = Array.from({ length: text.length }, () => false); + if (resolved.mode === "off" || !resolved.patterns.length || !text) { + return bitmap; + } + if (resolved.redactFormBodies) { + markUrlQueryPairRedactions(text, bitmap); + markFormBodyRedactions(text, bitmap); + } + for (const pattern of resolved.patterns) { + for (const match of text.matchAll(cloneGlobalPattern(pattern))) { + markPatternMatchRedaction(bitmap, text, pattern, match); + } + } + return bitmap; +} + function looksLikeAppSpecificPassword(candidate: string): boolean { return candidate.split("-").every((part) => !BENIGN_APP_PASSWORD_WORDS.has(part.toLowerCase())); } function redactAppSpecificPasswords(text: string): string { return replacePatternBounded(text, APP_SPECIFIC_PASSWORD_RE, (match: string, token: string) => - looksLikeAppSpecificPassword(token) ? redactMatch(match, [token]) : match, + looksLikeAppSpecificPassword(token) + ? redactMatch(match, [token], APP_SPECIFIC_PASSWORD_RE) + : match, ); } @@ -255,11 +962,14 @@ export function resolveRedactOptions(options?: RedactOptions): ResolvedRedactOpt return { mode, patterns: [], + redactFormBodies: false, }; } + const patterns = resolvePatterns(resolved.patterns); return { mode, - patterns: resolvePatterns(resolved.patterns), + patterns, + redactFormBodies: patterns.length > 0 && includesDefaultRedactPatterns(resolved.patterns), }; } @@ -278,15 +988,11 @@ export function redactSensitiveText(text: string, options?: RedactOptions): stri if (!resolved.patterns.length) { return text; } - return redactText(text, resolved.patterns); + return redactText(text, resolved.patterns, { redactFormBodies: resolved.redactFormBodies }); } export function redactToolDetail(detail: string): string { - const resolved = resolveConfigRedaction(); - if (normalizeMode(resolved.mode) !== "tools") { - return detail; - } - return redactSensitiveText(detail, resolved); + return redactToolPayloadText(detail); } function resolveToolPayloadRedaction( @@ -330,7 +1036,9 @@ function redactSensitiveFieldValueWithOptions( if (resolved.mode === "off") { return value; } - const redacted = redactText(value, resolved.patterns); + const redacted = redactText(value, resolved.patterns, { + redactFormBodies: resolved.redactFormBodies, + }); const shouldRedactAppPassword = redacted !== value || STRUCTURED_APP_PASSWORD_FIELD_RE.test(key); if (shouldRedactAppPassword) { const appRedacted = redactAppSpecificPasswords(redacted); @@ -443,5 +1151,8 @@ export function redactSensitiveLines(lines: string[], resolved: ResolvedRedactOp if (resolved.mode === "off" || !resolved.patterns.length || lines.length === 0) { return lines; } - return redactText(lines.join("\n"), resolved.patterns).split("\n"); + const redactedLines = resolved.redactFormBodies + ? lines.map((line) => redactFormBody(redactUrlQueryPairs(line))) + : lines; + return redactText(redactedLines.join("\n"), resolved.patterns).split("\n"); }