Redact tool output secrets (#85196)

* redact tool output secrets

* Expand tool-output secret redaction

* fix(security): keep redaction prefilter in sync with expanded defaults

- build DEFAULT_REDACT_PREFILTER_RE from sources covering every default
  pattern family: new vendor prefixes, webhook hosts, bare query/form keys,
  userinfo/connection-string passwords, and percent/plus/invisible
  obfuscated keys (including trailing separator splices)
- run default-pattern redaction tests through the default options path and
  redact the vendor corpus per token so prefilter gaps fail tests
- fix quoted standalone assignment values containing the other quote char
  or an unterminated quote; never re-mask *** placeholders
- align net-policy URL query-name separator stripping with logging key
  normalization (Hangul fillers)

* fix(security): keep base64-prefix redaction out of media payloads

- pure-base64-alphabet token prefixes (gAAAA, AKIA, ASIA, dapi,
  ATCTT3xFfG, ATATT, ATBB) now require a non-alphanumeric left boundary,
  skip explicit ;base64, payload spans, and run unchunked so chunk starts
  cannot fake the boundary or hide the container from the lookbehind
- tokens after URL/path delimiters or assignments still mask; data-URL
  media survives redaction byte-identical (fixes chat media mirror CI)
- regression tests: tiny-PNG data URL, in-blob plus boundary,
  chunk-aligned large data URL, reset-path Fernet token, path AWS key

---------

Co-authored-by: Alex Knight <15041791+amknight@users.noreply.github.com>
This commit is contained in:
Alex Knight 2026-06-10 14:34:50 -07:00 committed by GitHub
parent 2d404f1b86
commit 3faf669801
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 1512 additions and 220 deletions

View file

@ -22,6 +22,24 @@ describe("redactSensitiveUrl", () => {
);
});
it("redacts encoded and invisible-spliced sensitive query param names", () => {
expect(
redactSensitiveUrl("https://example.com/mcp?client%5Fse%E2%80%8Bcret=secret&safe=value"),
).toBe("https://example.com/mcp?client_se%E2%80%8Bcret=***&safe=value");
});
it("redacts encoded sensitive query names with decoded whitespace and control separators", () => {
expect(
redactSensitiveUrl("https://example.com/mcp?client%5Fse%20cret=space&client%5Fse%00cret=nul"),
).toBe("https://example.com/mcp?client_se+cret=***&client_se%00cret=***");
});
it("redacts query names with plus-encoded separators", () => {
expect(redactSensitiveUrl("https://example.com/mcp?client_se+cret=secret&safe=value")).toBe(
"https://example.com/mcp?client_se+cret=***&safe=value",
);
});
it("keeps non-sensitive URLs unchanged", () => {
expect(redactSensitiveUrl("https://example.com/mcp?safe=value")).toBe(
"https://example.com/mcp?safe=value",
@ -36,6 +54,26 @@ describe("redactSensitiveUrlLikeString", () => {
);
});
it("redacts encoded and invisible-spliced query names in invalid URL-like strings", () => {
expect(
redactSensitiveUrlLikeString("//example.com/mcp?client%5Fse%E2%80%8Bcret=secret&safe=value"),
).toBe("//example.com/mcp?client%5Fse%E2%80%8Bcret=***&safe=value");
});
it("redacts encoded query names with decoded whitespace and control separators in invalid URL-like strings", () => {
expect(
redactSensitiveUrlLikeString(
"//example.com/mcp?client%5Fse%20cret=space&client%5Fse%00cret=nul",
),
).toBe("//example.com/mcp?client%5Fse%20cret=***&client%5Fse%00cret=***");
});
it("redacts plus-spliced query names in invalid URL-like strings", () => {
expect(redactSensitiveUrlLikeString("//example.com/mcp?client_se+cret=secret&safe=value")).toBe(
"//example.com/mcp?client_se+cret=***&safe=value",
);
});
it("redacts every URL-like userinfo occurrence in arbitrary text", () => {
expect(
redactSensitiveUrlLikeString(
@ -61,6 +99,17 @@ describe("isSensitiveUrlQueryParamName", () => {
expect(isSensitiveUrlQueryParamName("hook-token")).toBe(true);
expect(isSensitiveUrlQueryParamName("passwd")).toBe(true);
expect(isSensitiveUrlQueryParamName("signature")).toBe(true);
expect(isSensitiveUrlQueryParamName("code")).toBe(true);
expect(isSensitiveUrlQueryParamName("x-amz-signature")).toBe(true);
expect(isSensitiveUrlQueryParamName("X-Amz-Security-Token")).toBe(true);
expect(isSensitiveUrlQueryParamName("id_token")).toBe(true);
expect(isSensitiveUrlQueryParamName("app_secret")).toBe(true);
expect(isSensitiveUrlQueryParamName("client%5Fse\u200Bcret")).toBe(true);
expect(isSensitiveUrlQueryParamName("client%5Fse%20cret")).toBe(true);
expect(isSensitiveUrlQueryParamName("client%5Fse%00cret")).toBe(true);
expect(isSensitiveUrlQueryParamName("client_se+cret")).toBe(true);
expect(isSensitiveUrlQueryParamName("client_se\u3164cret")).toBe(true);
expect(isSensitiveUrlQueryParamName("credential")).toBe(true);
expect(isSensitiveUrlQueryParamName("safe")).toBe(false);
});
});

View file

@ -22,15 +22,39 @@ const SENSITIVE_URL_QUERY_PARAM_NAMES = new Set([
"pass",
"passwd",
"auth",
"jwt",
"session",
"id_token",
"code",
"client_secret",
"app_secret",
"hook_token",
"refresh_token",
"signature",
"x_amz_signature",
"x_amz_security_token",
"private_key",
"credential",
"authorization",
]);
// Keep in sync with FORM_BODY_KEY_SEPARATOR_RE in src/logging/redact.ts: Hangul fillers are
// category Lo, so \p{C}\p{Z} alone would let them splice sensitive key names.
const URL_QUERY_NAME_SEPARATOR_RE = /[\p{C}\p{Z}\u115F\u1160\u3164\uFFA0+]/gu;
function normalizeUrlQueryParamName(name: string): string {
const stripped = name.replace(URL_QUERY_NAME_SEPARATOR_RE, "");
try {
return normalizeLowercaseStringOrEmpty(
decodeURIComponent(stripped).replace(URL_QUERY_NAME_SEPARATOR_RE, ""),
).replaceAll("-", "_");
} catch {
return normalizeLowercaseStringOrEmpty(stripped).replaceAll("-", "_");
}
}
/** True for auth-like URL query parameter names that should be redacted. */
export function isSensitiveUrlQueryParamName(name: string): boolean {
const normalized = normalizeLowercaseStringOrEmpty(name).replaceAll("-", "_");
const normalized = normalizeUrlQueryParamName(name);
return SENSITIVE_URL_QUERY_PARAM_NAMES.has(normalized);
}

View file

@ -648,6 +648,6 @@ describe("gateway run option collisions", () => {
).rejects.toThrow("__exit__:1");
},
);
expect(runtimeErrors[0]).toContain("Use either --passw***d or --password-file.");
expect(runtimeErrors[0]).toContain("Use either --password or --password-file.");
});
});

View file

@ -104,6 +104,21 @@ describe("sanitizeExecApprovalDisplayText", () => {
expect(result).toContain("https://api.example.com");
});
it("masks newly added vendor token prefixes through the default redaction path", () => {
const token = "glpat-abcdefghijklmnopqrstuv";
const result = sanitizeExecApprovalDisplayText(`deploy --with ${token}`);
expect(result).not.toContain(token);
});
it("does not let contextual secret matches hide split-token bypass detection", () => {
const discordToken = `${"A".repeat(24)}.${"B".repeat(6)}.${"C".repeat(27)}`;
const cmd = `discord sk-abc123\u200B456789012345678 ${discordToken}`;
const result = sanitizeExecApprovalDisplayText(cmd);
expect(result).not.toContain("sk-abc123");
expect(result).not.toContain("456789012345678");
expect(result).not.toContain(discordToken);
});
it("keeps PEM private-key context visible when raw redaction already covers the key (not a bypass)", () => {
const cmd =
"echo -----BEGIN RSA PRIVATE KEY-----\nABCDEF0123456789abcdef\n-----END RSA PRIVATE KEY----- > key.pem";
@ -157,6 +172,41 @@ describe("sanitizeExecApprovalDisplayText", () => {
expect(result).not.toContain("456789012345678");
expect(result).toContain("remainder");
});
it("masks form body values whose sensitive key is spliced with an invisible character", () => {
const cmd = "client_id=visible&app_se\u200Bcret=opaque-app-secret&safe=value";
const result = sanitizeExecApprovalDisplayText(cmd);
expect(result).not.toContain("opaque-app-secret");
expect(result).toContain("client_id=visible");
expect(result).toContain("safe=value");
});
it("masks form body values whose encoded sensitive key is spliced with an invisible character", () => {
const cmd = "client_id=visible&client%5Fse\u200Bcret=oauth-secret&safe=value";
const result = sanitizeExecApprovalDisplayText(cmd);
expect(result).not.toContain("oauth-secret");
expect(result).toContain("client_id=visible");
expect(result).toContain("safe=value");
});
it("masks form body values whose sensitive key is spliced with a plus separator", () => {
const cmd = "client_id=visible&client_se+cret=oauth-secret&safe=value";
const result = sanitizeExecApprovalDisplayText(cmd);
expect(result).not.toContain("oauth-secret");
expect(result).toContain("client_id=visible");
expect(result).toContain("safe=value");
});
it("keeps parsed form-body secrets masked when a separate spliced token triggers bypass rendering", () => {
const cmd =
"client_id=visible&client%5Fsecret=oauth,secret&safe=1 echo sk-abc123\u200B456789012345678";
const result = sanitizeExecApprovalDisplayText(cmd);
expect(result).not.toContain("oauth,secret");
expect(result).not.toContain(",secret");
expect(result).not.toContain("456789012345678");
expect(result).toContain("client_id=visible");
expect(result).toContain("safe=1");
});
});
describe("sanitizeExecApprovalWarningText", () => {

View file

@ -1,5 +1,9 @@
// Sanitizes command text before it is displayed in approval prompts.
import { redactSensitiveText, resolveRedactOptions } from "../logging/redact.js";
import {
computeSensitiveRedactionBitmap,
redactSensitiveText,
resolveRedactOptions,
} from "../logging/redact.js";
import type { ExecApprovalRequestPayload } from "./exec-approvals.js";
// Escape control characters, Unicode format/line/paragraph separators, and non-ASCII space
@ -60,31 +64,6 @@ function truncateForDisplay(text: string): SanitizedExecApprovalDisplayText {
};
}
// Build a boolean bitmap of positions in `text` that ANY redaction pattern would match.
// Patterns are applied independently to the raw text (not sequentially against a
// progressively-redacted view) so later patterns can still find matches that the in-place
// redaction would have replaced first. That is conservative — it may over-count overlapping
// matches — but that is acceptable for a coverage check. Indices are UTF-16 code-unit
// offsets, matching what `matchAll` returns and aligning with `String#length`.
function computeRedactionBitmap(text: string, patterns: RegExp[]): boolean[] {
const bitmap: boolean[] = Array.from({ length: text.length }, () => false);
for (const pattern of patterns) {
const iter = pattern.flags.includes("g")
? new RegExp(pattern.source, pattern.flags)
: new RegExp(pattern.source, `${pattern.flags}g`);
for (const match of text.matchAll(iter)) {
if (match.index === undefined) {
continue;
}
const end = match.index + match[0].length;
for (let i = match.index; i < end; i++) {
bitmap[i] = true;
}
}
}
return bitmap;
}
// Iterate by full Unicode code point so astral-plane invisibles (e.g. U+E0061 TAG LATIN
// SMALL LETTER A, category Cf) are matched as single characters instead of being seen as a
// surrogate pair whose halves are category Cs and would escape the invisible-char regex.
@ -126,17 +105,16 @@ function sanitizeExecApprovalDisplayTextInternal(
if (strippedRedacted === stripped) {
return truncateForDisplay(escapeInvisibles(rawRedacted, options));
}
// Detect bypass by position-bitmap coverage. Run each redaction pattern independently on
// both views and map stripped-view match positions back to original coordinates. If every
// position the stripped view would mask is also masked by the raw view, the raw view
// already covered everything — for example, an ordinary multi-line PEM private key where
// raw produces `BEGIN/…redacted…/END` while stripped collapses to `***`. A real bypass
// exists only when the stripped view masks at least one original position raw missed (e.g.
// the tail of an `sk-` token whose prefix-boundary was broken by a spliced zero-width or
// NBSP character).
const { patterns } = resolveRedactOptions({ mode: "tools" });
const rawMask = computeRedactionBitmap(commandText, patterns);
const strippedMask = computeRedactionBitmap(stripped, patterns);
// Detect bypass by position-bitmap coverage. Run the redaction matchers on both views and
// map stripped-view match positions back to original coordinates. If every position the
// stripped view would mask is also masked by the raw view, the raw view already covered
// everything — for example, an ordinary multi-line PEM private key where raw produces
// `BEGIN/…redacted…/END` while stripped collapses to `***`. A real bypass exists only when
// the stripped view masks at least one original position raw missed (e.g. the tail of an
// `sk-` token whose prefix-boundary was broken by a spliced zero-width or NBSP character).
const redaction = resolveRedactOptions({ mode: "tools" });
const rawMask = computeSensitiveRedactionBitmap(commandText, redaction);
const strippedMask = computeSensitiveRedactionBitmap(stripped, redaction);
let bypassDetected = false;
for (let i = 0; i < strippedMask.length; i++) {
if (strippedMask[i] && !rawMask[strippedToOrig[i]]) {

File diff suppressed because it is too large Load diff

View file

@ -15,9 +15,80 @@ const DEFAULT_REDACT_KEEP_END = 4;
const PAYMENT_CREDENTIAL_ENV_KEYS = String.raw`CARD[_-]?NUMBER|CARD[_-]?CVC|CARD[_-]?CVV|CVC|CVV|SECURITY[_-]?CODE|PAYMENT[_-]?CREDENTIAL|SHARED[_-]?PAYMENT[_-]?TOKEN`;
const PAYMENT_CREDENTIAL_QUERY_KEYS = String.raw`card[-_]?number|card[-_]?cvc|card[-_]?cvv|cvc|cvv|security[-_]?code|payment[-_]?credential|shared[-_]?payment[-_]?token`;
const AUTH_QUERY_KEYS = String.raw`access[-_]?token|auth[-_]?token|hook[-_]?token|refresh[-_]?token|id[-_]?token|api[-_]?key|apikey|client[-_]?secret|app[-_]?secret|private[-_]?key|credential|authorization|token|key|secret|password|pass|passwd|auth|jwt|session|code|signature|x[-_]?amz[-_]?(?:signature|security[-_]?token)`;
const FORM_BODY_FIRST_PAIR_KEYS = String.raw`${AUTH_QUERY_KEYS}|app[-_]?secret|credential|${PAYMENT_CREDENTIAL_QUERY_KEYS}`;
const STANDALONE_ASSIGNMENT_SECRET_KEYS = String.raw`access_token|refresh_token|id_token|auth[-_]?token|hook[-_]?token|api[-_]?key|client[-_]?secret|app[-_]?secret|private[-_]?key|authorization|jwt|token|secret|password|pass|passwd|credential|${PAYMENT_CREDENTIAL_QUERY_KEYS}`;
const BODY_SECRET_KEYS = new Set([
"access_token",
"auth_token",
"hook_token",
"refresh_token",
"id_token",
"token",
"api_key",
"apikey",
"client_secret",
"app_secret",
"password",
"pass",
"passwd",
"auth",
"jwt",
"session",
"code",
"signature",
"x_amz_signature",
"x_amz_security_token",
"secret",
"credential",
"private_key",
"authorization",
"key",
"card_number",
"card_cvc",
"card_cvv",
"cvc",
"cvv",
"security_code",
"payment_credential",
"shared_payment_token",
]);
const FORM_BODY_KEY_INVISIBLE_CHARS = String.raw`\p{C}\u00A0\u1680\u2000-\u200A\u202F\u205F\u3000\u115F\u1160\u3164\uFFA0`;
const FORM_BODY_KEY_OBFUSCATION_RE = new RegExp(
String.raw`[${FORM_BODY_KEY_INVISIBLE_CHARS}+]`,
"gu",
);
const FORM_BODY_KEY_SEPARATOR_RE = /[\p{C}\p{Z}\u115F\u1160\u3164\uFFA0+]/gu;
const FORM_BODY_PERCENT_ESCAPE_RE = /%[0-9A-Fa-f]{2}/u;
const FORM_BODY_KEY = String.raw`[${FORM_BODY_KEY_INVISIBLE_CHARS}+]*(?:[A-Za-z_]|%[0-9A-Fa-f]{2})(?:[A-Za-z0-9_.-]|%[0-9A-Fa-f]{2}|[${FORM_BODY_KEY_INVISIBLE_CHARS}+])*`;
const FORM_BODY_VALUE = "[^&\\s<>]*";
const URL_QUERY_VALUE = "[^&#\\s<>]*";
const FORM_BODY_PAIR = String.raw`${FORM_BODY_KEY}=${FORM_BODY_VALUE}`;
const FORM_BODY_RE = new RegExp(String.raw`^${FORM_BODY_PAIR}(?:&${FORM_BODY_PAIR})+$`, "u");
const FORM_BODY_SUBSTRING_RE = new RegExp(
String.raw`(^|[\s:({\[,="'` + "`" + String.raw`])(${FORM_BODY_PAIR}(?:&${FORM_BODY_PAIR})+)`,
"gu",
);
const ENCODED_FORM_PAIR_RE = new RegExp(
String.raw`(^|[\s:({\[,="'` + "`" + String.raw`&])(${FORM_BODY_KEY})=(${FORM_BODY_VALUE})`,
"gu",
);
const FORM_BODY_CONTEXT_SINGLE_PAIR_RE = new RegExp(
String.raw`(\b(?:body|form(?:[-_\s]?body)?)\s*[:=]\s*(["'\x60]?))(${FORM_BODY_KEY})=(${FORM_BODY_VALUE})(["'\x60]?)`,
"giu",
);
const URL_QUERY_PAIR_RE = new RegExp(
String.raw`([?&])(${FORM_BODY_KEY})=(${URL_QUERY_VALUE})`,
"gu",
);
const SECRET_VALUE_TRAILING_DELIMITER_RE = /(["'`,;)}\]]+)$/u;
const SECRET_VALUE_SUFFIX_RE = /^["'`,;)}\]]*$/u;
const SECRET_VALUE_QUOTE_CHARS = new Set(['"', "'", "`"]);
const FORM_BODY_LINE_BREAK_SPLIT_RE = /(\r\n|\r|\n)/u;
const FORM_BODY_LINE_BREAK_SEGMENT_RE = /^(?:\r\n|\r|\n)$/u;
const PAYMENT_CREDENTIAL_JSON_KEYS = String.raw`cardNumber|card_number|cardCvc|card_cvc|cardCvv|card_cvv|cvc|cvv|securityCode|security_code|paymentCredential|payment_credential|sharedPaymentToken|shared_payment_token`;
const STRUCTURED_SECRET_FIELD_RE = new RegExp(
String.raw`^(?:api[-_]?key|apiKey|token|secret|password|passwd|access[-_]?token|accessToken|refresh[-_]?token|refreshToken|id[-_]?token|idToken|auth[-_]?token|authToken|client[-_]?secret|clientSecret|app[-_]?secret|appSecret|${PAYMENT_CREDENTIAL_QUERY_KEYS}|${PAYMENT_CREDENTIAL_JSON_KEYS})$`,
String.raw`^(?:api[-_]?key|apiKey|token|secret|password|passwd|credential|authorization|private[-_]?key|privateKey|access[-_]?token|accessToken|refresh[-_]?token|refreshToken|id[-_]?token|idToken|auth[-_]?token|authToken|client[-_]?secret|clientSecret|app[-_]?secret|appSecret|secret[-_]?value|secretValue|raw[-_]?secret|rawSecret|secret[-_]?input|secretInput|key[-_]?material|keyMaterial|${PAYMENT_CREDENTIAL_QUERY_KEYS}|${PAYMENT_CREDENTIAL_JSON_KEYS})$`,
"i",
);
const STRUCTURED_APP_PASSWORD_FIELD_RE =
@ -42,13 +113,26 @@ const STRUCTURED_SECRET_ENV_FIELD_RE = new RegExp(
const ENV_ASSIGNMENT_REDACT_PATTERN = String.raw`/\b[A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|${PAYMENT_CREDENTIAL_ENV_KEYS})\b\s*[=:]\s*(["']?)([^\s"'\\]+)\1/g`;
const ESCAPED_ENV_ASSIGNMENT_REDACT_PATTERN = String.raw`/\b[A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|${PAYMENT_CREDENTIAL_ENV_KEYS})\b\s*[=:]\s*\\+(["'])([^\s"'\\]+)\\+\1/g`;
const STANDALONE_ASSIGNMENT_REDACT_PATTERN = String.raw`(^|[\s,;])(?:access_token|refresh_token|auth[-_]?token|api[-_]?key|client[-_]?secret|app[-_]?secret|token|secret|password|passwd|${PAYMENT_CREDENTIAL_QUERY_KEYS})=([^\s&#]+)`;
// Quoted values may contain the other quote characters (`password="it's"`); only the matching
// closing quote ends the value. The unquoted variant accepts one leading quote so unterminated
// quotes still mask like plain values instead of escaping both patterns.
const STANDALONE_ASSIGNMENT_QUOTED_REDACT_PATTERN = String.raw`(^|[\s,;])(?:${STANDALONE_ASSIGNMENT_SECRET_KEYS})=(["'\x60])((?:(?!\2)[^\r\n])+)\2`;
const STANDALONE_ASSIGNMENT_REDACT_PATTERN = String.raw`(^|[\s,;])(?:${STANDALONE_ASSIGNMENT_SECRET_KEYS})=(["'\x60]?[^\s&#"'\x60<>]+)`;
// Pure-base64-alphabet token prefixes: require a non-alphanumeric left boundary (URL/path
// delimiters like `/` and `=` still qualify) but skip explicit `;base64,` payload spans, so
// data-URL media is never corrupted while tokens in URL paths or assignments still redact.
const BASE64_SAFE_TOKEN_BOUNDARY = String.raw`(^|[^A-Za-z0-9])(?<!;base64,[A-Za-z0-9+/=]*)`;
const SHELL_REFERENCE_PRESERVING_PATTERN_SOURCES = new Set([
ENV_ASSIGNMENT_REDACT_PATTERN,
ESCAPED_ENV_ASSIGNMENT_REDACT_PATTERN,
STANDALONE_ASSIGNMENT_QUOTED_REDACT_PATTERN,
STANDALONE_ASSIGNMENT_REDACT_PATTERN,
]);
const shellReferencePreservingPatterns = new WeakSet<RegExp>();
// Patterns whose left-context assertions (BASE64_SAFE_TOKEN_BOUNDARY) break under chunked
// replacement: a chunk start satisfies `^` and hides the `;base64,` container from the
// lookbehind, so these must always run against the full string.
const chunkUnsafePatterns = new WeakSet<RegExp>();
const DEFAULT_REDACT_PATTERNS: string[] = [
// ENV-style assignments. Keep this case-sensitive so diagnostics like
@ -57,42 +141,107 @@ const DEFAULT_REDACT_PATTERNS: string[] = [
ESCAPED_ENV_ASSIGNMENT_REDACT_PATTERN,
// URL query parameters. Keep this separate from ENV-style assignments so
// lower-case URL secrets stay redacted without hiding config-key diagnostics.
String.raw`/[?&](?:access[-_]?token|auth[-_]?token|hook[-_]?token|refresh[-_]?token|api[-_]?key|client[-_]?secret|token|key|secret|password|pass|passwd|auth|signature|${PAYMENT_CREDENTIAL_QUERY_KEYS})=([^&\s"'<>]+)/gi`,
String.raw`/[?&](?:${AUTH_QUERY_KEYS}|${PAYMENT_CREDENTIAL_QUERY_KEYS})=([^&#\s<>]+)/gi`,
// JSON fields.
String.raw`"(?:apiKey|token|secret|password|passwd|accessToken|refreshToken|${PAYMENT_CREDENTIAL_JSON_KEYS})"\s*:\s*"([^"]+)"`,
String.raw`"(?:apiKey|api_key|token|secret|password|passwd|credential|authorization|accessToken|access_token|refreshToken|refresh_token|idToken|id_token|authToken|auth_token|clientSecret|client_secret|privateKey|private_key|secret_value|raw_secret|secret_input|key_material|${PAYMENT_CREDENTIAL_JSON_KEYS})"\s*:\s*"([^"]+)"`,
// HTTP client diagnostics often stringify request config objects using
// JSON or util.inspect-style fields rather than env/CLI syntax.
String.raw`(^|[\s,{])["']?(?:api[-_]key|access[-_]token|refresh[-_]token|authToken|auth[-_]token|clientSecret|client[-_]secret|appSecret|app[-_]secret)["']?\s*[:=]\s*(["'])([^"'\r\n]+)\2`,
String.raw`(^|[\s,{])["']?(?:api[-_]key|access[-_]token|refresh[-_]token|id[-_]token|authToken|auth[-_]token|clientSecret|client[-_]secret|appSecret|app[-_]secret|private[-_]key|credential|authorization|secret[-_]value|raw[-_]secret|secret[-_]input|key[-_]material)["']?\s*[:=]\s*(["'])([^"'\r\n]+)\2`,
String.raw`(^|[\s,{])["']?(?:authorization|proxy-authorization|cookie|set-cookie|x-api-key|x-auth-token)["']?\s*[:=]\s*(["'])([^"'\r\n]+)\2`,
// CLI flags.
String.raw`--(?:api[-_]?key|hook[-_]?token|token|secret|password|passwd|${PAYMENT_CREDENTIAL_QUERY_KEYS})\s+(["']?)([^\s"']+)\1`,
String.raw`--(?:api[-_]?key|hook[-_]?token|access[-_]?token|refresh[-_]?token|id[-_]?token|token|secret|password|passwd|credential|private[-_]?key|client[-_]?secret|${PAYMENT_CREDENTIAL_QUERY_KEYS})\s+(?!(?:or|and)\b(?=\s+--))(["']?)([^\s"']+)\1`,
// Authorization headers.
String.raw`Authorization\s*[:=]\s*Bearer\s+([A-Za-z0-9._\-+=]+)`,
String.raw`Authorization\s*[:=]\s*Basic\s+([A-Za-z0-9+/=]+)`,
String.raw`Authorization\s*[:=]\s*Bot\s+([A-Za-z0-9._\-+=]{18,})`,
String.raw`(?:X-OpenClaw-Token|x-pomerium-jwt-assertion|X-Api-Key|X-Auth-Token)\s*[:=]\s*([^\s"',;]+)`,
String.raw`\bBearer\s+([A-Za-z0-9._\-+=]{18,})\b`,
// URL userinfo and common connection-string password slots.
String.raw`\b(?:https?|wss?|ftp):\/\/[^\/\s:@]*:([^\/\s@]+)@`,
String.raw`\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|rediss?|amqps?):\/\/[^:\s/@]*:([^@\s]+)@`,
// First pair in form-urlencoded bodies embedded in larger log lines.
String.raw`(^|[\s,;])(?:${FORM_BODY_FIRST_PAIR_KEYS})=([^&\s]+)(?=&[A-Za-z_][A-Za-z0-9_.-]*=)`,
// Standalone token assignments in CLI or HTTP diagnostics. URL query params
// are handled above so non-secret params survive and long values stay hinted.
STANDALONE_ASSIGNMENT_QUOTED_REDACT_PATTERN,
STANDALONE_ASSIGNMENT_REDACT_PATTERN,
// PEM blocks.
String.raw`-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]+?-----END [A-Z ]*PRIVATE KEY-----`,
// Common token prefixes.
String.raw`\b(sk-[A-Za-z0-9_-]{8,})\b`,
String.raw`(ghp_[A-Za-z0-9]{20,})`,
String.raw`(github_pat_[A-Za-z0-9_]{20,})`,
String.raw`(ghp_[A-Za-z0-9]{10,})`,
String.raw`(github_pat_[A-Za-z0-9_]{10,})`,
String.raw`(gho_[A-Za-z0-9]{10,})`,
String.raw`(ghu_[A-Za-z0-9]{10,})`,
String.raw`(ghs_[A-Za-z0-9]{10,})`,
String.raw`(ghr_[A-Za-z0-9]{10,})`,
String.raw`(glpat-[A-Za-z0-9._=\-]{20,})`,
String.raw`(gloas-[A-Fa-f0-9]{32,})`,
String.raw`(xox[baprs]-[A-Za-z0-9-]{10,})`,
String.raw`(xapp-[A-Za-z0-9-]{10,})`,
String.raw`(https:\/\/hooks\.slack\.com\/(?:services\/T[A-Z0-9]+\/B[A-Z0-9]+|workflows\/T[A-Z0-9]+\/A[A-Z0-9]+\/[0-9]{17,19})\/[A-Za-z0-9]{20,})`,
String.raw`(https:\/\/discord(?:app)?\.com\/api\/webhooks\/[0-9]{17,20}\/[A-Za-z0-9_-]{60,})`,
String.raw`discord(?:.|\n|\r){0,40}?\b([A-Za-z0-9_-]{24}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27})\b`,
String.raw`(gsk_[A-Za-z0-9_-]{10,})`,
String.raw`(AIza[0-9A-Za-z\-_]{20,})`,
String.raw`(ya29\.[0-9A-Za-z_\-./+=]{10,})`,
String.raw`(1//0[0-9A-Za-z_\-./+=]{10,})`,
String.raw`(eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})`,
String.raw`(pplx-[A-Za-z0-9_-]{10,})`,
String.raw`(fal_[A-Za-z0-9_-]{10,})`,
String.raw`(fc-[A-Za-z0-9]{10,})`,
String.raw`(bb_live_[A-Za-z0-9_-]{10,})`,
// Prefixes made only of standard-base64 characters need a non-base64 left boundary so they
// do not fire inside unrelated base64 blobs (e.g. data-URL media), corrupting the payload.
String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(gAAAA[A-Za-z0-9_=-]{20,})`,
String.raw`(sk_live_[A-Za-z0-9]{10,})`,
String.raw`(sk_test_[A-Za-z0-9]{10,})`,
String.raw`(rk_live_[A-Za-z0-9]{10,})`,
String.raw`(SG\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})`,
String.raw`(npm_[A-Za-z0-9]{10,})`,
String.raw`(pypi-[A-Za-z0-9_-]{10,})`,
String.raw`(dop_v1_[A-Za-z0-9]{10,})`,
String.raw`(doo_v1_[A-Za-z0-9]{10,})`,
String.raw`(dor_v1_[A-Za-z0-9]{10,})`,
String.raw`(dp\.(?:ct|pt|sa|scim|audit)\.[A-Za-z0-9]{40,44})`,
String.raw`(dp\.st\.[A-Za-z0-9]{40,44})`,
String.raw`(dp\.st\.[a-z0-9_-]{2,35}\.[A-Za-z0-9]{40,44})`,
String.raw`(dckr_(?:pat|oat)_[A-Za-z0-9_-]{27,32})`,
String.raw`(bkua_[a-z0-9]{40})`,
String.raw`(CCIPAT_[A-Za-z0-9]{22}_[A-Fa-f0-9]{40})`,
String.raw`(sbp_[a-z0-9]{40})`,
String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(dapi[0-9a-f]{32}(?:-\d)?)`,
String.raw`(dd[pw]_[A-Za-z0-9]{36})`,
String.raw`(glsa_[A-Za-z0-9_]{41})`,
String.raw`(glc_eyJ[A-Za-z0-9+/=]{60,160})`,
String.raw`(nfp_[A-Za-z0-9_]{36})`,
String.raw`(CFPAT-[A-Za-z0-9_\-]{40,})`,
String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(ATCTT3xFfG[A-Za-z0-9+/=_-]+=[A-Za-z0-9]{8})`,
String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(ATATT[A-Za-z0-9+/=_-]+=[A-Za-z0-9]{8})`,
String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(ATBB[A-Za-z0-9_=.-]{16,})`,
String.raw`(BBDC-[A-Za-z0-9+/@_-]{40,50})`,
String.raw`(HRKU-AA[A-Za-z0-9_-]{20,})`,
String.raw`(pat-(?:eu|na)1-[A-Za-z0-9]{8}\-[A-Za-z0-9]{4}\-[A-Za-z0-9]{4}\-[A-Za-z0-9]{4}\-[A-Za-z0-9]{12})`,
String.raw`(apify_api_[A-Za-z0-9\-]{20,})`,
String.raw`(FlyV1 fm\d+_[A-Za-z0-9+/=,_-]{100,})`,
String.raw`(fio-u-[A-Za-z0-9_-]{40,})`,
String.raw`(^|[^A-Za-z0-9_])(am_[A-Za-z0-9_-]{10,})`,
String.raw`(^|[^A-Za-z0-9_])(sk_[A-Za-z0-9_]{10,})`,
String.raw`(tvly-[A-Za-z0-9]{10,})`,
String.raw`(exa_[A-Za-z0-9]{10,})`,
String.raw`(syt_[A-Za-z0-9]{10,})`,
String.raw`(retaindb_[A-Za-z0-9]{10,})`,
String.raw`(hsk-[A-Za-z0-9]{10,})`,
String.raw`(mem0_[A-Za-z0-9]{10,})`,
String.raw`(brv_[A-Za-z0-9]{10,})`,
String.raw`(xai-[A-Za-z0-9]{30,})`,
// Additional access-key and token-style prefixes.
String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(AKIA[A-Z0-9]{16})`,
String.raw`${BASE64_SAFE_TOKEN_BOUNDARY}(ASIA[A-Z0-9]{16})`,
String.raw`(AKID[A-Za-z0-9]{10,})`,
String.raw`(LTAI[A-Za-z0-9]{10,})`,
String.raw`(hf_[A-Za-z0-9]{10,})`,
String.raw`(api_org_[A-Za-z0-9]{20,})`,
String.raw`(r8_[A-Za-z0-9]{10,})`,
// Telegram Bot API URLs embed the token as `/bot<token>/...` (no word-boundary before digits).
String.raw`\bbot(\d{6,}:[A-Za-z0-9_-]{20,})\b`,
@ -100,8 +249,34 @@ const DEFAULT_REDACT_PATTERNS: string[] = [
];
let defaultResolvedPatterns: RegExp[] | undefined;
const DEFAULT_REDACT_PREFILTER_RE =
/(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|AUTH|COOKIE|SIGNATURE|CARD|CVC|CVV|PAYMENT|PRIVATE KEY|[?&]pass=|security[-_]?code|securityCode|\bBearer\s+|sk-|ghp_|github_pat_|xox[baprs]-|xapp-|gsk_|AIza|ya29\.|1\/\/0|eyJ|pplx-|npm_|AKID|LTAI|hf_|r8_|\bbot\d{6,}:|\b\d{6,}:[A-Za-z0-9_-]{20,})/i;
// Fast-path gate: with no user-configured patterns, redactSensitiveText skips the full
// default-pattern walk unless one of these triggers matches. Every DEFAULT_REDACT_PATTERNS
// entry and sensitive form/URL key must stay reachable here — a missing trigger silently
// leaks that secret shape, so each family keeps a default-options fixture in redact.test.ts.
const DEFAULT_REDACT_PREFILTER_SOURCES: string[] = [
// Sensitive key names shared by the env/JSON/query/form/header/assignment families.
String.raw`KEY|TOKEN|SECRET|PASSWORD|PASSWD|AUTH|COOKIE|SIGNATURE|CREDENTIAL|CARD|CVC|CVV|PAYMENT|PRIVATE KEY`,
String.raw`security[-_]?code|\bpass=|jwt=|session=|code=`,
String.raw`\bBearer\s+`,
// URL userinfo and connection-string password slots (`scheme://user:pass@host`).
String.raw`:\/\/[^\/\s:@]*:[^\/\s@]+@`,
// Vendor token prefixes and webhook hosts, ordered like DEFAULT_REDACT_PATTERNS.
String.raw`sk-|gh[opsur]_|github_pat_|glpat-|gloas-|xox[baprs]-|xapp-|hooks\.slack\.com|discord|gsk_|AIza|ya29\.|1\/\/0|eyJ|pplx-|fal_|fc-|bb_live_|gAAAA|[sr]k_(?:live|test)_|\bSG\.|npm_|pypi-|do[opr]_v1_|dp\.(?:ct|pt|sa|st|scim|audit)\.|dckr_|bkua_|CCIPAT_|sbp_|dapi[0-9a-f]|dd[pw]_|glsa_|nfp_|CFPAT-|ATCTT3|ATATT|ATBB|BBDC-|HRKU-|pat-(?:eu|na)1-|apify_api_|FlyV1|fio-u-|tvly-|exa_|syt_|retaindb_|mem0_|brv_|xai-`,
String.raw`(?:^|[^A-Za-z0-9_])(?:am_|sk_)`,
String.raw`A[KS]IA[A-Z0-9]|AKID|LTAI|hf_|api_org_|r8_`,
String.raw`\bbot\d{6,}:|\b\d{6,}:[A-Za-z0-9_-]{20,}`,
// Obfuscated form/URL keys: percent escapes can rewrite any key letter, while plus or
// invisible splices break the literal key-name triggers above mid-word. After a splice the
// tail may mix further splices with key characters (e.g. an interior plus a trailing
// filler), but at least one key character must follow a splice so bare `+=` or line-leading
// `===` separators do not trip the fast path.
String.raw`%[0-9A-Fa-f]{2}[A-Za-z0-9_%.-]*=`,
String.raw`(?:\+|[${FORM_BODY_KEY_INVISIBLE_CHARS}])(?:[${FORM_BODY_KEY_INVISIBLE_CHARS}+]*[A-Za-z0-9_%.-])+[${FORM_BODY_KEY_INVISIBLE_CHARS}+]*=`,
];
const DEFAULT_REDACT_PREFILTER_RE = new RegExp(
`(?:${DEFAULT_REDACT_PREFILTER_SOURCES.join("|")})`,
"iu",
);
export type RedactOptions = {
mode?: RedactSensitiveMode;
@ -111,6 +286,7 @@ export type RedactOptions = {
export type ResolvedRedactOptions = {
mode: RedactSensitiveMode;
patterns: RegExp[];
redactFormBodies: boolean;
};
function normalizeMode(value?: string): RedactSensitiveMode {
@ -137,6 +313,9 @@ function parsePattern(raw: RedactPattern): RegExp | null {
if (pattern && typeof raw === "string" && SHELL_REFERENCE_PRESERVING_PATTERN_SOURCES.has(raw)) {
shellReferencePreservingPatterns.add(pattern);
}
if (pattern && typeof raw === "string" && raw.startsWith(BASE64_SAFE_TOKEN_BOUNDARY)) {
chunkUnsafePatterns.add(pattern);
}
return pattern;
}
@ -150,7 +329,18 @@ function resolvePatterns(value?: RedactPattern[]): RegExp[] {
return value.map(parsePattern).filter((re): re is RegExp => Boolean(re));
}
function includesDefaultRedactPatterns(value?: RedactPattern[]): boolean {
if (!value?.length) {
return true;
}
const source = new Set(value.filter((pattern): pattern is string => typeof pattern === "string"));
return DEFAULT_REDACT_PATTERNS.every((pattern) => source.has(pattern));
}
function maskToken(token: string): string {
if (token === "***") {
return token;
}
if (token.length < DEFAULT_REDACT_MIN_LENGTH) {
return "***";
}
@ -159,6 +349,337 @@ function maskToken(token: string): string {
return `${start}${end}`;
}
function splitSecretValueForMask(token: string): {
maskable: string;
suffix: string;
maskStart: number;
maskEnd: number;
} {
const openingQuote = token[0] ?? "";
if (SECRET_VALUE_QUOTE_CHARS.has(openingQuote)) {
const closingQuoteIndex = token.lastIndexOf(openingQuote);
if (closingQuoteIndex > 0) {
const suffix = token.slice(closingQuoteIndex + 1);
if (SECRET_VALUE_SUFFIX_RE.test(suffix)) {
return {
maskable: token.slice(1, closingQuoteIndex),
suffix,
maskStart: 0,
maskEnd: closingQuoteIndex + 1,
};
}
}
const tokenWithoutLeadingQuote = token.slice(1);
const trailingDelimiter =
tokenWithoutLeadingQuote.match(SECRET_VALUE_TRAILING_DELIMITER_RE)?.[1] ?? "";
const maskable =
trailingDelimiter && trailingDelimiter.length < tokenWithoutLeadingQuote.length
? tokenWithoutLeadingQuote.slice(0, -trailingDelimiter.length)
: tokenWithoutLeadingQuote;
return {
maskable,
suffix:
trailingDelimiter && trailingDelimiter.length < tokenWithoutLeadingQuote.length
? trailingDelimiter
: "",
maskStart: 0,
maskEnd: 1 + maskable.length,
};
}
const trailingDelimiter = token.match(SECRET_VALUE_TRAILING_DELIMITER_RE)?.[1] ?? "";
const maskable =
trailingDelimiter && trailingDelimiter.length < token.length
? token.slice(0, -trailingDelimiter.length)
: token;
return {
maskable,
suffix: maskable === token ? "" : trailingDelimiter,
maskStart: 0,
maskEnd: maskable.length,
};
}
function maskSecretValue(token: string, options?: { hinted?: boolean }): string {
const { maskable, suffix } = splitSecretValueForMask(token);
return `${options?.hinted ? maskToken(maskable) : "***"}${suffix}`;
}
function normalizeSensitiveKeyName(value: string): string {
const stripped = value.replace(FORM_BODY_KEY_SEPARATOR_RE, "");
try {
return decodeURIComponent(stripped)
.replace(FORM_BODY_KEY_SEPARATOR_RE, "")
.toLowerCase()
.replaceAll("-", "_");
} catch {
return stripped.toLowerCase().replaceAll("-", "_");
}
}
function isSensitiveBodyKey(key: string): boolean {
return BODY_SECRET_KEYS.has(normalizeSensitiveKeyName(key));
}
function hasEncodedOrInvisibleFormKey(key: string): boolean {
return (
FORM_BODY_PERCENT_ESCAPE_RE.test(key) || key.replace(FORM_BODY_KEY_OBFUSCATION_RE, "") !== key
);
}
function redactFormEncodedPairs(
value: string,
options?: { maskValues?: "fixed" | "hinted"; onlyEncodedOrInvisibleKeys?: boolean },
): string {
return value
.split("&")
.map((pair) => {
const equalsIndex = pair.indexOf("=");
if (equalsIndex < 0) {
return pair;
}
const key = pair.slice(0, equalsIndex);
if (options?.onlyEncodedOrInvisibleKeys && !hasEncodedOrInvisibleFormKey(key)) {
return pair;
}
if (!isSensitiveBodyKey(key)) {
return pair;
}
const token = pair.slice(equalsIndex + 1);
const masked = maskSecretValue(token, { hinted: options?.maskValues === "hinted" });
return `${key}=${masked}`;
})
.join("&");
}
function markBitmapRange(bitmap: boolean[], start: number, end: number): void {
const boundedStart = Math.max(0, start);
const boundedEnd = Math.min(bitmap.length, end);
for (let i = boundedStart; i < boundedEnd; i++) {
bitmap[i] = true;
}
}
function markSensitiveFormEncodedPairValues(
bitmap: boolean[],
value: string,
offset: number,
options?: { onlyEncodedOrInvisibleKeys?: boolean },
): void {
let cursor = 0;
for (const pair of value.split("&")) {
const pairStart = cursor;
const pairEnd = pairStart + pair.length;
cursor = pairEnd + 1;
const equalsIndex = pair.indexOf("=");
if (equalsIndex < 0) {
continue;
}
const key = pair.slice(0, equalsIndex);
if (options?.onlyEncodedOrInvisibleKeys && !hasEncodedOrInvisibleFormKey(key)) {
continue;
}
if (!isSensitiveBodyKey(key)) {
continue;
}
const token = pair.slice(equalsIndex + 1);
const secretValue = splitSecretValueForMask(token);
const valueStart = pairStart + equalsIndex + 1 + secretValue.maskStart;
const valueEnd = pairStart + equalsIndex + 1 + secretValue.maskEnd;
markBitmapRange(bitmap, offset + valueStart, offset + valueEnd);
}
}
function redactUrlQueryPairs(text: string): string {
if (!text || !text.includes("?")) {
return text;
}
return text.replace(URL_QUERY_PAIR_RE, (match, prefix: string, key: string, token: string) => {
if (!hasEncodedOrInvisibleFormKey(key) || !isSensitiveBodyKey(key)) {
return match;
}
return `${prefix}${key}=${maskSecretValue(token, { hinted: true })}`;
});
}
function markUrlQueryPairRedactions(text: string, bitmap: boolean[]): void {
if (!text || !text.includes("?")) {
return;
}
for (const match of text.matchAll(URL_QUERY_PAIR_RE)) {
if (match.index === undefined) {
continue;
}
const prefix = match[1] ?? "";
const key = match[2] ?? "";
const token = match[3] ?? "";
if (!hasEncodedOrInvisibleFormKey(key) || !isSensitiveBodyKey(key)) {
continue;
}
const secretValue = splitSecretValueForMask(token);
const valueOffset = match.index + prefix.length + key.length + 1;
markBitmapRange(bitmap, valueOffset + secretValue.maskStart, valueOffset + secretValue.maskEnd);
}
}
function redactEncodedFormPairs(text: string): string {
if (!text || (!text.includes("%") && text.replace(FORM_BODY_KEY_OBFUSCATION_RE, "") === text)) {
return text;
}
return text.replace(ENCODED_FORM_PAIR_RE, (match, prefix: string, key: string, token: string) => {
if (!hasEncodedOrInvisibleFormKey(key) || !isSensitiveBodyKey(key)) {
return match;
}
return `${prefix}${key}=${maskSecretValue(token)}`;
});
}
function markEncodedFormPairRedactions(text: string, bitmap: boolean[], offset = 0): void {
if (!text || (!text.includes("%") && text.replace(FORM_BODY_KEY_OBFUSCATION_RE, "") === text)) {
return;
}
for (const match of text.matchAll(ENCODED_FORM_PAIR_RE)) {
if (match.index === undefined) {
continue;
}
const prefix = match[1] ?? "";
const key = match[2] ?? "";
const token = match[3] ?? "";
if (!hasEncodedOrInvisibleFormKey(key) || !isSensitiveBodyKey(key)) {
continue;
}
const secretValue = splitSecretValueForMask(token);
const valueOffset = match.index + prefix.length + key.length + 1;
markBitmapRange(
bitmap,
offset + valueOffset + secretValue.maskStart,
offset + valueOffset + secretValue.maskEnd,
);
}
}
function redactFormBodyContextSinglePairs(text: string): string {
if (!text || !/[=:]/u.test(text)) {
return text;
}
return text.replace(
FORM_BODY_CONTEXT_SINGLE_PAIR_RE,
(match, prefix: string, _quote: string, key: string, token: string, suffix: string) => {
if (!isSensitiveBodyKey(key)) {
return match;
}
return `${prefix}${key}=${maskSecretValue(token)}${suffix}`;
},
);
}
function markFormBodyContextSinglePairRedactions(
text: string,
bitmap: boolean[],
offset = 0,
): void {
if (!text || !/[=:]/u.test(text)) {
return;
}
for (const match of text.matchAll(FORM_BODY_CONTEXT_SINGLE_PAIR_RE)) {
if (match.index === undefined) {
continue;
}
const prefix = match[1] ?? "";
const key = match[3] ?? "";
const token = match[4] ?? "";
if (!isSensitiveBodyKey(key)) {
continue;
}
const secretValue = splitSecretValueForMask(token);
const valueOffset = match.index + prefix.length + key.length + 1;
markBitmapRange(
bitmap,
offset + valueOffset + secretValue.maskStart,
offset + valueOffset + secretValue.maskEnd,
);
}
}
function redactFormBodyLine(text: string): string {
if (!text) {
return text;
}
const contextRedacted = redactFormBodyContextSinglePairs(redactEncodedFormPairs(text));
if (!contextRedacted.includes("&")) {
return contextRedacted;
}
if (FORM_BODY_RE.test(contextRedacted)) {
return redactFormEncodedPairs(contextRedacted);
}
const redacted = contextRedacted.replace(
FORM_BODY_SUBSTRING_RE,
(match, prefix: string, body: string) => {
const redactedBody = redactFormEncodedPairs(body);
return redactedBody === body ? match : `${prefix}${redactedBody}`;
},
);
return redactFormBodyContextSinglePairs(redactEncodedFormPairs(redacted));
}
function redactFormBody(text: string): string {
if (!text) {
return text;
}
if (FORM_BODY_LINE_BREAK_SPLIT_RE.test(text)) {
return text
.split(FORM_BODY_LINE_BREAK_SPLIT_RE)
.map((segment) =>
FORM_BODY_LINE_BREAK_SEGMENT_RE.test(segment) ? segment : redactFormBodyLine(segment),
)
.join("");
}
return redactFormBodyLine(text);
}
function markFormBodyLineRedactions(text: string, bitmap: boolean[], offset: number): void {
if (!text) {
return;
}
markEncodedFormPairRedactions(text, bitmap, offset);
markFormBodyContextSinglePairRedactions(text, bitmap, offset);
if (!text.includes("&")) {
return;
}
if (FORM_BODY_RE.test(text)) {
markSensitiveFormEncodedPairValues(bitmap, text, offset);
return;
}
for (const match of text.matchAll(FORM_BODY_SUBSTRING_RE)) {
if (match.index === undefined) {
continue;
}
const prefix = match[1] ?? "";
const body = match[2] ?? "";
markSensitiveFormEncodedPairValues(bitmap, body, offset + match.index + prefix.length);
}
}
function markFormBodyRedactions(text: string, bitmap: boolean[]): void {
if (!text) {
return;
}
if (!FORM_BODY_LINE_BREAK_SPLIT_RE.test(text)) {
markFormBodyLineRedactions(text, bitmap, 0);
return;
}
let offset = 0;
for (const segment of text.split(FORM_BODY_LINE_BREAK_SPLIT_RE)) {
if (!FORM_BODY_LINE_BREAK_SEGMENT_RE.test(segment)) {
markFormBodyLineRedactions(segment, bitmap, offset);
}
offset += segment.length;
}
}
function redactPemBlock(block: string): string {
const lines = block.split(/\r?\n/).filter(Boolean);
if (lines.length < 2) {
@ -192,36 +713,161 @@ function isEmptyShellParameterExpansionTail(token: string): boolean {
return /^[-=?+]\}$/.test(token);
}
function hasBackreferenceToGroup(pattern: RegExp, groupNumber: number): boolean {
return new RegExp(String.raw`\\${groupNumber}(?!\d)`).test(pattern.source);
}
type SecretCaptureSelection = {
captureCount: number;
index: number;
value: string;
};
function selectSecretCapture(match: string, groups: string[]): SecretCaptureSelection {
const tokens = groups
.map((value, index) => ({ index, value }))
.filter(({ value }) => typeof value === "string" && value.length > 0);
const selected = (tokens.length > 1 ? tokens[tokens.length - 1] : tokens[0]) ?? {
index: -1,
value: match,
};
return {
...selected,
captureCount: tokens.length,
};
}
function getIndexedCaptureStart(
pattern: RegExp,
input: string,
match: string,
matchOffset: number,
captureIndex: number,
): number | null {
if (matchOffset < 0 || !input) {
return null;
}
try {
const flags = pattern.flags.includes("d") ? pattern.flags : `${pattern.flags}d`;
const indexedPattern = new RegExp(pattern.source, flags);
indexedPattern.lastIndex = matchOffset;
const indexedMatch = indexedPattern.exec(input) as
| (RegExpExecArray & { indices?: Array<[number, number] | undefined> })
| null;
const captureIndices = indexedMatch?.indices?.[captureIndex + 1];
if (!indexedMatch || indexedMatch.index !== matchOffset || indexedMatch[0] !== match) {
return null;
}
if (!captureIndices) {
return null;
}
return captureIndices[0] - matchOffset;
} catch {
return null;
}
}
function getSecretCaptureStart(
pattern: RegExp,
input: string,
match: string,
matchOffset: number,
selected: SecretCaptureSelection,
): number {
const indexedTokenStart = getIndexedCaptureStart(
pattern,
input,
match,
matchOffset,
selected.index,
);
const preferFirstCapture =
selected.captureCount === 1 &&
selected.index >= 0 &&
hasBackreferenceToGroup(pattern, selected.index + 1);
return (
indexedTokenStart ??
(preferFirstCapture ? match.indexOf(selected.value) : match.lastIndexOf(selected.value))
);
}
function redactMatch(
match: string,
groups: string[],
options: { preserveShellReferences?: boolean } = {},
pattern: RegExp,
context?: { input?: string; offset?: number },
): string {
if (match.includes("PRIVATE KEY-----")) {
return redactPemBlock(match);
}
const token = groups.findLast((value) => typeof value === "string" && value.length > 0) ?? match;
const preserveShellReferences =
options.preserveShellReferences &&
(shouldPreserveShellReferenceMatch(match, token) || isEmptyShellParameterExpansionTail(token));
if (preserveShellReferences) {
const selected = selectSecretCapture(match, groups);
const token = selected.value;
// An earlier pass (form-body or quoted-assignment masking) may already have replaced this
// value with ***; re-masking would strip its quote wrapper around the placeholder.
if (splitSecretValueForMask(token).maskable === "***") {
return match;
}
const masked = maskToken(token);
const isShellReferencePattern = shellReferencePreservingPatterns.has(pattern);
// Preserve shell variable references (e.g. `MY_TOKEN=$MY_TOKEN`) for assignment patterns
// registered as shell-reference-preserving, so non-secret expansions that merely echo the
// assignment key are not masked.
if (
isShellReferencePattern &&
(shouldPreserveShellReferenceMatch(match, token) || isEmptyShellParameterExpansionTail(token))
) {
return match;
}
// Assignment values can legitimately include trailing shell/structural characters
// (e.g. `${VAR:-default}`); mask the captured token whole so those characters count toward the
// retained hint instead of being exposed by delimiter-aware masking.
const masked = isShellReferencePattern
? maskToken(token)
: maskSecretValue(token, { hinted: true });
if (token === match) {
return masked;
}
return match.replace(token, masked);
const tokenIndex = getSecretCaptureStart(
pattern,
context?.input ?? "",
match,
context?.offset ?? -1,
selected,
);
if (tokenIndex < 0) {
return match;
}
return `${match.slice(0, tokenIndex)}${masked}${match.slice(tokenIndex + token.length)}`;
}
function redactText(text: string, patterns: RegExp[]): string {
function redactText(
text: string,
patterns: RegExp[],
options?: { redactFormBodies?: boolean },
): string {
let next = text;
if (options?.redactFormBodies) {
next = redactUrlQueryPairs(next);
next = redactFormBody(next);
}
for (const pattern of patterns) {
next = replacePatternBounded(next, pattern, (...args: string[]) =>
redactMatch(args[0], args.slice(1, -2), {
preserveShellReferences: shellReferencePreservingPatterns.has(pattern),
}),
);
const replacer = (...args: unknown[]) => {
const hasNamedGroups =
args.length > 0 &&
typeof args[args.length - 1] === "object" &&
args[args.length - 1] !== null;
const inputIndex = hasNamedGroups ? args.length - 2 : args.length - 1;
const offsetIndex = inputIndex - 1;
const match = typeof args[0] === "string" ? args[0] : "";
const groups = args
.slice(1, offsetIndex)
.map((value) => (typeof value === "string" ? value : ""));
const offset = typeof args[offsetIndex] === "number" ? args[offsetIndex] : -1;
const input = typeof args[inputIndex] === "string" ? args[inputIndex] : "";
return redactMatch(match, groups, pattern, { input, offset });
};
next = chunkUnsafePatterns.has(pattern)
? next.replace(pattern, replacer)
: replacePatternBounded(next, pattern, replacer);
}
return next;
}
@ -230,13 +876,74 @@ function couldMatchDefaultRedactPatterns(text: string): boolean {
return DEFAULT_REDACT_PREFILTER_RE.test(text);
}
function cloneGlobalPattern(pattern: RegExp): RegExp {
return pattern.flags.includes("g")
? new RegExp(pattern.source, pattern.flags)
: new RegExp(pattern.source, `${pattern.flags}g`);
}
function markPatternMatchRedaction(
bitmap: boolean[],
input: string,
pattern: RegExp,
match: RegExpMatchArray,
): void {
if (match.index === undefined) {
return;
}
const fullMatch = match[0] ?? "";
if (fullMatch.includes("PRIVATE KEY-----")) {
markBitmapRange(bitmap, match.index, match.index + fullMatch.length);
return;
}
const selected = selectSecretCapture(
fullMatch,
match.slice(1).map((value) => (typeof value === "string" ? value : "")),
);
const tokenStart =
selected.value === fullMatch
? 0
: getSecretCaptureStart(pattern, input, fullMatch, match.index, selected);
if (tokenStart < 0) {
return;
}
const secretValue = splitSecretValueForMask(selected.value);
markBitmapRange(
bitmap,
match.index + tokenStart + secretValue.maskStart,
match.index + tokenStart + secretValue.maskEnd,
);
}
export function computeSensitiveRedactionBitmap(
text: string,
resolved: ResolvedRedactOptions,
): boolean[] {
const bitmap: boolean[] = Array.from({ length: text.length }, () => false);
if (resolved.mode === "off" || !resolved.patterns.length || !text) {
return bitmap;
}
if (resolved.redactFormBodies) {
markUrlQueryPairRedactions(text, bitmap);
markFormBodyRedactions(text, bitmap);
}
for (const pattern of resolved.patterns) {
for (const match of text.matchAll(cloneGlobalPattern(pattern))) {
markPatternMatchRedaction(bitmap, text, pattern, match);
}
}
return bitmap;
}
function looksLikeAppSpecificPassword(candidate: string): boolean {
return candidate.split("-").every((part) => !BENIGN_APP_PASSWORD_WORDS.has(part.toLowerCase()));
}
function redactAppSpecificPasswords(text: string): string {
return replacePatternBounded(text, APP_SPECIFIC_PASSWORD_RE, (match: string, token: string) =>
looksLikeAppSpecificPassword(token) ? redactMatch(match, [token]) : match,
looksLikeAppSpecificPassword(token)
? redactMatch(match, [token], APP_SPECIFIC_PASSWORD_RE)
: match,
);
}
@ -255,11 +962,14 @@ export function resolveRedactOptions(options?: RedactOptions): ResolvedRedactOpt
return {
mode,
patterns: [],
redactFormBodies: false,
};
}
const patterns = resolvePatterns(resolved.patterns);
return {
mode,
patterns: resolvePatterns(resolved.patterns),
patterns,
redactFormBodies: patterns.length > 0 && includesDefaultRedactPatterns(resolved.patterns),
};
}
@ -278,15 +988,11 @@ export function redactSensitiveText(text: string, options?: RedactOptions): stri
if (!resolved.patterns.length) {
return text;
}
return redactText(text, resolved.patterns);
return redactText(text, resolved.patterns, { redactFormBodies: resolved.redactFormBodies });
}
export function redactToolDetail(detail: string): string {
const resolved = resolveConfigRedaction();
if (normalizeMode(resolved.mode) !== "tools") {
return detail;
}
return redactSensitiveText(detail, resolved);
return redactToolPayloadText(detail);
}
function resolveToolPayloadRedaction(
@ -330,7 +1036,9 @@ function redactSensitiveFieldValueWithOptions(
if (resolved.mode === "off") {
return value;
}
const redacted = redactText(value, resolved.patterns);
const redacted = redactText(value, resolved.patterns, {
redactFormBodies: resolved.redactFormBodies,
});
const shouldRedactAppPassword = redacted !== value || STRUCTURED_APP_PASSWORD_FIELD_RE.test(key);
if (shouldRedactAppPassword) {
const appRedacted = redactAppSpecificPasswords(redacted);
@ -443,5 +1151,8 @@ export function redactSensitiveLines(lines: string[], resolved: ResolvedRedactOp
if (resolved.mode === "off" || !resolved.patterns.length || lines.length === 0) {
return lines;
}
return redactText(lines.join("\n"), resolved.patterns).split("\n");
const redactedLines = resolved.redactFormBodies
? lines.map((line) => redactFormBody(redactUrlQueryPairs(line)))
: lines;
return redactText(redactedLines.join("\n"), resolved.patterns).split("\n");
}