perf(classifier): linear bash separator split

The separator regex retried its leading \s* from every offset, which is
quadratic on the long whitespace-heavy commands agents emit; that one regex
was ~30% of a warm status run on a multi-GB corpus. Match the separator
alone and widen over whitespace by hand. Output is unchanged (differential
check over 42k real commands).
This commit is contained in:
iamtoruk 2026-08-16 18:31:41 -07:00
parent 897020591a
commit 1851687081
3 changed files with 169 additions and 3 deletions

View file

@ -3,6 +3,7 @@
## Unreleased
### Fixed
- **Bash command splitting was quadratic on long whitespace-heavy commands.** The separator regex retried its leading `\s*` from every offset; matching the separator alone and widening over whitespace by hand makes cold parse ~24% and warm ~40% faster on large corpora, output unchanged.
- **Cold parse no longer retains full message bodies through cached previews.** `flatSlice` skipped its Buffer round-trip for strings already within the bound, but provider adapters pre-truncate user-message previews with `.slice(0, 500)` before the cache-site call — those pre-sliced views are still V8 SlicedStrings pinning their large parent, so the retention that OOM'd cold parses of large histories survived. The round-trip now always runs.
- **Kiro sessions carry the real `projectPath`** (CLI meta.cwd, v2 `workspacePaths[0]`, workspace sessions' `workspaceDirectory`), so git-repo attribution can resolve them; previously they were attribution-blind. Bumps the kiro parse version, so the first run after upgrade re-parses kiro history once, and kiro sessions in linked git worktrees now group under the main repo.

View file

@ -1,6 +1,8 @@
import { basename } from 'path'
import stripAnsi from 'strip-ansi'
const WHITESPACE = /\s/
function stripQuotedStrings(command: string): string {
return command.replace(/"[^"]*"|'[^']*'/g, match => ' '.repeat(match.length))
}
@ -19,12 +21,22 @@ export function extractBashCommands(rawCommand: string): string[] {
const command = stripAnsi(rawCommand)
const stripped = stripQuotedStrings(command)
const separatorRegex = /\s*(?:&&|;|\|)\s*/g
// Match the separator alone, then widen over surrounding whitespace by hand.
// /\s*(?:&&|;|\|)\s*/ retried its leading \s* from every offset, quadratic on
// long whitespace-heavy commands. Widening is required (not cosmetic): stripQuotedStrings
// blanks quoted text, and segments are sliced from the original string.
const separatorRegex = /(?:&&|;|\|)/g
const separators: Array<{ start: number; end: number }> = []
let match: RegExpExecArray | null
while ((match = separatorRegex.exec(stripped)) !== null) {
separators.push({ start: match.index, end: match.index + match[0].length })
let start = match.index
while (start > 0 && WHITESPACE.test(stripped[start - 1]!)) start--
let end = match.index + match[0].length
while (end < stripped.length && WHITESPACE.test(stripped[end]!)) end++
const prevEnd = separators[separators.length - 1]?.end ?? 0
separators.push({ start: Math.max(start, prevEnd), end })
separatorRegex.lastIndex = end
}
const ranges: Array<[number, number]> = []
@ -93,7 +105,7 @@ const GIT_READ_SUBCOMMANDS = new Set([
export function isReadShapedBashCommand(rawCommand: string): boolean {
if (!rawCommand || !rawCommand.trim()) return false
const stripped = stripQuotedStrings(stripAnsi(rawCommand))
const segments = stripped.split(/\s*(?:&&|;|\|)\s*/)
const segments = stripped.split(/(?:&&|;|\|)/)
let sawCommand = false
for (const segment of segments) {
const trimmed = segment.trim()

View file

@ -1,4 +1,6 @@
import { describe, it, expect } from 'vitest'
import { basename } from 'path'
import stripAnsi from 'strip-ansi'
import { extractBashCommands, isReadShapedBashCommand } from '../src/bash-utils.js'
import { BASH_TOOLS } from '../src/classifier.js'
@ -118,6 +120,157 @@ describe('BASH_TOOLS', () => {
it('rejects unknown tools', () => { expect(BASH_TOOLS.has('Read')).toBe(false) })
})
// Regression coverage for the quadratic -> linear separator-matching rewrite.
// The old regex (/\s*(?:&&|;|\|)\s*/g, and the equivalent split form) is kept
// here verbatim as a reference so new/old output can be diffed on tricky inputs.
describe('separator regex fix: parity with pre-fix implementation', () => {
function stripQuotedStringsRef(command: string): string {
return command.replace(/"[^"]*"|'[^']*'/g, match => ' '.repeat(match.length))
}
const COMMAND_PREFIXES_REF = new Set([
'sudo', 'doas',
'npx', 'bunx',
'time',
'nice', 'nohup', 'stdbuf',
'rtk',
])
const READ_ONLY_BASH_REF = new Set([
'rg', 'grep', 'egrep', 'fgrep', 'ag',
'cat', 'head', 'tail', 'less', 'more',
'ls', 'find', 'fd', 'tree',
'wc', 'stat', 'file', 'du', 'df',
'which', 'type', 'pwd', 'printenv', 'env',
'readlink', 'realpath', 'basename', 'dirname',
'jq', 'diff',
])
const GIT_READ_SUBCOMMANDS_REF = new Set([
'log', 'diff', 'status', 'show', 'blame', 'grep',
'shortlog', 'describe', 'rev-parse', 'ls-files',
])
function extractBashCommandsOld(rawCommand: string): string[] {
if (!rawCommand || !rawCommand.trim()) return []
const command = stripAnsi(rawCommand)
const stripped = stripQuotedStringsRef(command)
const separatorRegex = /\s*(?:&&|;|\|)\s*/g
const separators: Array<{ start: number; end: number }> = []
let match: RegExpExecArray | null
while ((match = separatorRegex.exec(stripped)) !== null) {
separators.push({ start: match.index, end: match.index + match[0].length })
}
const ranges: Array<[number, number]> = []
let cursor = 0
for (const sep of separators) {
ranges.push([cursor, sep.start])
cursor = sep.end
}
ranges.push([cursor, command.length])
const commands: string[] = []
for (const [start, end] of ranges) {
const segment = command.slice(start, end).trim()
if (!segment) continue
const tokens = segment.split(/\s+/)
let i = 0
while (i < tokens.length) {
if (/^\w+=/.test(tokens[i]!)) { i++; continue }
const next = tokens[i + 1]
if (
next !== undefined &&
COMMAND_PREFIXES_REF.has(basename(tokens[i]!)) &&
!next.startsWith('-') &&
!/["']/.test(next)
) { i++; continue }
break
}
const base = i < tokens.length ? basename(tokens[i]!) : ''
if (base && base !== 'cd' && base !== 'true' && base !== 'false') {
commands.push(base)
}
}
return commands
}
function isReadShapedBashCommandOld(rawCommand: string): boolean {
if (!rawCommand || !rawCommand.trim()) return false
const stripped = stripQuotedStringsRef(stripAnsi(rawCommand))
const segments = stripped.split(/\s*(?:&&|;|\|)\s*/)
let sawCommand = false
for (const segment of segments) {
const trimmed = segment.trim()
if (!trimmed) continue
const tokens = trimmed.split(/\s+/)
let i = 0
while (i < tokens.length && (/^\w+=/.test(tokens[i]!) || COMMAND_PREFIXES_REF.has(basename(tokens[i]!)))) i++
const base = i < tokens.length ? basename(tokens[i]!) : ''
if (!base) continue
sawCommand = true
if (base === 'git') {
const sub = tokens[i + 1]
if (!sub || !GIT_READ_SUBCOMMANDS_REF.has(sub)) return false
continue
}
if (!READ_ONLY_BASH_REF.has(base)) return false
}
return sawCommand
}
function buildWhitespaceHeavyCommand(): string {
const parts: string[] = []
for (let i = 0; i < 10; i++) parts.push('git' + ' '.repeat(2000) + 'status')
return parts.join(' && ')
}
const TRICKY_INPUTS: string[] = [
'echo "a && b" && ls',
"foo 'x;y';bar",
'cat <<EOF\n some text with lots of whitespace \n\n\nEOF\n && ls -la',
'echo a\r\n&&\tls',
'foo\t;\tbar',
'a ; ; b',
'a|b',
'&& ls',
'ls &&',
'; ls ;',
buildWhitespaceHeavyCommand(),
'echo a && ls',
'foo;bar',
'',
' ',
'ls && pwd',
'git status',
]
it('extractBashCommands matches the old implementation across tricky separator inputs', () => {
for (const input of TRICKY_INPUTS) {
expect(extractBashCommands(input)).toEqual(extractBashCommandsOld(input))
}
})
it('isReadShapedBashCommand matches the old implementation across tricky separator inputs', () => {
for (const input of TRICKY_INPUTS) {
expect(isReadShapedBashCommand(input)).toBe(isReadShapedBashCommandOld(input))
}
})
it('runs the whitespace-heavy command in well under 50ms (old form is quadratic)', () => {
const big = buildWhitespaceHeavyCommand()
const t0 = Date.now()
extractBashCommands(big)
expect(Date.now() - t0).toBeLessThan(50)
})
})
describe('isReadShapedBashCommand (#941)', () => {
it('accepts single read commands and read-only git subcommands', () => {
expect(isReadShapedBashCommand('rg -n "x" src/')).toBe(true)