From fb24eea186d01f69af7b8b03fb5145606b303cdf Mon Sep 17 00:00:00 2001 From: iamtoruk Date: Sun, 19 Apr 2026 07:14:02 -0700 Subject: [PATCH] fix(compare): refine self-correction patterns, skip compact files, deduplicate Remove high-false-positive patterns (I'm sorry, I should have, sorry for). Add precise patterns (you're right I, that was incorrect, let me correct). Skip compact JSONL files that replay compressed context. Deduplicate by model+timestamp to prevent double-counting. Fix test timestamps to work with deduplication. --- src/compare-stats.ts | 33 ++++++++++++++++++++++----------- tests/compare-stats.test.ts | 16 ++++++++-------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/compare-stats.ts b/src/compare-stats.ts index 5c938fb..3406510 100644 --- a/src/compare-stats.ts +++ b/src/compare-stats.ts @@ -144,19 +144,20 @@ export function computeComparison(a: ModelStats, b: ModelStats): ComparisonRow[] } const SELF_CORRECTION_PATTERNS = [ - /\bI('m| am) sorry\b/i, /\bmy mistake\b/i, - /\bmy apolog/i, - /\bI made (a |an )?(error|mistake)\b/i, - /\bI was wrong\b/i, /\bmy bad\b/i, + /\bmy apolog/i, /\bI apologize\b/i, - /\bsorry about that\b/i, - /\bsorry for (the|that|this)\b/i, - /\bI should have\b/i, - /\bI shouldn't have\b/i, + /\bI was wrong\b/i, + /\bI was incorrect\b/i, + /\bI made (a |an )?(error|mistake)\b/i, /\bI incorrectly\b/i, /\bI mistakenly\b/i, + /\bthat was (incorrect|wrong|an error)\b/i, + /\blet me correct that\b/i, + /\bI need to correct\b/i, + /\byou're right[.,]? I/i, + /\bsorry about that\b/i, ] function extractText(content: unknown): string { @@ -168,16 +169,20 @@ function extractText(content: unknown): string { .join(' ') } +function isCompactFile(name: string): boolean { + return name.includes('compact') +} + async function collectJsonlFiles(sessionDir: string): Promise { const entries = await readdir(sessionDir, { withFileTypes: true }) const files: string[] = [] for (const entry of entries) { - if (entry.isFile() && entry.name.endsWith('.jsonl')) { + if (entry.isFile() && entry.name.endsWith('.jsonl') && !isCompactFile(entry.name)) { files.push(join(sessionDir, entry.name)) } else if (entry.isDirectory() && entry.name === 'subagents') { const subEntries = await readdir(join(sessionDir, entry.name), { withFileTypes: true }) for (const sub of subEntries) { - if (sub.isFile() && sub.name.endsWith('.jsonl')) { + if (sub.isFile() && sub.name.endsWith('.jsonl') && !isCompactFile(sub.name)) { files.push(join(sessionDir, entry.name, sub.name)) } } @@ -188,6 +193,7 @@ async function collectJsonlFiles(sessionDir: string): Promise { export async function scanSelfCorrections(projectDirs: string[]): Promise> { const counts = new Map() + const seen = new Set() for (const dir of projectDirs) { let entries @@ -200,7 +206,7 @@ export async function scanSelfCorrections(projectDirs: string[]): Promise if (!rec || typeof rec !== 'object' || rec['type'] !== 'assistant') continue + const ts = rec['timestamp'] const msg = rec['message'] if (msg === null || typeof msg !== 'object') continue @@ -241,6 +248,10 @@ export async function scanSelfCorrections(projectDirs: string[]): Promise') continue + const dedupeKey = `${model}:${ts}` + if (seen.has(dedupeKey)) continue + seen.add(dedupeKey) + const text = extractText(msgRec['content']) if (SELF_CORRECTION_PATTERNS.some(p => p.test(text))) { counts.set(model, (counts.get(model) ?? 0) + 1) diff --git a/tests/compare-stats.test.ts b/tests/compare-stats.test.ts index ad1c461..d1441a4 100644 --- a/tests/compare-stats.test.ts +++ b/tests/compare-stats.test.ts @@ -213,14 +213,14 @@ describe('computeComparison', () => { }) }) -function jsonlLine(type: string, model: string, text: string): string { +function jsonlLine(type: string, model: string, text: string, timestamp = '2026-04-15T10:00:00Z'): string { if (type === 'assistant') { return JSON.stringify({ - type: 'assistant', timestamp: '2026-04-15T10:00:00Z', + type: 'assistant', timestamp, message: { model, content: [{ type: 'text', text }], id: `msg-${Math.random()}`, usage: { input_tokens: 0, output_tokens: 0 } }, }) } - return JSON.stringify({ type: 'user', timestamp: '2026-04-15T10:00:00Z', message: { role: 'user', content: text } }) + return JSON.stringify({ type: 'user', timestamp, message: { role: 'user', content: text } }) } describe('scanSelfCorrections', () => { @@ -307,12 +307,12 @@ describe('scanSelfCorrections', () => { await mkdir(sessionB) await writeFile(join(sessionA, 'a.jsonl'), [ - jsonlLine('assistant', 'opus-4-6', 'I was wrong.'), - jsonlLine('assistant', 'opus-4-6', 'My bad!'), + jsonlLine('assistant', 'opus-4-6', 'I was wrong.', '2026-04-15T10:00:00Z'), + jsonlLine('assistant', 'opus-4-6', 'My bad!', '2026-04-15T10:01:00Z'), ].join('\n') + '\n') await writeFile(join(sessionB, 'b.jsonl'), [ - jsonlLine('assistant', 'opus-4-6', 'I apologize.'), + jsonlLine('assistant', 'opus-4-6', 'I apologize.', '2026-04-15T10:02:00Z'), ].join('\n') + '\n') const result = await scanSelfCorrections([tmpDir]) @@ -340,11 +340,11 @@ describe('scanSelfCorrections', () => { await mkdir(sessionB) await writeFile(join(sessionA, 'a.jsonl'), [ - jsonlLine('assistant', 'sonnet-4-6', 'My mistake.'), + jsonlLine('assistant', 'sonnet-4-6', 'My mistake.', '2026-04-15T10:00:00Z'), ].join('\n') + '\n') await writeFile(join(sessionB, 'b.jsonl'), [ - jsonlLine('assistant', 'sonnet-4-6', 'I was wrong.'), + jsonlLine('assistant', 'sonnet-4-6', 'I was wrong.', '2026-04-15T10:01:00Z'), ].join('\n') + '\n') const result = await scanSelfCorrections([tmpDir, dir2])