vrr/codeburn
2
0
Fork 0
mirror of https://github.com/AgentSeal/codeburn.git synced 2026-08-25 16:33:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

fix(compare): refine self-correction patterns, skip compact files, deduplicate

Browse source 
Remove high-false-positive patterns (I'm sorry, I should have, sorry for).
Add precise patterns (you're right I, that was incorrect, let me correct).
Skip compact JSONL files that replay compressed context.
Deduplicate by model+timestamp to prevent double-counting.
Fix test timestamps to work with deduplication.
This commit is contained in:
iamtoruk 2026-04-19 07:14:02 -07:00 • committed by AgentSeal
parent d04159a056
commit fb24eea186
2 changed files with 30 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

33
src/compare-stats.ts
View file

@ -144,19 +144,20 @@ export function computeComparison(a: ModelStats, b: ModelStats): ComparisonRow[]
}
const SELF_CORRECTION_PATTERNS = [
/\bI('m| am) sorry\b/i,
/\bmy mistake\b/i,
/\bmy apolog/i,
/\bI made (a |an )?(error|mistake)\b/i,
/\bI was wrong\b/i,
/\bmy bad\b/i,
/\bmy apolog/i,
/\bI apologize\b/i,
/\bsorry about that\b/i,
/\bsorry for (the|that|this)\b/i,
/\bI should have\b/i,
/\bI shouldn't have\b/i,
/\bI was wrong\b/i,
/\bI was incorrect\b/i,
/\bI made (a |an )?(error|mistake)\b/i,
/\bI incorrectly\b/i,
/\bI mistakenly\b/i,
/\bthat was (incorrect|wrong|an error)\b/i,
/\blet me correct that\b/i,
/\bI need to correct\b/i,
/\byou're right[.,]? I/i,
/\bsorry about that\b/i,
]
function extractText(content: unknown): string {
@ -168,16 +169,20 @@ function extractText(content: unknown): string {
.join(' ')
}
function isCompactFile(name: string): boolean {
return name.includes('compact')
}
async function collectJsonlFiles(sessionDir: string): Promise<string[]> {
const entries = await readdir(sessionDir, { withFileTypes: true })
const files: string[] = []
for (const entry of entries) {
if (entry.isFile() && entry.name.endsWith('.jsonl')) {
if (entry.isFile() && entry.name.endsWith('.jsonl') && !isCompactFile(entry.name)) {
files.push(join(sessionDir, entry.name))
} else if (entry.isDirectory() && entry.name === 'subagents') {
const subEntries = await readdir(join(sessionDir, entry.name), { withFileTypes: true })
for (const sub of subEntries) {
if (sub.isFile() && sub.name.endsWith('.jsonl')) {
if (sub.isFile() && sub.name.endsWith('.jsonl') && !isCompactFile(sub.name)) {
files.push(join(sessionDir, entry.name, sub.name))
}
}
@ -188,6 +193,7 @@ async function collectJsonlFiles(sessionDir: string): Promise<string[]> {
export async function scanSelfCorrections(projectDirs: string[]): Promise<Map<string, number>> {
const counts = new Map<string, number>()
const seen = new Set<string>()
for (const dir of projectDirs) {
let entries
@ -200,7 +206,7 @@ export async function scanSelfCorrections(projectDirs: string[]): Promise<Map<st
const allFiles: string[] = []
for (const entry of entries) {
if (entry.isFile() && entry.name.endsWith('.jsonl')) {
if (entry.isFile() && entry.name.endsWith('.jsonl') && !isCompactFile(entry.name)) {
allFiles.push(join(dir, entry.name))
} else if (entry.isDirectory()) {
try {
@ -234,6 +240,7 @@ export async function scanSelfCorrections(projectDirs: string[]): Promise<Map<st
const rec = parsed as Record<string, unknown>
if (!rec || typeof rec !== 'object' || rec['type'] !== 'assistant') continue
const ts = rec['timestamp']
const msg = rec['message']
if (msg === null || typeof msg !== 'object') continue
@ -241,6 +248,10 @@ export async function scanSelfCorrections(projectDirs: string[]): Promise<Map<st
const model = msgRec['model']
if (typeof model !== 'string' || model === '<synthetic>') continue
const dedupeKey = `${model}:${ts}`
if (seen.has(dedupeKey)) continue
seen.add(dedupeKey)
const text = extractText(msgRec['content'])
if (SELF_CORRECTION_PATTERNS.some(p => p.test(text))) {
counts.set(model, (counts.get(model) ?? 0) + 1)

16
tests/compare-stats.test.ts
View file

@ -213,14 +213,14 @@ describe('computeComparison', () => {
})
})
function jsonlLine(type: string, model: string, text: string): string {
function jsonlLine(type: string, model: string, text: string, timestamp = '2026-04-15T10:00:00Z'): string {
if (type === 'assistant') {
return JSON.stringify({
type: 'assistant', timestamp: '2026-04-15T10:00:00Z',
type: 'assistant', timestamp,
message: { model, content: [{ type: 'text', text }], id: `msg-${Math.random()}`, usage: { input_tokens: 0, output_tokens: 0 } },
})
}
return JSON.stringify({ type: 'user', timestamp: '2026-04-15T10:00:00Z', message: { role: 'user', content: text } })
return JSON.stringify({ type: 'user', timestamp, message: { role: 'user', content: text } })
}
describe('scanSelfCorrections', () => {
@ -307,12 +307,12 @@ describe('scanSelfCorrections', () => {
await mkdir(sessionB)
await writeFile(join(sessionA, 'a.jsonl'), [
jsonlLine('assistant', 'opus-4-6', 'I was wrong.'),
jsonlLine('assistant', 'opus-4-6', 'My bad!'),
jsonlLine('assistant', 'opus-4-6', 'I was wrong.', '2026-04-15T10:00:00Z'),
jsonlLine('assistant', 'opus-4-6', 'My bad!', '2026-04-15T10:01:00Z'),
].join('\n') + '\n')
await writeFile(join(sessionB, 'b.jsonl'), [
jsonlLine('assistant', 'opus-4-6', 'I apologize.'),
jsonlLine('assistant', 'opus-4-6', 'I apologize.', '2026-04-15T10:02:00Z'),
].join('\n') + '\n')
const result = await scanSelfCorrections([tmpDir])
@ -340,11 +340,11 @@ describe('scanSelfCorrections', () => {
await mkdir(sessionB)
await writeFile(join(sessionA, 'a.jsonl'), [
jsonlLine('assistant', 'sonnet-4-6', 'My mistake.'),
jsonlLine('assistant', 'sonnet-4-6', 'My mistake.', '2026-04-15T10:00:00Z'),
].join('\n') + '\n')
await writeFile(join(sessionB, 'b.jsonl'), [
jsonlLine('assistant', 'sonnet-4-6', 'I was wrong.'),
jsonlLine('assistant', 'sonnet-4-6', 'I was wrong.', '2026-04-15T10:01:00Z'),
].join('\n') + '\n')
const result = await scanSelfCorrections([tmpDir, dir2])