From b7235adb16cf7b82675103b6bbaaf880a539eb88 Mon Sep 17 00:00:00 2001 From: ozymandiashh <234437643+ozymandiashh@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:38:43 +0300 Subject: [PATCH] fix(parser): fold SQLite -wal siblings into source fingerprints Hermes, Cursor, OpenCode and copilot OTel sources all live in SQLite databases that their agents keep open in WAL mode for the life of the process. Committed writes park in -wal until a checkpoint, so the main file's stat can sit hours or days behind the newest committed data. fingerprintFile only statted the main file, which broke two ways: - The date-range mtime pre-filter in parseProviderSources read the stale mtime as "nothing in range" and skipped the source entirely. Every Hermes session committed after the last checkpoint vanished from reports: the today-parse skipped the db (mtime < local midnight) while the backfill only keeps days through yesterday. Exactly the "17 sessions in the DB, 14 reported, the 3 from today missing" report in issue #913. - reconcileFile saw an unchanged fingerprint between checkpoints and kept serving stale cached turns for sessions that had since grown. Fold the -wal sibling into the fingerprint: newest mtime wins and sizes add, so both WAL growth and a checkpoint (db grows, wal truncates) move the fingerprint. -shm is deliberately ignored (it mutates on reads). Bare SQLite paths get the fold only when the extension says database, so JSONL transcript fingerprints (offset-based append detection) are untouched. Refs #913 --- src/session-cache.ts | 59 ++++++++++++++++++++------- tests/providers/hermes.test.ts | 43 +++++++++++++++++++- tests/session-cache.test.ts | 74 +++++++++++++++++++++++++++++++++- 3 files changed, 160 insertions(+), 16 deletions(-) diff --git a/src/session-cache.ts b/src/session-cache.ts index 9405ce4..4d0e31b 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -607,34 +607,65 @@ async function retryCacheFileMutation(operation: () => Promise): Promise-wal`; the main +/// file's stat only moves on checkpoint, and a long-lived writer connection +/// (hermes, cursor and opencode keep their state DBs open for the life of +/// the agent process) can defer checkpoints for hours or days. A fingerprint +/// built from the main file alone then (a) carries an mtime older than the +/// newest committed data, so the date-range mtime pre-filter in +/// parseProviderSources skips the source and sessions committed after the +/// last checkpoint never parse (issue #913: today's Hermes sessions missing +/// from every report), and (b) does not change between checkpoints, so +/// reconcileFile keeps serving stale cached turns for sessions that grew. +/// Folding the WAL sibling in fixes both: the newest mtime wins, and the +/// sizes add so both WAL growth and a checkpoint (db grows, wal truncates) +/// move the fingerprint. `-shm` is deliberately ignored — it mutates on +/// reads too and would churn the fingerprint without any data change. +async function fingerprintSqliteFile(dbPath: string): Promise { + try { + const s = await stat(dbPath) + const wal = await stat(dbPath + '-wal').catch(() => null) + return { + dev: s.dev, + ino: s.ino, + mtimeMs: wal ? Math.max(s.mtimeMs, wal.mtimeMs) : s.mtimeMs, + sizeBytes: s.size + (wal?.size ?? 0), + } + } catch { + return null + } +} + export async function fingerprintFile(filePath: string): Promise { try { const s = await stat(filePath) + // A source path that IS a SQLite database (copilot OTel's agent-traces.db) + // needs the same WAL fold as the virtual-suffix forms below. + if (SQLITE_DB_PATH.test(filePath)) return fingerprintSqliteFile(filePath) return { dev: s.dev, ino: s.ino, mtimeMs: s.mtimeMs, sizeBytes: s.size } } catch { // Providers encode extra context into source paths using virtual suffixes: // - Cursor: `#cursor-ws=` (workspace-aware routing) // - OpenCode: `:` (session scoping) + // - Hermes: `#hermes-session=` (session scoping) // These compound paths don't exist on disk; strip the suffix to stat the - // underlying file. Try `#` first (rare in real paths), then `:` (must use - // lastIndexOf to tolerate Windows drive letters like C:\...). + // underlying database. Try `#` first (rare in real paths), then `:` (must + // use lastIndexOf to tolerate Windows drive letters like C:\...). const hashIdx = filePath.indexOf('#') if (hashIdx > 0) { - try { - const s = await stat(filePath.slice(0, hashIdx)) - return { dev: s.dev, ino: s.ino, mtimeMs: s.mtimeMs, sizeBytes: s.size } - } catch { - // fall through to colon check - } + const fp = await fingerprintSqliteFile(filePath.slice(0, hashIdx)) + if (fp) return fp + // fall through to colon check } const colonIdx = filePath.lastIndexOf(':') if (colonIdx > 0) { - try { - const s = await stat(filePath.slice(0, colonIdx)) - return { dev: s.dev, ino: s.ino, mtimeMs: s.mtimeMs, sizeBytes: s.size } - } catch { - return null - } + return fingerprintSqliteFile(filePath.slice(0, colonIdx)) } return null } diff --git a/tests/providers/hermes.test.ts b/tests/providers/hermes.test.ts index e475bf0..e17d1cd 100644 --- a/tests/providers/hermes.test.ts +++ b/tests/providers/hermes.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, rm } from 'fs/promises' +import { mkdir, mkdtemp, rm, utimes, writeFile } from 'fs/promises' import { basename, dirname, join } from 'path' import { tmpdir } from 'os' import { createRequire } from 'node:module' @@ -432,6 +432,47 @@ skipUnlessSqlite('hermes provider', () => { expect(modelTokens.reduce((sum, tokens) => sum + tokens.reasoningTokens, 0)).toBe(22) }) + // Regression for issue #913: Hermes writes state.db in WAL mode and keeps + // the writer connection open for the life of the agent, so recent sessions + // live in state.db-wal while the main file's mtime stays at the last + // checkpoint. The date-range mtime pre-filter in parseProviderSources + // then reads the source as "older than the range" and skips it, and every + // session committed since the last checkpoint disappears from reports. + // The fingerprint must fold the -wal sibling in so a stale main-file stat + // cannot hide fresh sessions. + it('still parses sessions committed to the WAL when the main db stat is checkpoint-stale', async () => { + const dbPath = createHermesDb(tmpDir) + withTestDb(dbPath, (db) => { + insertSession(db, { + id: 'wal-session', + inputTokens: 100, + outputTokens: 20, + cacheReadTokens: 0, + cacheWriteTokens: 0, + reasoningTokens: 0, + startedAt: 1779549200, + title: 'WAL Session', + }) + db.prepare('INSERT INTO messages (session_id, role, content, timestamp) VALUES (?, ?, ?, ?)') + .run('wal-session', 'user', 'Session committed after the last checkpoint', 1779549201) + }) + + // Simulate the WAL-mode stat shape: the main file's mtime predates the + // requested range (last checkpoint days ago), while a fresh -wal sibling + // holds the recent commits. The db itself was written in rollback-journal + // mode, so SQLite ignores the stray -wal on open; only its stat matters. + const beforeRange = new Date('2026-05-20T00:00:00.000Z') + await utimes(dbPath, beforeRange, beforeRange) + await writeFile(`${dbPath}-wal`, 'wal-frames') + + const { clearSessionCache, parseAllSessions } = await loadParserWithHermesHome(tmpDir, cacheDir) + clearSessionCache() + const projects = await parseAllSessions(dayRange(), 'hermes') + const sessions = projects.flatMap(project => project.sessions) + expect(sessions).toHaveLength(1) + expect(sessions[0]!.totalInputTokens).toBe(100) + }) + it('treats sibling profile-like directories as default sessions', async () => { const profileLikeDir = join(dirname(tmpDir), `${basename(tmpDir)}-profiles_backup`, 'coder') await mkdir(profileLikeDir, { recursive: true }) diff --git a/tests/session-cache.test.ts b/tests/session-cache.test.ts index 2c736b2..3591e43 100644 --- a/tests/session-cache.test.ts +++ b/tests/session-cache.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { readFile, rm, writeFile, mkdir } from 'fs/promises' +import { readFile, rm, utimes, writeFile, mkdir } from 'fs/promises' import { existsSync } from 'fs' import { tmpdir } from 'os' import { basename, join } from 'path' @@ -337,6 +337,78 @@ describe('fingerprintFile', () => { expect(fp).not.toBeNull() expect(fp!.sizeBytes).toBe(9) }) + + // SQLite WAL mode parks committed writes in `-wal`; the main file's + // stat only moves on checkpoint, which a long-lived writer defers for + // hours. A fingerprint from the main file alone reports data older than + // what is really committed (issue #913). The WAL sibling must be folded in. + it('folds -wal sibling into a # compound fingerprint (Hermes session)', async () => { + await mkdir(TMP_DIR, { recursive: true }) + const dbPath = join(TMP_DIR, 'state.db') + await writeFile(dbPath, 'main-db') + const past = new Date(Date.now() - 48 * 3600 * 1000) + await utimes(dbPath, past, past) + await writeFile(`${dbPath}-wal`, 'wal-frames') + + const fp = await fingerprintFile(`${dbPath}#hermes-session=abc`) + expect(fp).not.toBeNull() + // mtime: the fresh WAL wins over the checkpoint-stale main file. + expect(fp!.mtimeMs).toBeGreaterThan(past.getTime() + 3600 * 1000) + // size: main + wal, so WAL growth alone changes the fingerprint. + expect(fp!.sizeBytes).toBe('main-db'.length + 'wal-frames'.length) + }) + + it('folds -wal sibling into a : compound fingerprint (OpenCode session)', async () => { + await mkdir(TMP_DIR, { recursive: true }) + const dbPath = join(TMP_DIR, 'opencode.db') + await writeFile(dbPath, 'oc-db') + const past = new Date(Date.now() - 48 * 3600 * 1000) + await utimes(dbPath, past, past) + await writeFile(`${dbPath}-wal`, 'oc-wal') + + const fp = await fingerprintFile(`${dbPath}:ses_abc123`) + expect(fp).not.toBeNull() + expect(fp!.mtimeMs).toBeGreaterThan(past.getTime() + 3600 * 1000) + expect(fp!.sizeBytes).toBe('oc-db'.length + 'oc-wal'.length) + }) + + it('folds -wal sibling into a bare SQLite path (copilot agent-traces.db)', async () => { + await mkdir(TMP_DIR, { recursive: true }) + const dbPath = join(TMP_DIR, 'agent-traces.db') + await writeFile(dbPath, 'traces') + const past = new Date(Date.now() - 48 * 3600 * 1000) + await utimes(dbPath, past, past) + await writeFile(`${dbPath}-wal`, 'traces-wal') + + const fp = await fingerprintFile(dbPath) + expect(fp).not.toBeNull() + expect(fp!.mtimeMs).toBeGreaterThan(past.getTime() + 3600 * 1000) + expect(fp!.sizeBytes).toBe('traces'.length + 'traces-wal'.length) + }) + + it('keeps compound fingerprints working when no -wal sibling exists', async () => { + await mkdir(TMP_DIR, { recursive: true }) + const dbPath = join(TMP_DIR, 'state.db') + await writeFile(dbPath, 'main-only') + + const fp = await fingerprintFile(`${dbPath}#hermes-session=abc`) + expect(fp).not.toBeNull() + expect(fp!.sizeBytes).toBe('main-only'.length) + }) + + it('does not fold sibling files into non-SQLite fingerprints', async () => { + await mkdir(TMP_DIR, { recursive: true }) + const filePath = join(TMP_DIR, 'session.jsonl') + await writeFile(filePath, 'jsonl-data') + // A stray neighbor that happens to match the -wal naming must not leak + // into a transcript fingerprint (offset-based append detection relies on + // sizeBytes being the transcript's real byte length). + await writeFile(`${filePath}-wal`, 'stray') + + const fp = await fingerprintFile(filePath) + expect(fp).not.toBeNull() + expect(fp!.sizeBytes).toBe('jsonl-data'.length) + }) }) // ── reconcileFile ──────────────────────────────────────────────────────