diff --git a/src/session-cache.ts b/src/session-cache.ts index 9405ce4..4d0e31b 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -607,34 +607,65 @@ async function retryCacheFileMutation(operation: () => Promise): Promise-wal`; the main +/// file's stat only moves on checkpoint, and a long-lived writer connection +/// (hermes, cursor and opencode keep their state DBs open for the life of +/// the agent process) can defer checkpoints for hours or days. A fingerprint +/// built from the main file alone then (a) carries an mtime older than the +/// newest committed data, so the date-range mtime pre-filter in +/// parseProviderSources skips the source and sessions committed after the +/// last checkpoint never parse (issue #913: today's Hermes sessions missing +/// from every report), and (b) does not change between checkpoints, so +/// reconcileFile keeps serving stale cached turns for sessions that grew. +/// Folding the WAL sibling in fixes both: the newest mtime wins, and the +/// sizes add so both WAL growth and a checkpoint (db grows, wal truncates) +/// move the fingerprint. `-shm` is deliberately ignored — it mutates on +/// reads too and would churn the fingerprint without any data change. +async function fingerprintSqliteFile(dbPath: string): Promise { + try { + const s = await stat(dbPath) + const wal = await stat(dbPath + '-wal').catch(() => null) + return { + dev: s.dev, + ino: s.ino, + mtimeMs: wal ? Math.max(s.mtimeMs, wal.mtimeMs) : s.mtimeMs, + sizeBytes: s.size + (wal?.size ?? 0), + } + } catch { + return null + } +} + export async function fingerprintFile(filePath: string): Promise { try { const s = await stat(filePath) + // A source path that IS a SQLite database (copilot OTel's agent-traces.db) + // needs the same WAL fold as the virtual-suffix forms below. + if (SQLITE_DB_PATH.test(filePath)) return fingerprintSqliteFile(filePath) return { dev: s.dev, ino: s.ino, mtimeMs: s.mtimeMs, sizeBytes: s.size } } catch { // Providers encode extra context into source paths using virtual suffixes: // - Cursor: `#cursor-ws=` (workspace-aware routing) // - OpenCode: `:` (session scoping) + // - Hermes: `#hermes-session=` (session scoping) // These compound paths don't exist on disk; strip the suffix to stat the - // underlying file. Try `#` first (rare in real paths), then `:` (must use - // lastIndexOf to tolerate Windows drive letters like C:\...). + // underlying database. Try `#` first (rare in real paths), then `:` (must + // use lastIndexOf to tolerate Windows drive letters like C:\...). const hashIdx = filePath.indexOf('#') if (hashIdx > 0) { - try { - const s = await stat(filePath.slice(0, hashIdx)) - return { dev: s.dev, ino: s.ino, mtimeMs: s.mtimeMs, sizeBytes: s.size } - } catch { - // fall through to colon check - } + const fp = await fingerprintSqliteFile(filePath.slice(0, hashIdx)) + if (fp) return fp + // fall through to colon check } const colonIdx = filePath.lastIndexOf(':') if (colonIdx > 0) { - try { - const s = await stat(filePath.slice(0, colonIdx)) - return { dev: s.dev, ino: s.ino, mtimeMs: s.mtimeMs, sizeBytes: s.size } - } catch { - return null - } + return fingerprintSqliteFile(filePath.slice(0, colonIdx)) } return null } diff --git a/tests/providers/hermes.test.ts b/tests/providers/hermes.test.ts index e475bf0..e17d1cd 100644 --- a/tests/providers/hermes.test.ts +++ b/tests/providers/hermes.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, rm } from 'fs/promises' +import { mkdir, mkdtemp, rm, utimes, writeFile } from 'fs/promises' import { basename, dirname, join } from 'path' import { tmpdir } from 'os' import { createRequire } from 'node:module' @@ -432,6 +432,47 @@ skipUnlessSqlite('hermes provider', () => { expect(modelTokens.reduce((sum, tokens) => sum + tokens.reasoningTokens, 0)).toBe(22) }) + // Regression for issue #913: Hermes writes state.db in WAL mode and keeps + // the writer connection open for the life of the agent, so recent sessions + // live in state.db-wal while the main file's mtime stays at the last + // checkpoint. The date-range mtime pre-filter in parseProviderSources + // then reads the source as "older than the range" and skips it, and every + // session committed since the last checkpoint disappears from reports. + // The fingerprint must fold the -wal sibling in so a stale main-file stat + // cannot hide fresh sessions. + it('still parses sessions committed to the WAL when the main db stat is checkpoint-stale', async () => { + const dbPath = createHermesDb(tmpDir) + withTestDb(dbPath, (db) => { + insertSession(db, { + id: 'wal-session', + inputTokens: 100, + outputTokens: 20, + cacheReadTokens: 0, + cacheWriteTokens: 0, + reasoningTokens: 0, + startedAt: 1779549200, + title: 'WAL Session', + }) + db.prepare('INSERT INTO messages (session_id, role, content, timestamp) VALUES (?, ?, ?, ?)') + .run('wal-session', 'user', 'Session committed after the last checkpoint', 1779549201) + }) + + // Simulate the WAL-mode stat shape: the main file's mtime predates the + // requested range (last checkpoint days ago), while a fresh -wal sibling + // holds the recent commits. The db itself was written in rollback-journal + // mode, so SQLite ignores the stray -wal on open; only its stat matters. + const beforeRange = new Date('2026-05-20T00:00:00.000Z') + await utimes(dbPath, beforeRange, beforeRange) + await writeFile(`${dbPath}-wal`, 'wal-frames') + + const { clearSessionCache, parseAllSessions } = await loadParserWithHermesHome(tmpDir, cacheDir) + clearSessionCache() + const projects = await parseAllSessions(dayRange(), 'hermes') + const sessions = projects.flatMap(project => project.sessions) + expect(sessions).toHaveLength(1) + expect(sessions[0]!.totalInputTokens).toBe(100) + }) + it('treats sibling profile-like directories as default sessions', async () => { const profileLikeDir = join(dirname(tmpDir), `${basename(tmpDir)}-profiles_backup`, 'coder') await mkdir(profileLikeDir, { recursive: true }) diff --git a/tests/session-cache.test.ts b/tests/session-cache.test.ts index 2c736b2..3591e43 100644 --- a/tests/session-cache.test.ts +++ b/tests/session-cache.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { readFile, rm, writeFile, mkdir } from 'fs/promises' +import { readFile, rm, utimes, writeFile, mkdir } from 'fs/promises' import { existsSync } from 'fs' import { tmpdir } from 'os' import { basename, join } from 'path' @@ -337,6 +337,78 @@ describe('fingerprintFile', () => { expect(fp).not.toBeNull() expect(fp!.sizeBytes).toBe(9) }) + + // SQLite WAL mode parks committed writes in `-wal`; the main file's + // stat only moves on checkpoint, which a long-lived writer defers for + // hours. A fingerprint from the main file alone reports data older than + // what is really committed (issue #913). The WAL sibling must be folded in. + it('folds -wal sibling into a # compound fingerprint (Hermes session)', async () => { + await mkdir(TMP_DIR, { recursive: true }) + const dbPath = join(TMP_DIR, 'state.db') + await writeFile(dbPath, 'main-db') + const past = new Date(Date.now() - 48 * 3600 * 1000) + await utimes(dbPath, past, past) + await writeFile(`${dbPath}-wal`, 'wal-frames') + + const fp = await fingerprintFile(`${dbPath}#hermes-session=abc`) + expect(fp).not.toBeNull() + // mtime: the fresh WAL wins over the checkpoint-stale main file. + expect(fp!.mtimeMs).toBeGreaterThan(past.getTime() + 3600 * 1000) + // size: main + wal, so WAL growth alone changes the fingerprint. + expect(fp!.sizeBytes).toBe('main-db'.length + 'wal-frames'.length) + }) + + it('folds -wal sibling into a : compound fingerprint (OpenCode session)', async () => { + await mkdir(TMP_DIR, { recursive: true }) + const dbPath = join(TMP_DIR, 'opencode.db') + await writeFile(dbPath, 'oc-db') + const past = new Date(Date.now() - 48 * 3600 * 1000) + await utimes(dbPath, past, past) + await writeFile(`${dbPath}-wal`, 'oc-wal') + + const fp = await fingerprintFile(`${dbPath}:ses_abc123`) + expect(fp).not.toBeNull() + expect(fp!.mtimeMs).toBeGreaterThan(past.getTime() + 3600 * 1000) + expect(fp!.sizeBytes).toBe('oc-db'.length + 'oc-wal'.length) + }) + + it('folds -wal sibling into a bare SQLite path (copilot agent-traces.db)', async () => { + await mkdir(TMP_DIR, { recursive: true }) + const dbPath = join(TMP_DIR, 'agent-traces.db') + await writeFile(dbPath, 'traces') + const past = new Date(Date.now() - 48 * 3600 * 1000) + await utimes(dbPath, past, past) + await writeFile(`${dbPath}-wal`, 'traces-wal') + + const fp = await fingerprintFile(dbPath) + expect(fp).not.toBeNull() + expect(fp!.mtimeMs).toBeGreaterThan(past.getTime() + 3600 * 1000) + expect(fp!.sizeBytes).toBe('traces'.length + 'traces-wal'.length) + }) + + it('keeps compound fingerprints working when no -wal sibling exists', async () => { + await mkdir(TMP_DIR, { recursive: true }) + const dbPath = join(TMP_DIR, 'state.db') + await writeFile(dbPath, 'main-only') + + const fp = await fingerprintFile(`${dbPath}#hermes-session=abc`) + expect(fp).not.toBeNull() + expect(fp!.sizeBytes).toBe('main-only'.length) + }) + + it('does not fold sibling files into non-SQLite fingerprints', async () => { + await mkdir(TMP_DIR, { recursive: true }) + const filePath = join(TMP_DIR, 'session.jsonl') + await writeFile(filePath, 'jsonl-data') + // A stray neighbor that happens to match the -wal naming must not leak + // into a transcript fingerprint (offset-based append detection relies on + // sizeBytes being the transcript's real byte length). + await writeFile(`${filePath}-wal`, 'stray') + + const fp = await fingerprintFile(filePath) + expect(fp).not.toBeNull() + expect(fp!.sizeBytes).toBe('jsonl-data'.length) + }) }) // ── reconcileFile ──────────────────────────────────────────────────────