Merge pull request #915 from ozymandiashh/fix/913-sqlite-wal-fingerprint

fix(parser): fold SQLite -wal siblings into source fingerprints

src/session-cache.ts

@@ -607,34 +607,65 @@ async function retryCacheFileMutation(operation: () => Promise<void>): Promise<b
 // append-only transcripts keep changing. Fixing this properly means
 // multi-file fingerprints per source.
 
+// SQLite database files by extension. Bare-db sources (copilot's
+// agent-traces.db) and the virtual-suffix bases below all match one of these.
+const SQLITE_DB_PATH = /\.(db|sqlite3?|vscdb)$/i
+
+/// Fingerprint a SQLite database file together with its `-wal` sibling.
+///
+/// A database in WAL mode parks committed writes in `<db>-wal`; the main
+/// file's stat only moves on checkpoint, and a long-lived writer connection
+/// (hermes, cursor and opencode keep their state DBs open for the life of
+/// the agent process) can defer checkpoints for hours or days. A fingerprint
+/// built from the main file alone then (a) carries an mtime older than the
+/// newest committed data, so the date-range mtime pre-filter in
+/// parseProviderSources skips the source and sessions committed after the
+/// last checkpoint never parse (issue #913: today's Hermes sessions missing
+/// from every report), and (b) does not change between checkpoints, so
+/// reconcileFile keeps serving stale cached turns for sessions that grew.
+/// Folding the WAL sibling in fixes both: the newest mtime wins, and the
+/// sizes add so both WAL growth and a checkpoint (db grows, wal truncates)
+/// move the fingerprint. `-shm` is deliberately ignored — it mutates on
+/// reads too and would churn the fingerprint without any data change.
+async function fingerprintSqliteFile(dbPath: string): Promise<FileFingerprint | null> {
+  try {
+    const s = await stat(dbPath)
+    const wal = await stat(dbPath + '-wal').catch(() => null)
+    return {
+      dev: s.dev,
+      ino: s.ino,
+      mtimeMs: wal ? Math.max(s.mtimeMs, wal.mtimeMs) : s.mtimeMs,
+      sizeBytes: s.size + (wal?.size ?? 0),
+    }
+  } catch {
+    return null
+  }
+}
+
 export async function fingerprintFile(filePath: string): Promise<FileFingerprint | null> {
   try {
     const s = await stat(filePath)
+    // A source path that IS a SQLite database (copilot OTel's agent-traces.db)
+    // needs the same WAL fold as the virtual-suffix forms below.
+    if (SQLITE_DB_PATH.test(filePath)) return fingerprintSqliteFile(filePath)
     return { dev: s.dev, ino: s.ino, mtimeMs: s.mtimeMs, sizeBytes: s.size }
   } catch {
     // Providers encode extra context into source paths using virtual suffixes:
     // - Cursor: `<dbPath>#cursor-ws=<workspace>` (workspace-aware routing)
     // - OpenCode: `<dbPath>:<sessionId>` (session scoping)
+    // - Hermes: `<dbPath>#hermes-session=<sessionId>` (session scoping)
     // These compound paths don't exist on disk; strip the suffix to stat the
-    // underlying file. Try `#` first (rare in real paths), then `:` (must use
-    // lastIndexOf to tolerate Windows drive letters like C:\...).
+    // underlying database. Try `#` first (rare in real paths), then `:` (must
+    // use lastIndexOf to tolerate Windows drive letters like C:\...).
     const hashIdx = filePath.indexOf('#')
     if (hashIdx > 0) {
-      try {
-        const s = await stat(filePath.slice(0, hashIdx))
-        return { dev: s.dev, ino: s.ino, mtimeMs: s.mtimeMs, sizeBytes: s.size }
-      } catch {
-        // fall through to colon check
-      }
+      const fp = await fingerprintSqliteFile(filePath.slice(0, hashIdx))
+      if (fp) return fp
+      // fall through to colon check
     }
     const colonIdx = filePath.lastIndexOf(':')
     if (colonIdx > 0) {
-      try {
-        const s = await stat(filePath.slice(0, colonIdx))
-        return { dev: s.dev, ino: s.ino, mtimeMs: s.mtimeMs, sizeBytes: s.size }
-      } catch {
-        return null
-      }
+      return fingerprintSqliteFile(filePath.slice(0, colonIdx))
     }
     return null
   }

tests/providers/hermes.test.ts

@@ -1,4 +1,4 @@
-import { mkdir, mkdtemp, rm } from 'fs/promises'
+import { mkdir, mkdtemp, rm, utimes, writeFile } from 'fs/promises'
 import { basename, dirname, join } from 'path'
 import { tmpdir } from 'os'
 import { createRequire } from 'node:module'
@@ -432,6 +432,47 @@ skipUnlessSqlite('hermes provider', () => {
     expect(modelTokens.reduce((sum, tokens) => sum + tokens.reasoningTokens, 0)).toBe(22)
   })
 
+  // Regression for issue #913: Hermes writes state.db in WAL mode and keeps
+  // the writer connection open for the life of the agent, so recent sessions
+  // live in state.db-wal while the main file's mtime stays at the last
+  // checkpoint. The date-range mtime pre-filter in parseProviderSources
+  // then reads the source as "older than the range" and skips it, and every
+  // session committed since the last checkpoint disappears from reports.
+  // The fingerprint must fold the -wal sibling in so a stale main-file stat
+  // cannot hide fresh sessions.
+  it('still parses sessions committed to the WAL when the main db stat is checkpoint-stale', async () => {
+    const dbPath = createHermesDb(tmpDir)
+    withTestDb(dbPath, (db) => {
+      insertSession(db, {
+        id: 'wal-session',
+        inputTokens: 100,
+        outputTokens: 20,
+        cacheReadTokens: 0,
+        cacheWriteTokens: 0,
+        reasoningTokens: 0,
+        startedAt: 1779549200,
+        title: 'WAL Session',
+      })
+      db.prepare('INSERT INTO messages (session_id, role, content, timestamp) VALUES (?, ?, ?, ?)')
+        .run('wal-session', 'user', 'Session committed after the last checkpoint', 1779549201)
+    })
+
+    // Simulate the WAL-mode stat shape: the main file's mtime predates the
+    // requested range (last checkpoint days ago), while a fresh -wal sibling
+    // holds the recent commits. The db itself was written in rollback-journal
+    // mode, so SQLite ignores the stray -wal on open; only its stat matters.
+    const beforeRange = new Date('2026-05-20T00:00:00.000Z')
+    await utimes(dbPath, beforeRange, beforeRange)
+    await writeFile(`${dbPath}-wal`, 'wal-frames')
+
+    const { clearSessionCache, parseAllSessions } = await loadParserWithHermesHome(tmpDir, cacheDir)
+    clearSessionCache()
+    const projects = await parseAllSessions(dayRange(), 'hermes')
+    const sessions = projects.flatMap(project => project.sessions)
+    expect(sessions).toHaveLength(1)
+    expect(sessions[0]!.totalInputTokens).toBe(100)
+  })
+
   it('treats sibling profile-like directories as default sessions', async () => {
     const profileLikeDir = join(dirname(tmpDir), `${basename(tmpDir)}-profiles_backup`, 'coder')
     await mkdir(profileLikeDir, { recursive: true })

tests/session-cache.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, beforeEach, describe, expect, it } from 'vitest'
-import { readFile, rm, writeFile, mkdir } from 'fs/promises'
+import { readFile, rm, utimes, writeFile, mkdir } from 'fs/promises'
 import { existsSync } from 'fs'
 import { tmpdir } from 'os'
 import { basename, join } from 'path'
@@ -337,6 +337,78 @@ describe('fingerprintFile', () => {
     expect(fp).not.toBeNull()
     expect(fp!.sizeBytes).toBe(9)
   })
+
+  // SQLite WAL mode parks committed writes in `<db>-wal`; the main file's
+  // stat only moves on checkpoint, which a long-lived writer defers for
+  // hours. A fingerprint from the main file alone reports data older than
+  // what is really committed (issue #913). The WAL sibling must be folded in.
+  it('folds -wal sibling into a # compound fingerprint (Hermes session)', async () => {
+    await mkdir(TMP_DIR, { recursive: true })
+    const dbPath = join(TMP_DIR, 'state.db')
+    await writeFile(dbPath, 'main-db')
+    const past = new Date(Date.now() - 48 * 3600 * 1000)
+    await utimes(dbPath, past, past)
+    await writeFile(`${dbPath}-wal`, 'wal-frames')
+
+    const fp = await fingerprintFile(`${dbPath}#hermes-session=abc`)
+    expect(fp).not.toBeNull()
+    // mtime: the fresh WAL wins over the checkpoint-stale main file.
+    expect(fp!.mtimeMs).toBeGreaterThan(past.getTime() + 3600 * 1000)
+    // size: main + wal, so WAL growth alone changes the fingerprint.
+    expect(fp!.sizeBytes).toBe('main-db'.length + 'wal-frames'.length)
+  })
+
+  it('folds -wal sibling into a : compound fingerprint (OpenCode session)', async () => {
+    await mkdir(TMP_DIR, { recursive: true })
+    const dbPath = join(TMP_DIR, 'opencode.db')
+    await writeFile(dbPath, 'oc-db')
+    const past = new Date(Date.now() - 48 * 3600 * 1000)
+    await utimes(dbPath, past, past)
+    await writeFile(`${dbPath}-wal`, 'oc-wal')
+
+    const fp = await fingerprintFile(`${dbPath}:ses_abc123`)
+    expect(fp).not.toBeNull()
+    expect(fp!.mtimeMs).toBeGreaterThan(past.getTime() + 3600 * 1000)
+    expect(fp!.sizeBytes).toBe('oc-db'.length + 'oc-wal'.length)
+  })
+
+  it('folds -wal sibling into a bare SQLite path (copilot agent-traces.db)', async () => {
+    await mkdir(TMP_DIR, { recursive: true })
+    const dbPath = join(TMP_DIR, 'agent-traces.db')
+    await writeFile(dbPath, 'traces')
+    const past = new Date(Date.now() - 48 * 3600 * 1000)
+    await utimes(dbPath, past, past)
+    await writeFile(`${dbPath}-wal`, 'traces-wal')
+
+    const fp = await fingerprintFile(dbPath)
+    expect(fp).not.toBeNull()
+    expect(fp!.mtimeMs).toBeGreaterThan(past.getTime() + 3600 * 1000)
+    expect(fp!.sizeBytes).toBe('traces'.length + 'traces-wal'.length)
+  })
+
+  it('keeps compound fingerprints working when no -wal sibling exists', async () => {
+    await mkdir(TMP_DIR, { recursive: true })
+    const dbPath = join(TMP_DIR, 'state.db')
+    await writeFile(dbPath, 'main-only')
+
+    const fp = await fingerprintFile(`${dbPath}#hermes-session=abc`)
+    expect(fp).not.toBeNull()
+    expect(fp!.sizeBytes).toBe('main-only'.length)
+  })
+
+  it('does not fold sibling files into non-SQLite fingerprints', async () => {
+    await mkdir(TMP_DIR, { recursive: true })
+    const filePath = join(TMP_DIR, 'session.jsonl')
+    await writeFile(filePath, 'jsonl-data')
+    // A stray neighbor that happens to match the -wal naming must not leak
+    // into a transcript fingerprint (offset-based append detection relies on
+    // sizeBytes being the transcript's real byte length).
+    await writeFile(`${filePath}-wal`, 'stray')
+
+    const fp = await fingerprintFile(filePath)
+    expect(fp).not.toBeNull()
+    expect(fp!.sizeBytes).toBe('jsonl-data'.length)
+  })
 })
 
 // ── reconcileFile ──────────────────────────────────────────────────────