Merge remote-tracking branch 'origin/main' into pr995-rebase

CHANGELOG.md

@@ -28,6 +28,7 @@
 - **The resident `codeburn serve` child.** The first real panel request is also the cache warm-up, so startup never runs an artificial warm-up query beside a duplicate one-shot child; each served command carries its own read-only option allowlist, and anything outside it falls back to a normal spawn; the child exits when its stdin closes, so it can never outlive the app. Requests whose response exceeds the 16 MiB frame limit still replace the child, but that deliberate kill no longer spends the resident's unexpected-death budget. (#972)
 
 ### Fixed
+- **Old durable sources remain visible while they still exist.** The 90-day session-cache age-out now applies only after a durable source disappears from discovery, so an unchanged older Copilot source keeps reporting usage and reuses its persisted fingerprint instead of being reparsed and immediately discarded. (#987) On long-lived machines this makes previously dropped history reappear, so lifetime totals can jump once after upgrading.
 - **`optimize` no longer offers `claude mcp remove` for claude.ai connectors, and its MCP schema-cost estimate is per session.** A `claude_ai_*` namespace that no readable local MCP config claims is a claude.ai connector, managed through `/mcp` or claude.ai Settings rather than as a local MCP server (a local server that carries the prefix keeps its removal command and gains a same-name connector note); low-coverage findings now render them as a manual follow-up and build `--apply` plans only for exact local server names found in readable MCP config, so mixed findings remove only the local subset and the "apply-able" subtotal counts only that subset. The same change replaces the old global schema-cost cap with per-session, per-server proportional attribution — a more accurate model that lowers `mcp-low-coverage` estimates for everyone, connectors or not (on a large corpus roughly by half). (#975, #991)
 - **Bash command splitting was quadratic on long whitespace-heavy commands.** The separator regex retried its leading `\s*` from every offset; matching the separator alone and widening over whitespace by hand makes cold parse ~24% and warm ~40% faster on large corpora, output unchanged.
 - **Cold parse no longer retains full message bodies through cached previews.** `flatSlice` skipped its Buffer round-trip for strings already within the bound, but provider adapters pre-truncate user-message previews with `.slice(0, 500)` before the cache-site call — those pre-sliced views are still V8 SlicedStrings pinning their large parent, so the retention that OOM'd cold parses of large histories survived. The round-trip now always runs.

docs/providers/copilot.md

@@ -39,7 +39,8 @@ instead of trying to dedupe across stores.
   wrong-schema, OTel is skipped and the JSONL/transcript sources are used as a fallback.
 - **Durable cache (monotonic totals).** Copilot is marked `durableSources`: OTel-derived
   cache entries are never evicted when VS Code prunes old spans from the DB, so
-  month-to-date totals do not drop as the DB rotates. Entries age out after 90 days.
+  month-to-date totals do not drop as the DB rotates. Orphaned entries age out after
+  90 days; sources still present in discovery remain cached regardless of call age.
 - **Upgrade note.** The first run after upgrading to the OTel version bumps the copilot
   parse version, which discards the prior copilot cache. Spans already pruned from the DB
   before the upgrade cannot be recovered, so monotonicity starts from the upgrade point,

src/parser.ts

@@ -3271,8 +3271,9 @@ async function parseProviderSources(
     }
   }
 
-  // 90-day age-out for durable providers: remove entries whose newest call is
-  // older than 90 days so the cache doesn't grow unboundedly over time.
+  // 90-day age-out for durable providers: prune only orphaned entries whose
+  // newest call is older than 90 days. Still-discovered sources remain live
+  // regardless of age and keep their persisted fingerprint for reuse.
   if (!readOnly && provider.durableSources) {
     const cutoffMs = Date.now() - 90 * 24 * 60 * 60 * 1000
     for (const [cachedPath, cachedFile] of Object.entries(section.files)) {
@@ -3281,7 +3282,7 @@ async function parseProviderSources(
         .map(c => new Date(c.timestamp).getTime())
         .filter(ts => !isNaN(ts))
         .reduce((max, ts) => Math.max(max, ts), 0)
-      if (newestTs > 0 && newestTs < cutoffMs) {
+      if (!allDiscoveredFiles.has(cachedPath) && newestTs > 0 && newestTs < cutoffMs) {
         delete section.files[cachedPath]
         markCacheDirty(diskCache, providerName, cachedPath)
       }

tests/parser.test.ts

@@ -24,6 +24,7 @@ import type { SessionSource, SessionParser, ParsedProviderCall } from '../src/pr
 let _synthSources: SessionSource[] = []
 let _synthDurable = false
 let _synthYields: ParsedProviderCall[] = []
+let _synthParseCalls = 0
 let _synthOnParse: (() => void | Promise<void>) | null = null
 
 vi.mock('../src/providers/index.js', async (importOriginal) => {
@@ -54,6 +55,7 @@ vi.mock('../src/providers/index.js', async (importOriginal) => {
           createSessionParser(_s: SessionSource, _k: Set<string>): SessionParser {
             return {
               async *parse(): AsyncGenerator<ParsedProviderCall> {
+                _synthParseCalls++
                 await _synthOnParse?.()
                 for (const call of _synthYields) {
                   // Respect seenKeys so that when multiple sources share the same
@@ -193,6 +195,7 @@ beforeEach(async () => {
   _synthSources = []
   _synthDurable = false
   _synthYields  = []
+  _synthParseCalls = 0
   _synthOnParse = null
 })
 
@@ -360,7 +363,7 @@ describe('(d) non-durable provider evicts deleted sources', () => {
 // (e) 90-day age-out: orphan ≥ 91d old is pruned; ≤ 89d is retained
 // ═══════════════════════════════════════════════════════════════════════════
 describe('(e) 90-day age-out for durable providers', () => {
-  it('prunes an orphaned cache entry whose newest call is 91 days old', async () => {
+  it('keeps a discovered 91-day source persisted until discovery removes it', async () => {
     const synthFile = join(tmpHome, 'synth-age.txt')
     await writeFile(synthFile, 'placeholder')
 
@@ -380,15 +383,76 @@ describe('(e) 90-day age-out for durable providers', () => {
       userMessage: 'old', sessionId: 'synth-old',
     }]
 
-    // First parse: cached with 91d-old timestamp → immediately pruned by 90-day check
+    // First refresh: a still-discovered durable source is live and persisted,
+    // regardless of the age of its newest call.
     const proj1 = await parseAllSessions(undefined, 'test-synthetic')
-    expect(totalOutput(proj1)).toBe(0)  // pruned right away
+    expect.soft(totalOutput(proj1)).toBe(8)
+    expect.soft(_synthParseCalls).toBe(1)
 
-    // Confirm: entry is not in the persistent cache after first parse
+    const cache1 = await loadCache()
+    const persisted1 = cache1.providers['test-synthetic']?.files[synthFile]
+    expect.soft(persisted1).toBeDefined()
+
+    // Second refresh: force the public seam through the persisted cache. The
+    // unchanged fingerprint must serve the cached parse without invoking the
+    // provider parser again.
     clearSessionCache()
-    _synthSources = []  // no longer discovered
     const proj2 = await parseAllSessions(undefined, 'test-synthetic')
-    expect(totalOutput(proj2)).toBe(0)
+    expect.soft(totalOutput(proj2)).toBe(8)
+    expect.soft(_synthParseCalls).toBe(1)
+
+    const cache2 = await loadCache()
+    expect.soft(cache2.providers['test-synthetic']?.files[synthFile]?.fingerprint)
+      .toEqual(persisted1?.fingerprint)
+
+    // Third refresh: once discovery removes the old source, it becomes an
+    // orphan and the durable 90-day age-out prunes it from results and disk.
+    clearSessionCache()
+    _synthSources = []
+    const proj3 = await parseAllSessions(undefined, 'test-synthetic')
+    expect.soft(totalOutput(proj3)).toBe(0)
+
+    const cache3 = await loadCache()
+    expect.soft(cache3.providers['test-synthetic']?.files[synthFile]).toBeUndefined()
+  })
+
+  it('keeps a discovered 91-day source through a month-scoped refresh', async () => {
+    const synthFile = join(tmpHome, 'synth-scoped.txt')
+    await writeFile(synthFile, 'placeholder')
+
+    const ts91dAgo = new Date(Date.now() - 91 * 24 * 60 * 60 * 1000).toISOString()
+
+    _synthDurable = true
+    _synthSources = [{ path: synthFile, project: 'test', provider: 'test-synthetic' }]
+    _synthYields  = [{
+      provider: 'test-synthetic', model: 'gpt-4o',
+      inputTokens: 10, outputTokens: 8,
+      cacheCreationInputTokens: 0, cacheReadInputTokens: 0,
+      cachedInputTokens: 0, reasoningTokens: 0, webSearchRequests: 0,
+      costUSD: 0.002, tools: [], bashCommands: [],
+      timestamp: ts91dAgo,
+      speed: 'standard',
+      deduplicationKey: 'synth-age-out-91d-scoped',
+      userMessage: 'old', sessionId: 'synth-old-scoped',
+    }]
+
+    expect.soft(totalOutput(await parseAllSessions(undefined, 'test-synthetic'))).toBe(8)
+
+    // A today-ranged refresh loads under a month scope that excludes the entry's
+    // shard. Durable providers are never scoped, so the age-out still sees the
+    // entry as discovered and the save must carry its month across intact.
+    clearSessionCache()
+    const today = new Date()
+    const start = new Date(today); start.setHours(0, 0, 0, 0)
+    const end   = new Date(today); end.setHours(23, 59, 59, 999)
+    expect.soft(totalOutput(await parseAllSessions({ start, end }, 'test-synthetic'))).toBe(0)
+
+    clearSessionCache()
+    expect.soft(totalOutput(await parseAllSessions(undefined, 'test-synthetic'))).toBe(8)
+    expect.soft(_synthParseCalls).toBe(1)
+
+    const cache = await loadCache()
+    expect.soft(cache.providers['test-synthetic']?.files[synthFile]).toBeDefined()
   })
 
   it('retains an orphaned cache entry whose newest call is 89 days old', async () => {