Merge pull request #862 from AVSRPA1KR/feat/defer-report-baselines

feat(act): measure realized savings for defer-* actions in act report

node_modules

@@ -0,0 +1 @@
+/Users/husamsoboh/codeburn/node_modules

src/act/report.ts

@@ -39,16 +39,27 @@ const HONEST_FOOTER =
   'Estimates are scaled to the measured window for comparability; the at-apply estimate is kept in --json. '
   + 'MCP and archive realized figures are derived from per-session baselines times session counts, not independently measured. '
   + 'Each fix measures only its own metric; effects are never attributed across signals. '
-  + 'Guard rows are correlation, not attribution. Realized numbers are rounded down.'
+  + 'Guard rows are correlation, not attribution. Realized numbers are rounded down. '
+  + 'Deferral rows exclude servers an MCP remove/scope row already measures.'
 
 const MCP_KINDS = new Set<ActionKind>(['mcp-remove', 'mcp-project-scope'])
+// defer-* re-enable native MCP tool deferral (part 2 of #614): the same
+// prefix schema tokens mcp-remove eliminates, deferral moves out of the
+// upfront prefix. Realized the same way — per-session schema tokens times the
+// post-apply sessions that benefited — but "benefited" flips: instead of a
+// server no longer loading, it is deferral having become active (the session
+// now carries a deferred-tools inventory, the detector's own signal).
+const DEFER_KINDS = new Set<ActionKind>(['defer-enable', 'defer-alwaysload', 'defer-threshold'])
 const ARCHIVE_DEF_TOKENS: Partial<Record<ActionKind, number>> = {
   'archive-skill': TOKENS_PER_SKILL_DEF,
   'archive-agent': TOKENS_PER_AGENT_DEF,
   'archive-command': TOKENS_PER_COMMAND_DEF,
 }
 
-export type RealizedStatus = 'measured' | 'reverted' | 'not-measurable'
+// 'pending' means the applied change has not taken effect in any post-apply
+// session yet (e.g. deferral before a client restart) - distinct from
+// 'reverted', which asserts the user undid it.
+export type RealizedStatus = 'measured' | 'reverted' | 'not-measurable' | 'pending'
 
 export type ActReportRow = {
   id: string
@@ -178,6 +189,28 @@ function countSessionsLoading(projects: ProjectSummary[], servers: string[]): nu
   return allSessions(projects).filter(s => sessionLoadsAny(s, servers)).length
 }
 
+// Deferral is active in a session exactly when Claude Code emitted a
+// deferred-tools inventory for it — the same signal the mcp-deferral-off
+// detector uses (its absence, alongside MCP overhead, is what flags a gap).
+function sessionHasDeferralActive(s: SessionSummary): boolean {
+  return (s.mcpInventory?.length ?? 0) > 0
+}
+
+// MCP servers observed loading in the window (via inventory or invocation).
+// defer-enable / defer-threshold re-enable deferral for the whole MCP surface
+// rather than a named set, so the affected servers are derived here.
+function observedMcpServers(projects: ProjectSummary[]): string[] {
+  const servers = new Set<string>()
+  for (const s of allSessions(projects)) {
+    for (const fqn of s.mcpInventory ?? []) {
+      const seg = fqn.split('__')[1]
+      if (seg) servers.add(seg)
+    }
+    for (const server of Object.keys(s.mcpBreakdown)) servers.add(server)
+  }
+  return [...servers]
+}
+
 // A kind whose realized effect is a token saving (everything except guard,
 // which is a dollars/yield correlation, and out-of-scope kinds).
 function isTokenKind(kind: ActionKind): boolean {
@@ -226,6 +259,45 @@ function mcpRow(
   return { ...base, estimatedForWindow, status: 'measured', realizedTokens: Math.floor(perSessionTokens * savedSessions), confidence }
 }
 
+function deferRow(
+  base: ActReportRow, sessions: SessionSummary[],
+  baseline: ActionBaseline, afterStart: Date, now: Date,
+  mcpClaimedServers: ReadonlySet<string>,
+): ActReportRow {
+  // Sum only the servers no MCP row claims (see mcpClaimedServers in
+  // computeActReport), so the same schema tokens are never realized twice.
+  const counted = Object.entries(baseline.metrics).filter(([server]) => !mcpClaimedServers.has(server))
+  const excludedServers = Object.keys(baseline.metrics).length - counted.length
+  const perSessionTokens = counted.reduce((a, [, tokens]) => a + tokens, 0)
+  if (perSessionTokens === 0) {
+    return {
+      ...base,
+      note: excludedServers > 0
+        ? 'not measurable: every server in this baseline is already measured by an MCP remove/scope row'
+        : 'not measurable: empty baseline',
+    }
+  }
+  if (sessions.length === 0) return { ...base, note: 'not measurable: no sessions in the window yet' }
+  const estimatedForWindow = Math.floor(perSessionTokens * sessions.length)
+  // A post-apply session realized the saving only if deferral actually became
+  // active in it. ENABLE_TOOL_SEARCH is read at process start, so sessions
+  // begun before the user restarted still run deferral-off — those aren't
+  // counted, and if none benefited we report it plainly rather than claim a
+  // saving that hasn't taken effect.
+  const deferredSessions = sessions.filter(sessionHasDeferralActive).length
+  const confidence = confidenceFor(sessions.length, baseline, afterStart, now)
+  if (deferredSessions === 0) {
+    return {
+      ...base,
+      estimatedForWindow,
+      status: 'pending',
+      confidence,
+      note: `not yet in effect: deferral is still inactive in ${sessions.length} post-apply session${sessions.length === 1 ? '' : 's'} (takes effect on the next session; the client may not have restarted, or the change was reverted)`,
+    }
+  }
+  return { ...base, estimatedForWindow, status: 'measured', realizedTokens: Math.floor(perSessionTokens * deferredSessions), confidence }
+}
+
 function archiveRow(
   base: ActReportRow, rec: ActionRecord, sessions: SessionSummary[],
   baseline: ActionBaseline, afterStart: Date, now: Date,
@@ -358,7 +430,7 @@ async function modelDefaultRow(
 
 async function computeRow(
   rec: ActionRecord, sessions: SessionSummary[], afterStart: Date, now: Date,
-  opts: ActReportOptions, modelDefaultProjectFound = true,
+  mcpClaimedServers: ReadonlySet<string>, opts: ActReportOptions, modelDefaultProjectFound = true,
 ): Promise<ActReportRow> {
   const estimatedAtApply = rec.baseline?.estimatedTokens ?? 0
   const base: ActReportRow = {
@@ -378,6 +450,7 @@ async function computeRow(
   if (!baseline) return { ...base, note: 'not measurable: no baseline captured at apply time' }
 
   if (MCP_KINDS.has(rec.kind)) return mcpRow(base, rec, sessions, baseline, afterStart, now)
+  if (DEFER_KINDS.has(rec.kind)) return deferRow(base, sessions, baseline, afterStart, now, mcpClaimedServers)
   if (rec.kind in ARCHIVE_DEF_TOKENS) return archiveRow(base, rec, sessions, baseline, afterStart, now)
   if (rec.kind === 'claude-md-rule') return readEditRow(base, sessions, baseline, afterStart, now)
   if (rec.kind === 'shell-config') return { ...base, note: 'not measurable: bash result token sizes are not retained in the summary' }
@@ -434,6 +507,20 @@ export async function computeActReport(opts: ActReportOptions = {}): Promise<Act
   const projects = await loadProjects({ start: windowStart, end: now })
   const costRate = computeInputCostRate(projects)
 
+  // Servers a same-journal MCP row (mcp-remove / mcp-project-scope) already
+  // measures. Deferral baselines for defer-enable / defer-threshold span the
+  // whole observed MCP surface, so without this exclusion a defer row and an
+  // MCP row would both claim the same server's schema tokens over the same
+  // post-apply sessions, inflating totalRealizedTokens. Conservative by
+  // design: the defer row drops the server for its whole window even though
+  // pre-removal sessions were legitimately its own - under-claiming keeps the
+  // footer's "each fix measures only its own metric" literally true.
+  const mcpClaimedServers = new Set<string>()
+  for (const r of active) {
+    if (!MCP_KINDS.has(r.kind) || !r.baseline) continue
+    for (const server of Object.keys(r.baseline.metrics)) mcpClaimedServers.add(server)
+  }
+
   const rows: ActReportRow[] = []
   for (const rec of eligible) {
     const afterStart = new Date(Math.max(new Date(rec.at).getTime(), windowStart.getTime()))
@@ -441,7 +528,7 @@ export async function computeActReport(opts: ActReportOptions = {}): Promise<Act
       ? modelDefaultSessionsInWindow(rec, projects, afterStart, now)
       : undefined
     const sessions = modelDefaultWindow?.sessions ?? sessionsInWindow(projects, afterStart, now)
-    rows.push(await computeRow(rec, sessions, afterStart, now, opts, modelDefaultWindow?.projectFound))
+    rows.push(await computeRow(rec, sessions, afterStart, now, mcpClaimedServers, opts, modelDefaultWindow?.projectFound))
   }
 
   const measuredRows = rows.filter(r => r.status === 'measured' && isTokenKind(r.kind))
@@ -483,6 +570,7 @@ export function buildOptimizeAppliedHeader(report: ActReport): string | null {
 
 function realizedCell(r: ActReportRow): string {
   if (r.status === 'reverted') return 'reverted'
+  if (r.status === 'pending') return 'not yet in effect'
   if (r.status === 'not-measurable') return 'not measurable'
   if (r.correlation) return `abandoned ${r.correlation.abandonedPctThen}% -> ${r.correlation.abandonedPctNow}% (corr.)`
   if (r.kind === 'model-default') return 'correlation'
@@ -585,7 +673,15 @@ function mcpServersFromApply(finding: WasteFinding): string[] {
 }
 
 function needsConfigBaseline(kind: ActionKind): boolean {
-  return MCP_KINDS.has(kind) || kind in ARCHIVE_DEF_TOKENS || kind === 'claude-md-rule' || kind === 'shell-config'
+  return MCP_KINDS.has(kind) || DEFER_KINDS.has(kind) || kind in ARCHIVE_DEF_TOKENS || kind === 'claude-md-rule' || kind === 'shell-config'
+}
+
+// Servers whose upfront schema deferral removes from the prefix. defer-alwaysload
+// names them; defer-enable / defer-threshold re-enable deferral across the whole
+// observed MCP surface.
+function deferServers(finding: WasteFinding, ctx: CaptureCtx): string[] {
+  if (finding.apply?.kind === 'defer-alwaysload') return finding.apply.servers.map(s => s.server)
+  return observedMcpServers(ctx.projects)
 }
 
 export function captureBaseline(finding: WasteFinding, kind: ActionKind, ctx: CaptureCtx): ActionBaseline | undefined {
@@ -608,6 +704,19 @@ export function captureBaseline(finding: WasteFinding, kind: ActionKind, ctx: Ca
     return { ...common, sessions: countSessionsLoading(ctx.projects, servers), metrics }
   }
 
+  if (DEFER_KINDS.has(kind)) {
+    const servers = deferServers(finding, ctx)
+    if (servers.length === 0) return undefined
+    const covByServer = new Map(ctx.coverage.map(c => [c.server, c]))
+    const metrics: Record<string, number> = {}
+    for (const server of servers) {
+      const cov = covByServer.get(server)
+      const tools = cov && cov.toolsAvailable > 0 ? cov.toolsAvailable : TOOLS_PER_MCP_SERVER
+      metrics[server] = tools * TOKENS_PER_MCP_TOOL
+    }
+    return { ...common, sessions: countSessionsLoading(ctx.projects, servers), metrics }
+  }
+
   const defTokens = ARCHIVE_DEF_TOKENS[kind]
   if (defTokens !== undefined) {
     const names = finding.apply?.kind === 'archive' ? finding.apply.names : []

tests/act-report.test.ts

@@ -7,10 +7,12 @@ import { journalPath } from '../src/act/journal.js'
 import {
   buildActReportJson,
   buildOptimizeAppliedHeader,
+  captureBaseline,
   computeActReport,
   renderActReport,
 } from '../src/act/report.js'
 import type { ActionRecord } from '../src/act/types.js'
+import type { WasteFinding } from '../src/optimize.js'
 import type { ClassifiedTurn, ProjectSummary } from '../src/types.js'
 
 type Session = ProjectSummary['sessions'][number]
@@ -584,3 +586,179 @@ describe('json + render shape', () => {
     expect(out).toMatch(/scaled to the measured window/)
   })
 })
+
+// ---------------------------------------------------------------------------
+// defer-* realized deltas (part 2 of #614)
+// ---------------------------------------------------------------------------
+
+function deferRecord(over: Partial<ActionRecord> = {}): ActionRecord {
+  const at = daysAgo(10)
+  return {
+    id: 'd1',
+    at,
+    kind: 'defer-enable',
+    findingId: 'mcp-deferral-off',
+    description: 'Remove the ENABLE_TOOL_SEARCH=false override from settings.json',
+    changes: [],
+    status: 'applied',
+    // 2 servers x 2000 tokens/session = 4000 prefix tokens/session.
+    baseline: { windowDays: 14, capturedAt: at, estimatedTokens: 40_000, sessions: 5, metrics: { everything: 2000, 'fs-tools': 2000 } },
+    ...over,
+  }
+}
+
+// NOTE: an mcpInventory on a post-apply session means the OPPOSITE for defer-*
+// vs mcp-remove. For mcp-remove it is "the server loaded again" (reverted);
+// for defer-* it is "deferral became active" (saved). Same fixture, inverse
+// meaning — exactly the design.
+const DEFERRED = { mcpInventory: ['mcp__everything__get-sum'] }
+
+describe('defer realized delta', () => {
+  it('measures savings across post-apply sessions where deferral became active', async () => {
+    const actionsDir = await writeJournal([deferRecord()])
+    const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), DEFERRED))]) })
+
+    const row = report.rows[0]!
+    expect(row.status).toBe('measured')
+    expect(row.realizedTokens).toBe(20_000) // 4000/session * 5 deferred sessions
+    expect(row.estimatedForWindow).toBe(20_000)
+    expect(report.totalRealizedTokens).toBe(20_000)
+  })
+
+  it('reports "not yet in effect" with zero savings when no post-apply session shows deferral', async () => {
+    const actionsDir = await writeJournal([deferRecord()])
+    // sessions with MCP activity but NO inventory = deferral still off
+    const off = sessionsAt(4, daysAgo(5), { mcpBreakdown: { everything: { calls: 2, savingsUSD: 0, costUSD: 0 } } })
+    const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(off)]) })
+
+    const row = report.rows[0]!
+    expect(row.status).toBe('pending')
+    expect(row.realizedTokens ?? 0).toBe(0)
+    expect(row.note).toMatch(/not yet in effect/)
+    expect(report.totalRealizedTokens).toBe(0)
+  })
+
+  it('counts only the sessions where deferral actually became active (partial)', async () => {
+    const actionsDir = await writeJournal([deferRecord()])
+    const active = sessionsAt(3, daysAgo(5), DEFERRED)
+    const stillOff = sessionsAt(2, daysAgo(4))
+    const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf([...active, ...stillOff])]) })
+
+    const row = report.rows[0]!
+    expect(row.status).toBe('measured')
+    expect(row.realizedTokens).toBe(12_000) // 4000 * 3 active (2 still-off excluded)
+    expect(row.estimatedForWindow).toBe(20_000) // 4000 * all 5 window sessions
+  })
+
+  it('is not measurable when no post-apply sessions exist yet', async () => {
+    const actionsDir = await writeJournal([deferRecord()])
+    const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf([])]) })
+    expect(report.rows[0]!.note).toMatch(/no sessions in the window yet/)
+  })
+
+  it('is not measurable with an empty baseline (zero prefix tokens)', async () => {
+    const rec = deferRecord({ baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 0, sessions: 5, metrics: { everything: 0 } } })
+    const actionsDir = await writeJournal([rec])
+    const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), DEFERRED))]) })
+    expect(report.rows[0]!.note).toMatch(/empty baseline/)
+  })
+
+  it('falls back to the no-baseline note for records applied before baselines existed', async () => {
+    const rec = deferRecord({ baseline: undefined })
+    const actionsDir = await writeJournal([rec])
+    const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), DEFERRED))]) })
+    expect(report.rows[0]!.note).toMatch(/no baseline captured at apply time/)
+  })
+
+  it('measures defer-alwaysload against its named servers', async () => {
+    const rec = deferRecord({
+      kind: 'defer-alwaysload',
+      findingId: 'mcp-alwaysload-hygiene',
+      description: 'Unpin an alwaysLoad MCP server',
+      baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 30_000, sessions: 6, metrics: { 'heavy-server': 5000 } },
+    })
+    const actionsDir = await writeJournal([rec])
+    const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(6, daysAgo(5), DEFERRED))]) })
+    const row = report.rows[0]!
+    expect(row.status).toBe('measured')
+    expect(row.realizedTokens).toBe(30_000) // 5000 * 6
+  })
+
+  it('measures defer-threshold like the other defer kinds', async () => {
+    const rec = deferRecord({ kind: 'defer-threshold', findingId: 'mcp-defer-threshold', description: 'Tighten the auto threshold' })
+    const actionsDir = await writeJournal([rec])
+    const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), DEFERRED))]) })
+    expect(report.rows[0]!.status).toBe('measured')
+    expect(report.rows[0]!.realizedTokens).toBe(20_000)
+  })
+
+  it('excludes servers an mcp-remove row already claims, so the two rows never double count', async () => {
+    const actionsDir = await writeJournal([
+      mcpRecord({ baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 10_000, sessions: 5, metrics: { everything: 2000 } } }),
+      deferRecord(),
+    ])
+    const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), { mcpInventory: ['mcp__fs-tools__ls'] }))]) })
+
+    const mcpRow = report.rows.find(r => r.kind === 'mcp-remove')!
+    const deferRow = report.rows.find(r => r.kind === 'defer-enable')!
+    expect(mcpRow.status).toBe('measured')
+    expect(mcpRow.realizedTokens).toBe(10_000) // 2000 x 5 saved sessions
+    expect(deferRow.status).toBe('measured')
+    expect(deferRow.realizedTokens).toBe(10_000) // 'fs-tools' only; 'everything' is claimed by the mcp row
+    expect(report.totalRealizedTokens).toBe(20_000) // disjoint sum; without dedup it would be 30_000
+  })
+
+  it('is not measurable when every deferred server is already claimed by an MCP row', async () => {
+    const actionsDir = await writeJournal([
+      mcpRecord({ baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 20_000, sessions: 5, metrics: { everything: 2000, 'fs-tools': 2000 } } }),
+      deferRecord(),
+    ])
+    const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5)))]) })
+
+    const deferRow = report.rows.find(r => r.kind === 'defer-enable')!
+    expect(deferRow.status).toBe('not-measurable')
+    expect(deferRow.realizedTokens ?? 0).toBe(0)
+    expect(deferRow.note).toMatch(/already measured by an MCP remove\/scope row/)
+  })
+
+  it('surfaces the pending status to --json consumers instead of asserting a revert', async () => {
+    const actionsDir = await writeJournal([deferRecord()])
+    const off = sessionsAt(4, daysAgo(5), { mcpBreakdown: { everything: { calls: 2, savingsUSD: 0, costUSD: 0 } } })
+    const json = buildActReportJson(await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(off)]) })) as { actions: Array<{ status: string; realizedTokens: number | null; note: string }> }
+    expect(json.actions[0]!.status).toBe('pending')
+    expect(json.actions[0]!.realizedTokens).toBeNull()
+    expect(json.actions[0]!.note).toMatch(/not yet in effect/)
+  })
+})
+
+describe('defer baseline capture', () => {
+  const finding = (apply: WasteFinding['apply']): WasteFinding => ({
+    id: 'mcp-deferral-off',
+    title: 't', explanation: 'e', impact: 'medium', tokensSaved: 40_000,
+    fix: { type: 'command', label: 'l', text: 'x' },
+    apply,
+  })
+  const ctx = (projects: ProjectSummary[]) => ({ projects, coverage: [], windowDays: 14, now: NOW })
+
+  it('derives servers from observed MCP usage for defer-enable', () => {
+    const projects = [projectOf(sessionsAt(3, daysAgo(5), { mcpBreakdown: { everything: { calls: 2, savingsUSD: 0, costUSD: 0 }, 'fs-tools': { calls: 1, savingsUSD: 0, costUSD: 0 } } }))]
+    const b = captureBaseline(finding({ kind: 'defer-enable', cause: 'env-false', settingPath: '/x', settingScope: 'project settings', value: 'false' }), 'defer-enable', ctx(projects))
+    expect(b).toBeDefined()
+    // no coverage -> 5 tools x 400 fallback per server
+    expect(b!.metrics.everything).toBe(2000)
+    expect(b!.metrics['fs-tools']).toBe(2000)
+  })
+
+  it('uses the named servers for defer-alwaysload', () => {
+    const projects = [projectOf(sessionsAt(2, daysAgo(5)))]
+    const b = captureBaseline(finding({ kind: 'defer-alwaysload', servers: [{ server: 'pinned', paths: ['/a/.mcp.json'] }] }), 'defer-alwaysload', ctx(projects))
+    expect(b).toBeDefined()
+    expect(Object.keys(b!.metrics)).toEqual(['pinned'])
+  })
+
+  it('returns undefined when there is no observed MCP surface to defer', () => {
+    const projects = [projectOf(sessionsAt(3, daysAgo(5)))] // no mcpBreakdown, no inventory
+    const b = captureBaseline(finding({ kind: 'defer-enable', cause: 'env-false', settingPath: '/x', settingScope: 'project settings', value: 'false' }), 'defer-enable', ctx(projects))
+    expect(b).toBeUndefined()
+  })
+})