mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-05-17 12:20:43 +00:00
Adds a per-tool optimizer finding for MCP servers whose schema is loaded
on every turn but rarely invoked. Builds on the existing server-level
`detectUnusedMcp` (zero invocations) by reporting partial-use cases:
"loaded 54 tools, called 0" or "loaded 26 tools, called 2 (8% coverage)".
Inventory comes from Claude Code's JSONL `attachment.deferred_tools_delta`
entries: `addedNames` lists the exact tools available at that turn,
including every fully-qualified `mcp__<server>__<tool>` name. We union
across all delta entries in a session (not just the first) because tool
availability can change mid-session when the user reloads MCP config or
a subagent inherits a different tool set. Names that don't match the
`mcp__<server>__<tool>` shape with both segments non-empty are rejected
at extraction so downstream `split('__')` consumers can't be poisoned.
Token-savings estimates are cache-aware. MCP tool schemas live in the
cached prefix of the system prompt: a session pays the full input price
on each cache-creation turn (rebuilds happen every ~5 minutes of
inactivity) and the cache-read discount on subsequent turns. Each call's
contribution is capped at its observed `cacheCreationInputTokens` /
`cacheReadInputTokens` so we never claim more MCP overhead than the
call's own cache buckets could contain.
When multiple servers are flagged, costing happens in a single combined
pass: the per-call cap applies to the total unused-schema budget across
all flagged servers, not per server. Two flagged servers cannot both
independently claim the same call's cache bucket, which would otherwise
overstate `tokensSaved` and misclassify findings as high impact.
A session counts toward `loadedSessions` (and toward the cost estimate)
only if its observed inventory included the server. Pure invocation-only
sessions, where the server appears in `mcpBreakdown` or `call.mcpTools`
without any matching `deferred_tools_delta`, do not satisfy the
`>= 2 sessions` threshold on their own. The same invariant applies in
`estimateMcpSchemaCost` so the two passes agree.
Coverage is computed against the inventory only: invocations of names
not present in any observed inventory (older config, hallucinated tool,
typo) do not inflate `toolsInvoked` and cannot drive `unusedCount`
negative. `toolsInvoked` is derived as `inventory.size - unusedTools.length`
to keep both numbers consistent.
`detectUnusedMcp` and the new detector are explicitly disjoint:
`detectUnusedMcp` skips servers that the coverage detector will report,
not every server that happens to be in any inventory, so a small
inventoried-but-uninvoked server below the coverage thresholds still
gets flagged as "configured but never called."
Thresholds for the coverage finding:
- > 10 tools available (small servers are noise)
- < 20% coverage
- >= 2 sessions with observed inventory
- High impact when total effective tokens >= 200_000 or >= 3 servers flagged
Smoke-tested on a real account: 7 servers flagged across 93 sessions
(`office-word-mcp` 0/54, `notebooklm-mcp` 0/38, `office-ppt-mcp` 0/37,
`excel-mcp-server` 0/25, `github-mcp-server` 2/26, `peekaboo` 3/22, plus
`claude_ai_Asana`). Combined-cap costing keeps `tokensSaved` honest.
Changes:
- src/types.ts: optional `mcpInventory: string[]` on `SessionSummary`.
Provider-agnostic field; currently populated only by the Claude parser.
- src/parser.ts: `extractMcpInventory` walks all entries, validates
fully-qualified names, returns sorted unique list. `buildSessionSummary`
passes it through; field is omitted when empty so JSON exports stay
clean.
- src/optimize.ts: `aggregateMcpCoverage`, `estimateMcpSchemaCost`
(single- and multi-server signatures), `detectMcpToolCoverage`. Wired
into `scanAndDetect`. `detectUnusedMcp` updated to disjoint with the
new detector.
- tests/mcp-coverage.test.ts: 23 cases covering aggregation, costing,
combined-cap behaviour, threshold gates, invocation-only-session
filtering, foreign-tool invocations, cache rebuild events, write+read
on the same call, multi-server pluralisation.
- tests/parser-mcp-inventory.test.ts: 12 cases for the JSONL extractor
including malformed name rejection and tolerant attachment parsing.
- CHANGELOG.md: entry under Unreleased / Added (CLI).
Closes #2
126 lines
4.2 KiB
TypeScript
126 lines
4.2 KiB
TypeScript
import { describe, it, expect } from 'vitest'
|
|
|
|
import { extractMcpInventory } from '../src/parser.js'
|
|
import type { JournalEntry } from '../src/types.js'
|
|
|
|
function entry(overrides: Partial<JournalEntry> & Record<string, unknown>): JournalEntry {
|
|
return { type: 'attachment', ...overrides } as JournalEntry
|
|
}
|
|
|
|
describe('extractMcpInventory', () => {
|
|
it('returns empty array when no entries have an attachment', () => {
|
|
expect(extractMcpInventory([entry({ type: 'user' })])).toEqual([])
|
|
})
|
|
|
|
it('returns empty array when no deferred_tools_delta is present', () => {
|
|
expect(extractMcpInventory([
|
|
entry({ attachment: { type: 'something_else', addedNames: ['mcp__a__b'] } }),
|
|
])).toEqual([])
|
|
})
|
|
|
|
it('extracts mcp__server__tool names from a single delta', () => {
|
|
const result = extractMcpInventory([
|
|
entry({
|
|
attachment: {
|
|
type: 'deferred_tools_delta',
|
|
addedNames: ['Bash', 'Edit', 'mcp__hf__hub_repo_search', 'mcp__hf__paper_search'],
|
|
},
|
|
}),
|
|
])
|
|
expect(result).toEqual(['mcp__hf__hub_repo_search', 'mcp__hf__paper_search'])
|
|
})
|
|
|
|
it('filters out built-in tools (no mcp__ prefix)', () => {
|
|
const result = extractMcpInventory([
|
|
entry({
|
|
attachment: {
|
|
type: 'deferred_tools_delta',
|
|
addedNames: ['Bash', 'Edit', 'WebFetch', 'mcp__svc__t1'],
|
|
},
|
|
}),
|
|
])
|
|
expect(result).toEqual(['mcp__svc__t1'])
|
|
})
|
|
|
|
it('rejects malformed names: empty server segment', () => {
|
|
const result = extractMcpInventory([
|
|
entry({
|
|
attachment: {
|
|
type: 'deferred_tools_delta',
|
|
addedNames: ['mcp____tool', 'mcp__svc__t1'],
|
|
},
|
|
}),
|
|
])
|
|
expect(result).toEqual(['mcp__svc__t1'])
|
|
})
|
|
|
|
it('rejects malformed names: missing tool segment (no second `__`)', () => {
|
|
const result = extractMcpInventory([
|
|
entry({
|
|
attachment: {
|
|
type: 'deferred_tools_delta',
|
|
addedNames: ['mcp__server', 'mcp__svc__t1'],
|
|
},
|
|
}),
|
|
])
|
|
expect(result).toEqual(['mcp__svc__t1'])
|
|
})
|
|
|
|
it('rejects malformed names: empty tool segment (trailing `__`)', () => {
|
|
const result = extractMcpInventory([
|
|
entry({
|
|
attachment: {
|
|
type: 'deferred_tools_delta',
|
|
addedNames: ['mcp__server__', 'mcp__svc__t1'],
|
|
},
|
|
}),
|
|
])
|
|
expect(result).toEqual(['mcp__svc__t1'])
|
|
})
|
|
|
|
it('unions across multiple delta entries (incremental adds)', () => {
|
|
const result = extractMcpInventory([
|
|
entry({ attachment: { type: 'deferred_tools_delta', addedNames: ['mcp__a__t1'] } }),
|
|
entry({ attachment: { type: 'deferred_tools_delta', addedNames: ['mcp__a__t2', 'mcp__b__t1'] } }),
|
|
])
|
|
expect(result).toEqual(['mcp__a__t1', 'mcp__a__t2', 'mcp__b__t1'])
|
|
})
|
|
|
|
it('deduplicates names seen in multiple deltas', () => {
|
|
const result = extractMcpInventory([
|
|
entry({ attachment: { type: 'deferred_tools_delta', addedNames: ['mcp__a__t1', 'mcp__a__t1'] } }),
|
|
entry({ attachment: { type: 'deferred_tools_delta', addedNames: ['mcp__a__t1'] } }),
|
|
])
|
|
expect(result).toEqual(['mcp__a__t1'])
|
|
})
|
|
|
|
it('tolerates missing or non-string addedNames', () => {
|
|
const result = extractMcpInventory([
|
|
entry({ attachment: { type: 'deferred_tools_delta' } }),
|
|
entry({ attachment: { type: 'deferred_tools_delta', addedNames: 'not-an-array' } }),
|
|
entry({ attachment: { type: 'deferred_tools_delta', addedNames: [42, null, 'mcp__svc__t1', undefined] } }),
|
|
])
|
|
expect(result).toEqual(['mcp__svc__t1'])
|
|
})
|
|
|
|
it('tolerates malformed attachment object', () => {
|
|
const result = extractMcpInventory([
|
|
entry({ attachment: null }),
|
|
entry({ attachment: 'string-not-object' }),
|
|
entry({ attachment: { type: 'deferred_tools_delta', addedNames: ['mcp__svc__t1'] } }),
|
|
])
|
|
expect(result).toEqual(['mcp__svc__t1'])
|
|
})
|
|
|
|
it('returns names in sorted order', () => {
|
|
const result = extractMcpInventory([
|
|
entry({
|
|
attachment: {
|
|
type: 'deferred_tools_delta',
|
|
addedNames: ['mcp__zzz__a', 'mcp__aaa__z', 'mcp__mmm__m'],
|
|
},
|
|
}),
|
|
])
|
|
expect(result).toEqual(['mcp__aaa__z', 'mcp__mmm__m', 'mcp__zzz__a'])
|
|
})
|
|
})
|