mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-11-22 12:59:06 +00:00
Every finding's dollar figure was its token estimate times one blended rate: 70% of the period's spend over all input-side tokens. That rate is close to a cache-read price, so it misprices anything that is not a cache read. Schema findings already weight cache reads at 0.1, so their reads were discounted twice; findings whose tokens include output were priced as input. A finding can now carry savingsUSD, computed from the calls it names. Every surface reads it through findingUSD: the CLI and TUI optimize views, the JSON report (and so the desktop Optimize page), the menubar payload and MCP get_savings, and the --apply list. classTotals sums it, scaled to the apply-able share for fix findings. Findings without it keep tokensSaved times the blended rate, and any token whose session or model has no price still falls back to that rate. Measured now: - low-worth-sessions: each session's real cost times its recovery fraction - retry-heavy-capabilities: the retry turns' real cost times the fraction - redundant-rereads, build-folder-reads: the reads at the input rate of their session's most-called model - warmup-heavy: the excess cache writes at the session model's write rate - mcp-low-coverage, mcp-project-scope: each call's charged schema tokens at that call model's cache-write or cache-read rate - unused-mcp, mcp-deferral-off, mcp-alwaysload-hygiene, mcp-defer-threshold: the per-session schema at the input rate of the sessions charged Still on the blended rate, because no calls can be named: read-edit-ratio (reads that never happened), context-heavy-sessions, cost-outliers, low-cache-hit-sessions, claude-md-too-long, bash-output-cap, recurring-context and the unused agents, skills and commands.
1118 lines
44 KiB
TypeScript
1118 lines
44 KiB
TypeScript
import { describe, it, expect, vi, beforeAll, afterAll } from 'vitest'
|
|
|
|
import {
|
|
aggregateMcpCoverage,
|
|
buildOptimizeJsonReport,
|
|
classTotals,
|
|
findingClass,
|
|
detectMcpProfileAdvisor,
|
|
detectMcpToolCoverage,
|
|
estimateMcpSchemaCost,
|
|
findingPricing,
|
|
runOptimize,
|
|
} from '../src/optimize.js'
|
|
import type {
|
|
ClassifiedTurn,
|
|
ParsedApiCall,
|
|
ProjectSummary,
|
|
SessionSummary,
|
|
TaskCategory,
|
|
TokenUsage,
|
|
} from '../src/types.js'
|
|
import { setPriceOverrides } from '../src/models.js'
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test fixtures
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const ZERO_USAGE: TokenUsage = {
|
|
inputTokens: 0,
|
|
outputTokens: 0,
|
|
cacheCreationInputTokens: 0,
|
|
cacheReadInputTokens: 0,
|
|
cachedInputTokens: 0,
|
|
reasoningTokens: 0,
|
|
webSearchRequests: 0,
|
|
}
|
|
|
|
function makeCall(opts: {
|
|
tools?: string[]
|
|
cacheCreation?: number
|
|
cacheRead?: number
|
|
cost?: number
|
|
} = {}): ParsedApiCall {
|
|
const tools = opts.tools ?? []
|
|
return {
|
|
provider: 'claude',
|
|
model: 'Opus 4.7',
|
|
usage: {
|
|
...ZERO_USAGE,
|
|
cacheCreationInputTokens: opts.cacheCreation ?? 0,
|
|
cacheReadInputTokens: opts.cacheRead ?? 0,
|
|
},
|
|
costUSD: opts.cost ?? 0,
|
|
tools,
|
|
mcpTools: tools.filter(t => t.startsWith('mcp__')),
|
|
skills: [],
|
|
hasAgentSpawn: false,
|
|
hasPlanMode: false,
|
|
speed: 'standard',
|
|
timestamp: '2026-05-04T00:00:00Z',
|
|
bashCommands: [],
|
|
deduplicationKey: 'k',
|
|
}
|
|
}
|
|
|
|
function makeTurn(calls: ParsedApiCall[]): ClassifiedTurn {
|
|
return {
|
|
userMessage: '',
|
|
assistantCalls: calls,
|
|
timestamp: '2026-05-04T00:00:00Z',
|
|
sessionId: 's1',
|
|
category: 'coding',
|
|
retries: 0,
|
|
hasEdits: false,
|
|
}
|
|
}
|
|
|
|
function makeSession(opts: {
|
|
sessionId?: string
|
|
inventory?: string[]
|
|
turns?: ClassifiedTurn[]
|
|
mcpBreakdown?: Record<string, { calls: number }>
|
|
}): SessionSummary {
|
|
const turns = opts.turns ?? []
|
|
const apiCalls = turns.reduce((s, t) => s + t.assistantCalls.length, 0)
|
|
const emptyCategoryBreakdown = {} as Record<TaskCategory, { turns: number; costUSD: number; retries: number; editTurns: number; oneShotTurns: number }>
|
|
return {
|
|
sessionId: opts.sessionId ?? 's1',
|
|
project: 'p',
|
|
firstTimestamp: '2026-05-04T00:00:00Z',
|
|
lastTimestamp: '2026-05-04T00:00:00Z',
|
|
totalCostUSD: 0,
|
|
totalInputTokens: 0,
|
|
totalOutputTokens: 0,
|
|
totalCacheReadTokens: 0,
|
|
totalCacheWriteTokens: 0,
|
|
apiCalls,
|
|
turns,
|
|
modelBreakdown: {},
|
|
toolBreakdown: {},
|
|
mcpBreakdown: opts.mcpBreakdown ?? {},
|
|
bashBreakdown: {},
|
|
categoryBreakdown: emptyCategoryBreakdown,
|
|
skillBreakdown: {},
|
|
...(opts.inventory ? { mcpInventory: opts.inventory } : {}),
|
|
}
|
|
}
|
|
|
|
function project(sessions: SessionSummary[]): ProjectSummary {
|
|
return projectNamed('p', sessions)
|
|
}
|
|
|
|
function projectNamed(name: string, sessions: SessionSummary[]): ProjectSummary {
|
|
return {
|
|
project: name,
|
|
projectPath: `/tmp/${name}`,
|
|
sessions,
|
|
totalCostUSD: 0,
|
|
totalApiCalls: sessions.reduce((s, ses) => s + ses.apiCalls, 0),
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// aggregateMcpCoverage
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('aggregateMcpCoverage', () => {
|
|
it('returns empty list when no session has MCP inventory', () => {
|
|
const projects = [project([makeSession({})])]
|
|
expect(aggregateMcpCoverage(projects)).toEqual([])
|
|
})
|
|
|
|
it('reports per-server tools available, invoked, and unused', () => {
|
|
const inventory = [
|
|
'mcp__hf__hub_repo_search',
|
|
'mcp__hf__paper_search',
|
|
'mcp__hf__hf_doc_search',
|
|
]
|
|
const turns = [
|
|
makeTurn([makeCall({ tools: ['mcp__hf__hub_repo_search'] })]),
|
|
]
|
|
const sessions = [
|
|
makeSession({ inventory, turns, mcpBreakdown: { hf: { calls: 1 } } }),
|
|
]
|
|
const result = aggregateMcpCoverage([project(sessions)])
|
|
|
|
expect(result).toHaveLength(1)
|
|
expect(result[0]!.server).toBe('hf')
|
|
expect(result[0]!.toolsAvailable).toBe(3)
|
|
expect(result[0]!.toolsInvoked).toBe(1)
|
|
expect(result[0]!.unusedTools).toEqual([
|
|
'mcp__hf__hf_doc_search',
|
|
'mcp__hf__paper_search',
|
|
])
|
|
expect(result[0]!.coverageRatio).toBeCloseTo(1 / 3, 5)
|
|
expect(result[0]!.invocations).toBe(1)
|
|
expect(result[0]!.loadedSessions).toBe(1)
|
|
})
|
|
|
|
it('unions inventory across multiple sessions for the same server', () => {
|
|
const sessions = [
|
|
makeSession({ sessionId: 'a', inventory: ['mcp__x__a', 'mcp__x__b'] }),
|
|
makeSession({ sessionId: 'b', inventory: ['mcp__x__b', 'mcp__x__c'] }),
|
|
]
|
|
const result = aggregateMcpCoverage([project(sessions)])
|
|
expect(result[0]!.toolsAvailable).toBe(3)
|
|
expect(result[0]!.loadedSessions).toBe(2)
|
|
})
|
|
|
|
it('separates servers with similar names', () => {
|
|
const sessions = [
|
|
makeSession({ inventory: ['mcp__hf__a', 'mcp__hugface__a'] }),
|
|
]
|
|
const result = aggregateMcpCoverage([project(sessions)])
|
|
expect(result.map(r => r.server).sort()).toEqual(['hf', 'hugface'])
|
|
})
|
|
|
|
it('skips invocations without inventory (foreign server, no inventory observed)', () => {
|
|
// A server can show up only via a call. We still report it so the
|
|
// operator knows it was invoked, but coverage is 0/0 and it is not a
|
|
// candidate for the unused-coverage finding.
|
|
const turns = [makeTurn([makeCall({ tools: ['mcp__ghost__t1'] })])]
|
|
const sessions = [
|
|
makeSession({ turns, mcpBreakdown: { ghost: { calls: 1 } } }),
|
|
]
|
|
const result = aggregateMcpCoverage([project(sessions)])
|
|
// No inventory entry -> aggregator drops the server from the report
|
|
// because we cannot reason about coverage without an inventory baseline.
|
|
expect(result).toEqual([])
|
|
})
|
|
})
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// estimateMcpSchemaCost — cache-aware accounting
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('estimateMcpSchemaCost', () => {
|
|
it('charges first cacheCreation turn at full price, subsequent turns at cache-read', () => {
|
|
const turns = [
|
|
makeTurn([makeCall({ cacheCreation: 50_000 })]), // first turn: write
|
|
makeTurn([makeCall({ cacheRead: 60_000 })]), // ongoing: read
|
|
makeTurn([makeCall({ cacheRead: 60_000 })]),
|
|
]
|
|
const sessions = [makeSession({
|
|
inventory: Array.from({ length: 30 }, (_, i) => `mcp__svc__t${i}`),
|
|
turns,
|
|
mcpBreakdown: { svc: { calls: 0 } },
|
|
})]
|
|
// 30 deferred tools * 9 token name lines = 270 tokens
|
|
const cost = estimateMcpSchemaCost(30, [project(sessions)], 'svc')
|
|
expect(cost.cacheWriteTokens).toBe(270)
|
|
expect(cost.cacheReadTokens).toBe(540) // 270 + 270 across two ongoing turns
|
|
// effective = write * 1.25 + read * 0.10 (cache pricing)
|
|
expect(cost.effectiveInputTokens).toBeCloseTo(270 * 1.25 + 540 * 0.10, 5)
|
|
})
|
|
|
|
it('caps by available cache bucket so we never overclaim', () => {
|
|
const turns = [makeTurn([makeCall({ cacheCreation: 100 })])]
|
|
const sessions = [makeSession({
|
|
inventory: Array.from({ length: 30 }, (_, i) => `mcp__svc__t${i}`),
|
|
turns,
|
|
mcpBreakdown: { svc: { calls: 0 } },
|
|
})]
|
|
// 30*9 = 270 tokens, but the call only had 100 cache-creation,
|
|
// so we should not claim more than 100 of overhead for that turn.
|
|
const cost = estimateMcpSchemaCost(30, [project(sessions)], 'svc')
|
|
expect(cost.cacheWriteTokens).toBe(100)
|
|
})
|
|
|
|
it('returns zero when no unused tools', () => {
|
|
const sessions = [makeSession({
|
|
inventory: ['mcp__svc__t1'],
|
|
turns: [makeTurn([makeCall({ cacheCreation: 5000 })])],
|
|
})]
|
|
const cost = estimateMcpSchemaCost(0, [project(sessions)], 'svc')
|
|
expect(cost).toEqual({ cacheWriteTokens: 0, cacheReadTokens: 0, effectiveInputTokens: 0 })
|
|
})
|
|
|
|
it('charges a call that read the cached prefix once, as a read', () => {
|
|
// Agent loops write new tool output on nearly every call, so a cache
|
|
// write next to a cache read is not a prefix rebuild: the tool lines
|
|
// were read, not written again.
|
|
const turns = [makeTurn([
|
|
makeCall({ cacheCreation: 50_000, cacheRead: 30_000 }),
|
|
])]
|
|
const sessions = [makeSession({
|
|
inventory: Array.from({ length: 30 }, (_, i) => `mcp__svc__t${i}`),
|
|
turns,
|
|
mcpBreakdown: { svc: { calls: 0 } },
|
|
})]
|
|
const cost = estimateMcpSchemaCost(30, [project(sessions)], 'svc')
|
|
expect(cost.cacheWriteTokens).toBe(0)
|
|
expect(cost.cacheReadTokens).toBe(270)
|
|
})
|
|
|
|
it('counts every cache rebuild, not just the first one', () => {
|
|
// Sessions that span more than 5 minutes can rebuild the cache
|
|
// multiple times. The estimator should treat every cacheCreation
|
|
// bucket as another write.
|
|
const turns = [makeTurn([
|
|
makeCall({ cacheCreation: 50_000 }),
|
|
makeCall({ cacheCreation: 50_000 }), // rebuild after cache TTL
|
|
makeCall({ cacheRead: 60_000 }),
|
|
])]
|
|
const sessions = [makeSession({
|
|
inventory: Array.from({ length: 30 }, (_, i) => `mcp__svc__t${i}`),
|
|
turns,
|
|
mcpBreakdown: { svc: { calls: 0 } },
|
|
})]
|
|
const cost = estimateMcpSchemaCost(30, [project(sessions)], 'svc')
|
|
expect(cost.cacheWriteTokens).toBe(540) // both rebuilds counted
|
|
expect(cost.cacheReadTokens).toBe(270)
|
|
})
|
|
|
|
it('skips sessions where the server was never loaded', () => {
|
|
const turns = [makeTurn([makeCall({ cacheCreation: 100_000 })])]
|
|
const sessions = [makeSession({
|
|
inventory: ['mcp__other__t1'],
|
|
turns,
|
|
})]
|
|
const cost = estimateMcpSchemaCost(10, [project(sessions)], 'svc')
|
|
expect(cost.cacheWriteTokens).toBe(0)
|
|
})
|
|
|
|
it('requires observed inventory for the server, not just invocations', () => {
|
|
// Session invoked the server (mcpBreakdown set, mcpTools called) but
|
|
// never reported a deferred_tools_delta for it. Cost should be 0 to
|
|
// stay consistent with aggregateMcpCoverage's loadedSessions rule.
|
|
const turns = [makeTurn([
|
|
makeCall({ tools: ['mcp__svc__t1'], cacheCreation: 100_000 }),
|
|
])]
|
|
const sessions = [makeSession({
|
|
// No inventory at all
|
|
turns,
|
|
mcpBreakdown: { svc: { calls: 1 } },
|
|
})]
|
|
const cost = estimateMcpSchemaCost(10, [project(sessions)], 'svc')
|
|
expect(cost.cacheWriteTokens).toBe(0)
|
|
expect(cost.cacheReadTokens).toBe(0)
|
|
})
|
|
|
|
it('caps combined unused-schema budget across multiple flagged servers', () => {
|
|
// Two flagged servers, each with 30 unused tools (270 tokens each =
|
|
// 540 combined). One call has a 300 cache-creation bucket. The
|
|
// combined cap means total write tokens reported is min(540, 300) =
|
|
// 300, not min(270, 300) per server = 540.
|
|
const inventory = [
|
|
...Array.from({ length: 30 }, (_, i) => `mcp__a__t${i}`),
|
|
...Array.from({ length: 30 }, (_, i) => `mcp__b__t${i}`),
|
|
]
|
|
const turns = [makeTurn([makeCall({ cacheCreation: 300 })])]
|
|
const sessions = [makeSession({ inventory, turns })]
|
|
const cost = estimateMcpSchemaCost(
|
|
{ a: 30, b: 30 },
|
|
[project(sessions)],
|
|
['a', 'b'],
|
|
)
|
|
expect(cost.cacheWriteTokens).toBe(300)
|
|
})
|
|
|
|
it('does not count a duplicated server identifier twice', () => {
|
|
const inventory = Array.from({ length: 20 }, (_, i) => `mcp__svc__t${i}`)
|
|
const sessions = [makeSession({
|
|
inventory,
|
|
turns: [makeTurn([makeCall({ cacheCreation: 50_000 })])],
|
|
})]
|
|
|
|
const cost = estimateMcpSchemaCost(
|
|
{ svc: 20 },
|
|
[project(sessions)],
|
|
['svc', 'svc'],
|
|
)
|
|
|
|
expect(cost.cacheWriteTokens).toBe(180)
|
|
expect(cost.effectiveInputTokens).toBe(225)
|
|
})
|
|
|
|
it('still works with the single-server signature (backward compat)', () => {
|
|
const turns = [makeTurn([makeCall({ cacheCreation: 50_000 })])]
|
|
const sessions = [makeSession({
|
|
inventory: Array.from({ length: 30 }, (_, i) => `mcp__svc__t${i}`),
|
|
turns,
|
|
})]
|
|
const cost = estimateMcpSchemaCost(30, [project(sessions)], 'svc')
|
|
expect(cost.cacheWriteTokens).toBe(270)
|
|
})
|
|
})
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// detectMcpToolCoverage — finding emission with thresholds
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('detectMcpToolCoverage', () => {
|
|
it('returns null when no inventory exists at all', () => {
|
|
expect(detectMcpToolCoverage([project([makeSession({})])])).toBeNull()
|
|
})
|
|
|
|
it('keeps claude.ai connector evidence but emits manual guidance instead of a local remove command', () => {
|
|
const server = 'claude_ai_Netlify'
|
|
const inventory = Array.from({ length: 20 }, (_, i) => `mcp__${server}__t${i}`)
|
|
const turns = [makeTurn([makeCall({ cacheCreation: 50_000 })])]
|
|
const sessions = [
|
|
makeSession({ sessionId: 'a', inventory, turns }),
|
|
makeSession({ sessionId: 'b', inventory, turns }),
|
|
]
|
|
|
|
const finding = detectMcpToolCoverage([project(sessions)])
|
|
|
|
expect(finding).not.toBeNull()
|
|
expect(finding!.tokensSaved).toBe(450)
|
|
// Keep the transcript namespace as evidence, but name the connector the
|
|
// way users actually see it in /mcp and claude.ai Settings.
|
|
expect(finding!.explanation).toContain(server)
|
|
expect(finding!.explanation).toContain('claude.ai Netlify')
|
|
expect(finding!.explanation).toContain('/mcp')
|
|
expect(finding!.explanation).toContain('claude.ai Settings > Connectors')
|
|
expect(finding!.fix.type).toBe('paste')
|
|
if (finding!.fix.type === 'paste') {
|
|
expect(finding!.fix.destination).toBe('manual')
|
|
expect(finding!.fix.text).toContain('/mcp')
|
|
expect(finding!.fix.text).toContain('claude.ai Netlify')
|
|
expect(finding!.fix.text).not.toContain(server)
|
|
expect(finding!.fix.text).toContain('claude.ai Settings > Connectors')
|
|
}
|
|
expect(JSON.stringify(finding)).not.toContain('claude mcp remove')
|
|
expect(finding!.apply).toBeUndefined()
|
|
})
|
|
|
|
it('renders connector-only remediation as a manual action, never an Ask Claude prompt', async () => {
|
|
const server = 'claude_ai_Google_Calendar'
|
|
const inventory = Array.from({ length: 20 }, (_, i) => `mcp__${server}__t${i}`)
|
|
const turns = [makeTurn([makeCall({ cacheCreation: 50_000 })])]
|
|
const projects = [project([
|
|
makeSession({ sessionId: 'a', inventory, turns }),
|
|
makeSession({ sessionId: 'b', inventory, turns }),
|
|
])]
|
|
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
|
|
|
try {
|
|
await runOptimize(projects, 'Test period')
|
|
const output = log.mock.calls.map(args => args.join(' ')).join('\n')
|
|
expect(output).toContain('Manual action')
|
|
expect(output).toContain('claude.ai Google Calendar')
|
|
expect(output).not.toContain('Ask Claude in the current session')
|
|
} finally {
|
|
log.mockRestore()
|
|
}
|
|
})
|
|
|
|
it('keeps the public optimize JSON envelope while marking connector guidance manual', () => {
|
|
const server = 'claude_ai_Slack'
|
|
const coverage = [{
|
|
server,
|
|
toolsAvailable: 20,
|
|
toolsInvoked: 0,
|
|
unusedTools: Array.from({ length: 20 }, (_, i) => `mcp__${server}__t${i}`),
|
|
invocations: 0,
|
|
loadedSessions: 2,
|
|
coverageRatio: 0,
|
|
}]
|
|
const finding = detectMcpToolCoverage([], coverage)!
|
|
|
|
const report = buildOptimizeJsonReport([], 'Test period', {
|
|
findings: [finding],
|
|
costRate: 0,
|
|
healthScore: 90,
|
|
healthGrade: 'A',
|
|
})
|
|
|
|
expect(report.findings[0]).toMatchObject({
|
|
id: 'mcp-low-coverage',
|
|
tokensSaved: 0,
|
|
fix: {
|
|
type: 'paste',
|
|
destination: 'manual',
|
|
text: expect.stringContaining('claude.ai Slack'),
|
|
},
|
|
})
|
|
expect(report.findings[0]).not.toHaveProperty('apply')
|
|
expect(report.findings[0]).not.toHaveProperty('applyTokensSaved')
|
|
expect(report.findings[0]).not.toHaveProperty('applyTokensSavedByServer')
|
|
expect(report.findings[0]).not.toHaveProperty('manualFollowUp')
|
|
})
|
|
|
|
it('intersects globally unused tool identities with each session inventory', () => {
|
|
const server = 'filesystem'
|
|
const coverage = [{
|
|
server,
|
|
toolsAvailable: 20,
|
|
toolsInvoked: 0,
|
|
unusedTools: Array.from({ length: 20 }, (_, i) => `mcp__${server}__t${i}`),
|
|
invocations: 0,
|
|
loadedSessions: 2,
|
|
coverageRatio: 0,
|
|
}]
|
|
const sessions = [5, 20].map((count, index) => makeSession({
|
|
sessionId: `s${index}`,
|
|
inventory: Array.from({ length: count }, (_, i) => `mcp__${server}__t${i}`),
|
|
turns: [makeTurn([makeCall({ cacheCreation: 50_000 })])],
|
|
}))
|
|
|
|
const finding = detectMcpToolCoverage([project(sessions)], coverage)
|
|
|
|
// 5*9 and 20*9 name-line tokens, each at 1.25x cache-write pricing.
|
|
expect(finding).toMatchObject({ tokensSaved: 281 })
|
|
expect(finding!.applyTokensSavedByServer?.filesystem).toBe(281.25)
|
|
})
|
|
|
|
it('conserves a capped cache-read bucket with fractional shares', () => {
|
|
const inventory = [
|
|
...Array.from({ length: 15 }, (_, i) => `mcp__filesystem__t${i}`),
|
|
...Array.from({ length: 11 }, (_, i) => `mcp__claude_ai_Slack__t${i}`),
|
|
]
|
|
const coverage: McpServerCoverage[] = [
|
|
{
|
|
server: 'filesystem', toolsAvailable: 15, toolsInvoked: 0,
|
|
unusedTools: inventory.slice(0, 15), invocations: 0, loadedSessions: 2, coverageRatio: 0,
|
|
},
|
|
{
|
|
server: 'claude_ai_Slack', toolsAvailable: 11, toolsInvoked: 0,
|
|
unusedTools: inventory.slice(15), invocations: 0, loadedSessions: 2, coverageRatio: 0,
|
|
},
|
|
]
|
|
// Duplicate inventory entries must not increase the schema share.
|
|
const sessionInventory = [...inventory, inventory[0]!, inventory[15]!]
|
|
const sessions = ['a', 'b'].map(sessionId => makeSession({
|
|
sessionId,
|
|
inventory: sessionInventory,
|
|
turns: [makeTurn([makeCall({ cacheCreation: 5_001, cacheRead: 101 })])],
|
|
}))
|
|
|
|
const finding = detectMcpToolCoverage([project(sessions)], coverage)!
|
|
// 26 tools * 9 = 234 tokens, capped at the 101-token read; the call read
|
|
// the prefix, so its cache write is new content, not the tool lines.
|
|
const total = 2 * 101 * 0.10
|
|
const local = total * (15 / 26)
|
|
|
|
expect(finding.tokensSaved).toBe(Math.round(total))
|
|
expect(finding.applyTokensSaved).toBe(Math.round(local))
|
|
expect(finding.applyTokensSavedByServer?.filesystem).toBeCloseTo(local, 8)
|
|
})
|
|
|
|
it('pluralises manual guidance when only claude.ai connectors are flagged', () => {
|
|
const coverage = ['claude_ai_Slack', 'claude_ai_Google_Calendar'].map(server => ({
|
|
server,
|
|
toolsAvailable: 20,
|
|
toolsInvoked: 0,
|
|
unusedTools: Array.from({ length: 20 }, (_, i) => `mcp__${server}__t${i}`),
|
|
invocations: 0,
|
|
loadedSessions: 2,
|
|
coverageRatio: 0,
|
|
}))
|
|
|
|
const finding = detectMcpToolCoverage([], coverage)
|
|
|
|
expect(finding).not.toBeNull()
|
|
expect(finding!.fix).toMatchObject({
|
|
type: 'paste',
|
|
destination: 'manual',
|
|
label: 'Manage the underused claude.ai connectors where they load:',
|
|
})
|
|
if (finding!.fix.type === 'paste') {
|
|
expect(finding!.fix.text).toContain('manage them in claude.ai Settings > Connectors')
|
|
}
|
|
expect(finding!.apply).toBeUndefined()
|
|
})
|
|
|
|
it('does not flag a server with healthy coverage', () => {
|
|
const inventory = Array.from({ length: 20 }, (_, i) => `mcp__svc__t${i}`)
|
|
const turns = [makeTurn(
|
|
Array.from({ length: 8 }, (_, i) => makeCall({ tools: [`mcp__svc__t${i}`] })),
|
|
)]
|
|
const sessions = [
|
|
makeSession({ sessionId: 'a', inventory, turns }),
|
|
makeSession({ sessionId: 'b', inventory, turns }),
|
|
]
|
|
// 8/20 = 40% coverage, above the 20% threshold -> no finding
|
|
expect(detectMcpToolCoverage([project(sessions)])).toBeNull()
|
|
})
|
|
|
|
it('does not flag a server with too few tools (signal too noisy)', () => {
|
|
// Below MCP_COVERAGE_MIN_TOOLS=10
|
|
const inventory = ['mcp__svc__a', 'mcp__svc__b']
|
|
const sessions = [
|
|
makeSession({ sessionId: 'a', inventory }),
|
|
makeSession({ sessionId: 'b', inventory }),
|
|
]
|
|
expect(detectMcpToolCoverage([project(sessions)])).toBeNull()
|
|
})
|
|
|
|
it('does not flag if seen in only one session (insufficient evidence)', () => {
|
|
const inventory = Array.from({ length: 20 }, (_, i) => `mcp__svc__t${i}`)
|
|
const sessions = [makeSession({ inventory })]
|
|
expect(detectMcpToolCoverage([project(sessions)])).toBeNull()
|
|
})
|
|
|
|
it('flags a large server with low coverage across multiple sessions', () => {
|
|
const inventory = Array.from({ length: 30 }, (_, i) => `mcp__hf__t${i}`)
|
|
const turns = [makeTurn([
|
|
makeCall({ tools: ['mcp__hf__t0'], cacheCreation: 100_000 }),
|
|
])]
|
|
const sessions = [
|
|
makeSession({ sessionId: 'a', inventory, turns, mcpBreakdown: { hf: { calls: 1 } } }),
|
|
makeSession({ sessionId: 'b', inventory, turns, mcpBreakdown: { hf: { calls: 1 } } }),
|
|
]
|
|
const finding = detectMcpToolCoverage([project(sessions)])
|
|
expect(finding).not.toBeNull()
|
|
expect(finding!.title).toContain('1 MCP server')
|
|
expect(finding!.title).toContain('low tool coverage')
|
|
expect(finding!.explanation).toContain('hf')
|
|
expect(finding!.explanation).toContain('1/30')
|
|
expect(finding!.fix.type).toBe('command')
|
|
expect((finding!.fix as { text: string }).text).toContain("claude mcp remove 'hf'")
|
|
expect(finding!.apply).toEqual({ kind: 'mcp-remove', servers: ['hf'] })
|
|
expect(finding!.tokensSaved).toBeGreaterThan(0)
|
|
})
|
|
|
|
it('keeps mixed connector guidance visible while making only the local server executable', () => {
|
|
const inventory = ['filesystem', 'claude_ai_Slack'].flatMap(server =>
|
|
Array.from({ length: 20 }, (_, i) => `mcp__${server}__t${i}`),
|
|
)
|
|
const sessions: SessionSummary[] = [
|
|
makeSession({ sessionId: 'mixed-a', inventory, turns: [makeTurn([makeCall({ cacheCreation: 50_000 })])] }),
|
|
makeSession({ sessionId: 'mixed-b', inventory, turns: [makeTurn([makeCall({ cacheCreation: 50_000 })])] }),
|
|
]
|
|
|
|
const finding = detectMcpToolCoverage([project(sessions)])
|
|
|
|
expect(finding).not.toBeNull()
|
|
// The finding describes both opportunities: 40 unused tool lines across
|
|
// two sessions = 900 effective tokens. The automatic mutation owns only
|
|
// the 20 local lines = 450; the connector portion remains manual.
|
|
expect(finding).toMatchObject({ tokensSaved: 900, applyTokensSaved: 450 })
|
|
expect(finding!.explanation).toContain('claude_ai_Slack')
|
|
expect(finding!.explanation).toContain('/mcp')
|
|
expect(finding!.explanation).toContain('claude.ai Settings > Connectors')
|
|
expect(finding!.fix).toEqual({
|
|
type: 'command',
|
|
label: 'Remove the underused local server, or trim its tools in your MCP config:',
|
|
text: "claude mcp remove 'filesystem'",
|
|
})
|
|
expect(finding!.apply).toEqual({ kind: 'mcp-remove', servers: ['filesystem'] })
|
|
})
|
|
|
|
it('attributes a capped mixed cache bucket proportionally to the local action', () => {
|
|
const inventory = ['filesystem', 'claude_ai_Slack'].flatMap(server =>
|
|
Array.from({ length: 20 }, (_, i) => `mcp__${server}__t${i}`),
|
|
)
|
|
const sessions = ['a', 'b'].map(sessionId => makeSession({
|
|
sessionId,
|
|
inventory,
|
|
turns: [makeTurn([makeCall({ cacheCreation: 200 })])],
|
|
}))
|
|
|
|
const finding = detectMcpToolCoverage([project(sessions)])
|
|
|
|
// Each call's 200-token cache bucket is shared evenly by two 180-token
|
|
// tool lists. Total: 2 * 200 * 1.25 = 500. The local mutation owns half.
|
|
expect(finding).toMatchObject({ tokensSaved: 500, applyTokensSaved: 250 })
|
|
})
|
|
|
|
it('charges only the flagged servers actually loaded in each session', () => {
|
|
const sessions = ['filesystem', 'claude_ai_Slack'].flatMap(server =>
|
|
['a', 'b'].map(suffix => makeSession({
|
|
sessionId: `${server}-${suffix}`,
|
|
inventory: Array.from({ length: 20 }, (_, i) => `mcp__${server}__t${i}`),
|
|
turns: [makeTurn([makeCall({ cacheCreation: 50_000 })])],
|
|
})),
|
|
)
|
|
|
|
const finding = detectMcpToolCoverage([project(sessions)])
|
|
|
|
// Four sessions each load one 180-token tool list. The combined finding
|
|
// must not charge both to every session merely because both are flagged.
|
|
expect(finding).toMatchObject({ tokensSaved: 900, applyTokensSaved: 450 })
|
|
})
|
|
|
|
it('disambiguates a claude.ai connector from a similarly named local server', () => {
|
|
const sessions: SessionSummary[] = []
|
|
for (const server of ['claude_ai_Netlify', 'netlify']) {
|
|
const inventory = Array.from({ length: 20 }, (_, i) => `mcp__${server}__t${i}`)
|
|
sessions.push(
|
|
makeSession({ sessionId: `${server}-a`, inventory }),
|
|
makeSession({ sessionId: `${server}-b`, inventory }),
|
|
)
|
|
}
|
|
|
|
const finding = detectMcpToolCoverage([project(sessions)])
|
|
|
|
expect(finding).not.toBeNull()
|
|
expect(finding!.explanation).toContain('claude_ai_Netlify')
|
|
expect(finding!.explanation).toContain('separate from any similarly named local MCP server')
|
|
expect(finding!.fix.type).toBe('command')
|
|
if (finding!.fix.type === 'command') {
|
|
expect(finding!.fix.text).toBe("claude mcp remove 'netlify'")
|
|
expect(finding!.fix.text).not.toContain('claude_ai_Netlify')
|
|
}
|
|
expect(finding!.apply).toEqual({ kind: 'mcp-remove', servers: ['netlify'] })
|
|
})
|
|
|
|
it('escalates impact to high when token waste crosses the threshold', () => {
|
|
const inventory = Array.from({ length: 200 }, (_, i) => `mcp__big__t${i}`)
|
|
// 200 tools * 9 tokens = 1,800 per call: one write (2,250 effective) and
|
|
// 100 reads (18,000 effective) is ~20k per session, so ten sessions clear
|
|
// the 200k high-impact threshold.
|
|
const turns = [makeTurn([
|
|
makeCall({ tools: ['mcp__big__t0'], cacheCreation: 50_000 }),
|
|
...Array.from({ length: 100 }, () => makeCall({ cacheRead: 60_000 })),
|
|
])]
|
|
const sessions = Array.from({ length: 10 }, (_, i) =>
|
|
makeSession({ sessionId: `s${i}`, inventory, turns, mcpBreakdown: { big: { calls: 1 } } }),
|
|
)
|
|
const finding = detectMcpToolCoverage([project(sessions)])
|
|
expect(finding).not.toBeNull()
|
|
expect(finding!.impact).toBe('high')
|
|
})
|
|
|
|
it('does not count invocation-only sessions toward loadedSessions', () => {
|
|
// Server `svc` has inventory in only one session, but is invoked in
|
|
// a second session that never observed the schema. Pre-fix this
|
|
// would have satisfied the >=2 session threshold; it must not now.
|
|
const inventory = Array.from({ length: 20 }, (_, i) => `mcp__svc__t${i}`)
|
|
const turns = [makeTurn([
|
|
makeCall({ tools: ['mcp__svc__t0'], cacheCreation: 50_000 }),
|
|
])]
|
|
const sessions = [
|
|
makeSession({ sessionId: 'a', inventory, turns, mcpBreakdown: { svc: { calls: 1 } } }),
|
|
// No inventory — this shouldn't be considered a "loaded" session.
|
|
makeSession({ sessionId: 'b', turns, mcpBreakdown: { svc: { calls: 1 } } }),
|
|
]
|
|
expect(detectMcpToolCoverage([project(sessions)])).toBeNull()
|
|
})
|
|
|
|
it('does not let invocations of un-inventoried tools inflate coverage', () => {
|
|
// Inventory has 20 tools, none invoked. Calls hit a 21st tool that
|
|
// never appeared in any deferred_tools_delta (could be a renamed/
|
|
// removed tool from an older session config). Coverage must stay 0%
|
|
// and unusedCount must not go negative.
|
|
const inventory = Array.from({ length: 20 }, (_, i) => `mcp__svc__t${i}`)
|
|
const turns = [makeTurn([makeCall({ tools: ['mcp__svc__ghost'] })])]
|
|
const sessions = [
|
|
makeSession({ sessionId: 'a', inventory, turns, mcpBreakdown: { svc: { calls: 1 } } }),
|
|
makeSession({ sessionId: 'b', inventory, turns, mcpBreakdown: { svc: { calls: 1 } } }),
|
|
]
|
|
const result = aggregateMcpCoverage([project(sessions)])
|
|
expect(result[0]!.toolsAvailable).toBe(20)
|
|
expect(result[0]!.toolsInvoked).toBe(0)
|
|
expect(result[0]!.coverageRatio).toBe(0)
|
|
expect(result[0]!.unusedTools).toHaveLength(20)
|
|
})
|
|
|
|
it('handles multiple flagged servers and pluralises the title', () => {
|
|
const sessions: SessionSummary[] = []
|
|
for (const server of ['svc1', 'svc2']) {
|
|
const inventory = Array.from({ length: 20 }, (_, i) => `mcp__${server}__t${i}`)
|
|
const turns = [makeTurn([
|
|
makeCall({ tools: [`mcp__${server}__t0`], cacheCreation: 50_000 }),
|
|
])]
|
|
sessions.push(
|
|
makeSession({ sessionId: `${server}-a`, inventory, turns, mcpBreakdown: { [server]: { calls: 1 } } }),
|
|
makeSession({ sessionId: `${server}-b`, inventory, turns, mcpBreakdown: { [server]: { calls: 1 } } }),
|
|
)
|
|
}
|
|
const finding = detectMcpToolCoverage([project(sessions)])
|
|
expect(finding).not.toBeNull()
|
|
expect(finding!.title).toContain('2 MCP servers')
|
|
expect((finding!.fix as { text: string }).text.split('\n')).toHaveLength(2)
|
|
})
|
|
})
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// detectMcpProfileAdvisor — project-scoping recommendations
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('detectMcpProfileAdvisor', () => {
|
|
const smallInventory = Array.from({ length: 4 }, (_, i) => `mcp__github__t${i}`)
|
|
|
|
it('flags a server loaded across projects but invoked only in one hot project', () => {
|
|
const hotTurns = [makeTurn([
|
|
makeCall({ tools: ['mcp__github__t0'], cacheCreation: 10_000 }),
|
|
makeCall({ tools: ['mcp__github__t1'], cacheCreation: 10_000 }),
|
|
])]
|
|
const coldTurns = [makeTurn([makeCall({ cacheCreation: 10_000 })])]
|
|
const projects = [
|
|
projectNamed('api', [
|
|
makeSession({ inventory: smallInventory, turns: hotTurns, mcpBreakdown: { github: { calls: 2 } } }),
|
|
]),
|
|
projectNamed('web', [
|
|
makeSession({ inventory: smallInventory, turns: coldTurns, mcpBreakdown: { github: { calls: 0 } } }),
|
|
]),
|
|
projectNamed('docs', [
|
|
makeSession({ inventory: smallInventory, turns: coldTurns, mcpBreakdown: { github: { calls: 0 } } }),
|
|
]),
|
|
]
|
|
|
|
const finding = detectMcpProfileAdvisor(projects)
|
|
expect(finding).not.toBeNull()
|
|
expect(finding!.title).toContain('project-scoped')
|
|
expect(finding!.explanation).toContain('github')
|
|
expect(finding!.explanation).toContain('/tmp/api')
|
|
expect(finding!.explanation).toContain('/tmp/web')
|
|
expect(finding!.explanation).toContain('/tmp/docs')
|
|
expect(finding!.tokensSaved).toBe(90)
|
|
expect(finding!.fix.type).toBe('paste')
|
|
if (finding!.fix.type === 'paste') {
|
|
expect(finding!.fix.destination).toBe('prompt')
|
|
expect(finding!.fix.text).toContain('Keep github available for /tmp/api')
|
|
expect(finding!.fix.text).toContain('/tmp/web')
|
|
expect(finding!.fix.text).toContain('/tmp/docs')
|
|
}
|
|
})
|
|
|
|
it('scopes the remediation label to --provider codex', () => {
|
|
const hotTurns = [makeTurn([
|
|
makeCall({ tools: ['mcp__github__t0'], cacheCreation: 10_000 }),
|
|
makeCall({ tools: ['mcp__github__t1'], cacheCreation: 10_000 }),
|
|
])]
|
|
const coldTurns = [makeTurn([makeCall({ cacheCreation: 10_000 })])]
|
|
const projects = [
|
|
projectNamed('api', [
|
|
makeSession({ inventory: smallInventory, turns: hotTurns, mcpBreakdown: { github: { calls: 2 } } }),
|
|
]),
|
|
projectNamed('web', [
|
|
makeSession({ inventory: smallInventory, turns: coldTurns, mcpBreakdown: { github: { calls: 0 } } }),
|
|
]),
|
|
projectNamed('docs', [
|
|
makeSession({ inventory: smallInventory, turns: coldTurns, mcpBreakdown: { github: { calls: 0 } } }),
|
|
]),
|
|
]
|
|
const finding = detectMcpProfileAdvisor(projects, undefined, 'codex')
|
|
expect(finding!.fix.label).toBe('Ask Codex to turn this into a project-scoped MCP profile:')
|
|
expect(finding!.fix.label).not.toContain('Claude')
|
|
expect(finding!.fix.label).not.toContain('CLAUDE.md')
|
|
})
|
|
|
|
it('does not flag servers used evenly across loaded projects', () => {
|
|
const projects = ['api', 'web', 'docs'].map(name => projectNamed(name, [
|
|
makeSession({
|
|
inventory: smallInventory,
|
|
turns: [makeTurn([makeCall({ tools: ['mcp__github__t0'], cacheCreation: 10_000 })])],
|
|
mcpBreakdown: { github: { calls: 2 } },
|
|
}),
|
|
]))
|
|
|
|
expect(detectMcpProfileAdvisor(projects)).toBeNull()
|
|
})
|
|
|
|
it('allows a hot profile shared by two projects', () => {
|
|
const projects = [
|
|
projectNamed('api', [
|
|
makeSession({
|
|
inventory: smallInventory,
|
|
turns: [makeTurn([
|
|
makeCall({ tools: ['mcp__github__t0'], cacheCreation: 10_000 }),
|
|
makeCall({ tools: ['mcp__github__t1'], cacheCreation: 10_000 }),
|
|
])],
|
|
mcpBreakdown: { github: { calls: 2 } },
|
|
}),
|
|
]),
|
|
projectNamed('web', [
|
|
makeSession({
|
|
inventory: smallInventory,
|
|
turns: [makeTurn([
|
|
makeCall({ tools: ['mcp__github__t0'], cacheCreation: 10_000 }),
|
|
makeCall({ tools: ['mcp__github__t1'], cacheCreation: 10_000 }),
|
|
])],
|
|
mcpBreakdown: { github: { calls: 2 } },
|
|
}),
|
|
]),
|
|
projectNamed('docs', [
|
|
makeSession({
|
|
inventory: smallInventory,
|
|
turns: [makeTurn([makeCall({ cacheCreation: 10_000 })])],
|
|
mcpBreakdown: { github: { calls: 0 } },
|
|
}),
|
|
]),
|
|
projectNamed('playground', [
|
|
makeSession({
|
|
inventory: smallInventory,
|
|
turns: [makeTurn([makeCall({ cacheCreation: 10_000 })])],
|
|
mcpBreakdown: { github: { calls: 0 } },
|
|
}),
|
|
]),
|
|
]
|
|
|
|
const finding = detectMcpProfileAdvisor(projects)
|
|
expect(finding).not.toBeNull()
|
|
expect(finding!.explanation).toContain('/tmp/api')
|
|
expect(finding!.explanation).toContain('/tmp/web')
|
|
expect(finding!.explanation).toContain('/tmp/docs')
|
|
expect(finding!.explanation).toContain('/tmp/playground')
|
|
})
|
|
|
|
it('caps profile savings once when multiple candidate servers share cold sessions', () => {
|
|
const githubInventory = Array.from({ length: 4 }, (_, i) => `mcp__github__t${i}`)
|
|
const slackInventory = Array.from({ length: 4 }, (_, i) => `mcp__slack__t${i}`)
|
|
const inventory = [...githubInventory, ...slackInventory]
|
|
const projects = [
|
|
projectNamed('api', [
|
|
makeSession({
|
|
inventory,
|
|
turns: [makeTurn([
|
|
makeCall({ tools: ['mcp__github__t0'] }),
|
|
makeCall({ tools: ['mcp__github__t1'] }),
|
|
makeCall({ tools: ['mcp__slack__t0'] }),
|
|
makeCall({ tools: ['mcp__slack__t1'] }),
|
|
])],
|
|
mcpBreakdown: { github: { calls: 2 }, slack: { calls: 2 } },
|
|
}),
|
|
]),
|
|
projectNamed('web', [
|
|
makeSession({
|
|
inventory,
|
|
turns: [makeTurn([makeCall({ cacheCreation: 50 })])],
|
|
mcpBreakdown: { github: { calls: 0 }, slack: { calls: 0 } },
|
|
}),
|
|
]),
|
|
projectNamed('docs', [
|
|
makeSession({
|
|
inventory,
|
|
turns: [makeTurn([makeCall({ cacheCreation: 50 })])],
|
|
mcpBreakdown: { github: { calls: 0 }, slack: { calls: 0 } },
|
|
}),
|
|
]),
|
|
]
|
|
|
|
const finding = detectMcpProfileAdvisor(projects)
|
|
expect(finding).not.toBeNull()
|
|
expect(finding!.title).toContain('2 MCP servers')
|
|
expect(finding!.explanation).toContain('github')
|
|
expect(finding!.explanation).toContain('slack')
|
|
// 2 * 36 tool-line tokens capped once at the 50-token bucket, per cold
|
|
// session: 2 * 50 * 1.25.
|
|
expect(finding!.tokensSaved).toBe(125)
|
|
})
|
|
|
|
it('requires at least three loaded projects before recommending a profile', () => {
|
|
const projects = [
|
|
projectNamed('api', [
|
|
makeSession({
|
|
inventory: smallInventory,
|
|
turns: [makeTurn([makeCall({ tools: ['mcp__github__t0'], cacheCreation: 10_000 })])],
|
|
mcpBreakdown: { github: { calls: 2 } },
|
|
}),
|
|
]),
|
|
projectNamed('web', [
|
|
makeSession({
|
|
inventory: smallInventory,
|
|
turns: [makeTurn([makeCall({ cacheCreation: 10_000 })])],
|
|
mcpBreakdown: { github: { calls: 0 } },
|
|
}),
|
|
]),
|
|
]
|
|
|
|
expect(detectMcpProfileAdvisor(projects)).toBeNull()
|
|
})
|
|
|
|
it('does not duplicate low tool coverage findings for the same server', () => {
|
|
const inventory = Array.from({ length: 12 }, (_, i) => `mcp__huge__t${i}`)
|
|
const projects = [
|
|
projectNamed('api', [
|
|
makeSession({
|
|
inventory,
|
|
turns: [makeTurn([makeCall({ tools: ['mcp__huge__t0'], cacheCreation: 20_000 })])],
|
|
mcpBreakdown: { huge: { calls: 3 } },
|
|
}),
|
|
]),
|
|
projectNamed('web', [
|
|
makeSession({
|
|
inventory,
|
|
turns: [makeTurn([makeCall({ cacheCreation: 20_000 })])],
|
|
mcpBreakdown: { huge: { calls: 0 } },
|
|
}),
|
|
]),
|
|
projectNamed('docs', [
|
|
makeSession({
|
|
inventory,
|
|
turns: [makeTurn([makeCall({ cacheCreation: 20_000 })])],
|
|
mcpBreakdown: { huge: { calls: 0 } },
|
|
}),
|
|
]),
|
|
]
|
|
const coverage = [{
|
|
server: 'huge',
|
|
toolsAvailable: 12,
|
|
toolsInvoked: 1,
|
|
unusedTools: Array.from({ length: 11 }, (_, i) => `mcp__huge__t${i + 1}`),
|
|
invocations: 3,
|
|
loadedSessions: 3,
|
|
coverageRatio: 1 / 12,
|
|
}]
|
|
|
|
expect(detectMcpProfileAdvisor(projects, coverage)).toBeNull()
|
|
})
|
|
})
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Connector findings under the fix/nudge/keep classification (#1019)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('connector findings and finding class', () => {
|
|
const inventoryFor = (servers: string[]) => servers.flatMap(server =>
|
|
Array.from({ length: 20 }, (_, i) => `mcp__${server}__t${i}`),
|
|
)
|
|
const twoSessions = (servers: string[]) => ['a', 'b'].map(sessionId => makeSession({
|
|
sessionId,
|
|
inventory: inventoryFor(servers),
|
|
turns: [makeTurn([makeCall({ cacheCreation: 50_000 })])],
|
|
}))
|
|
|
|
it('classifies a connector-only finding as a nudge, since nothing is appliable', () => {
|
|
const finding = detectMcpToolCoverage([project(twoSessions(['claude_ai_Gmail']))])
|
|
|
|
expect(finding).not.toBeNull()
|
|
expect(finding!.apply).toBeUndefined()
|
|
expect(findingClass(finding!)).toBe('nudge')
|
|
expect(finding!.tokensSaved).toBeGreaterThan(0)
|
|
// Never lands in the "apply-able" subtotal.
|
|
expect(classTotals([finding!], 0.00002).fix).toEqual({ tokensSaved: 0, savingsUSD: 0, count: 0 })
|
|
})
|
|
|
|
it('counts only the local subset of a mixed finding towards the apply-able subtotal', () => {
|
|
const finding = detectMcpToolCoverage([project(twoSessions(['filesystem', 'claude_ai_Slack']))])
|
|
|
|
expect(finding).not.toBeNull()
|
|
expect(findingClass(finding!)).toBe('fix')
|
|
expect(finding).toMatchObject({ tokensSaved: 900, applyTokensSaved: 450 })
|
|
expect(classTotals([finding!], 0.00002).fix).toEqual({ tokensSaved: 450, savingsUSD: expect.closeTo(0.009, 10), count: 1 })
|
|
})
|
|
|
|
it("leaves a local-only finding's subtotal at its full estimate", () => {
|
|
const finding = detectMcpToolCoverage([project(twoSessions(['filesystem']))])
|
|
|
|
expect(finding).not.toBeNull()
|
|
expect(finding!.applyTokensSaved).toBeUndefined()
|
|
expect(classTotals([finding!], 0.00002).fix.tokensSaved).toBe(finding!.tokensSaved)
|
|
})
|
|
|
|
it('charges each session only for the local schemas it actually loaded', () => {
|
|
// Same per-session scoping the connector split relies on, with no
|
|
// connector in play: two flagged local servers in disjoint sessions are
|
|
// charged one schema each, not both schemas everywhere.
|
|
const sessions = ['filesystem', 'playwright'].flatMap(server =>
|
|
['a', 'b'].map(suffix => makeSession({
|
|
sessionId: `${server}-${suffix}`,
|
|
inventory: inventoryFor([server]),
|
|
turns: [makeTurn([makeCall({ cacheCreation: 50_000 })])],
|
|
})),
|
|
)
|
|
|
|
const finding = detectMcpToolCoverage([project(sessions)])
|
|
|
|
expect(finding).toMatchObject({ tokensSaved: 900 })
|
|
expect(finding!.applyTokensSaved).toBeUndefined()
|
|
expect(classTotals([finding!], 0.00002).fix.tokensSaved).toBe(900)
|
|
})
|
|
|
|
it('treats a claude_ai_* name owned by local config as a local server', () => {
|
|
const finding = detectMcpToolCoverage(
|
|
[project(twoSessions(['claude_ai_homegrown']))],
|
|
undefined,
|
|
new Set(['claude_ai_homegrown']),
|
|
)
|
|
|
|
expect(finding!.fix).toEqual({
|
|
type: 'command',
|
|
label: 'Remove the underused local server, or trim its tools in your MCP config:',
|
|
text: "claude mcp remove 'claude_ai_homegrown'",
|
|
})
|
|
expect(finding!.apply).toEqual({ kind: 'mcp-remove', servers: ['claude_ai_homegrown'] })
|
|
expect(findingClass(finding!)).toBe('fix')
|
|
// Local config owns the name, so the finding must not claim it is a connector.
|
|
expect(finding!.explanation).not.toContain('is a claude.ai connector namespace')
|
|
// ...but the transcript cannot rule out a same-name connector.
|
|
expect(finding!.explanation).toContain('If you also use a claude.ai connector named claude_ai_homegrown')
|
|
expect(finding!.manualFollowUp?.label).toBe('Check for a same-name claude.ai connector:')
|
|
// The whole estimate is appliable: nothing is reserved for a connector.
|
|
expect(finding!.applyTokensSaved).toBeUndefined()
|
|
expect(classTotals([finding!], 0.00002).fix.tokensSaved).toBe(finding!.tokensSaved)
|
|
})
|
|
|
|
it('keeps a claude_ai_* name absent from local config a connector', () => {
|
|
const finding = detectMcpToolCoverage(
|
|
[project(twoSessions(['claude_ai_Gmail']))],
|
|
undefined,
|
|
new Set(['filesystem', 'playwright']),
|
|
)
|
|
|
|
expect(finding!.fix.type).toBe('paste')
|
|
expect(finding!.apply).toBeUndefined()
|
|
expect(findingClass(finding!)).toBe('nudge')
|
|
expect(finding!.explanation).toContain('is a claude.ai connector namespace')
|
|
})
|
|
|
|
it('falls back to prefix-only when no local config could be read', () => {
|
|
// Unreadable or absent config contributes no names, which leaves every
|
|
// claude_ai_* namespace on the conservative connector path.
|
|
const finding = detectMcpToolCoverage([project(twoSessions(['claude_ai_homegrown']))])
|
|
|
|
expect(finding!.fix.type).toBe('paste')
|
|
expect(finding!.apply).toBeUndefined()
|
|
expect(findingClass(finding!)).toBe('nudge')
|
|
})
|
|
|
|
it('applies the local entry and notes the connector when a name collides', () => {
|
|
const finding = detectMcpToolCoverage(
|
|
[project(twoSessions(['filesystem', 'claude_ai_Slack']))],
|
|
undefined,
|
|
new Set(['filesystem', 'claude_ai_Slack']),
|
|
)
|
|
|
|
// Both are local: the removal owns both entries and nothing is deferred.
|
|
expect(finding!.apply).toEqual({ kind: 'mcp-remove', servers: ['filesystem', 'claude_ai_Slack'] })
|
|
expect(finding!.applyTokensSaved).toBeUndefined()
|
|
expect(classTotals([finding!], 0.00002).fix.tokensSaved).toBe(900)
|
|
expect(finding!.explanation).not.toContain('is a claude.ai connector namespace')
|
|
expect(finding!.manualFollowUp?.text)
|
|
.toBe('If you also use a claude.ai connector named claude_ai_Slack, manage it with /mcp or in claude.ai Settings > Connectors.')
|
|
})
|
|
})
|
|
|
|
describe('MCP schema findings measured dollars', () => {
|
|
// Per token: cache write 5e-6, cache read 4e-7 for the priced model.
|
|
const MODEL = 'measured-test-model'
|
|
const BLENDED = 1e-6
|
|
beforeAll(() => setPriceOverrides({ [MODEL]: { input: 4, output: 20, cacheCreation: 5, cacheRead: 0.4 } }))
|
|
afterAll(() => setPriceOverrides({}))
|
|
const on = (model: string, call: ParsedApiCall): ParsedApiCall => ({ ...call, model })
|
|
|
|
it('prices low-coverage schema at each call model cache rate, unpriced calls at the blended rate', () => {
|
|
const inventory = Array.from({ length: 20 }, (_, i) => `mcp__big__t${i}`)
|
|
const projects = [project([
|
|
makeSession({ sessionId: 'a', inventory, turns: [makeTurn([on(MODEL, makeCall({ cacheCreation: 50_000 })), on(MODEL, makeCall({ cacheRead: 50_000 }))])] }),
|
|
makeSession({ sessionId: 'b', inventory, turns: [makeTurn([on('unpriced-test-model', makeCall({ cacheCreation: 50_000 }))])] }),
|
|
])]
|
|
const finding = detectMcpToolCoverage(projects, undefined, new Set(['big']), findingPricing(projects, BLENDED))!
|
|
// 20 unused tools x 9 tokens = 180 schema tokens per call.
|
|
expect(finding.savingsUSD).toBeCloseTo(180 * 5e-6 + 180 * 4e-7 + 180 * 1.25 * BLENDED, 12)
|
|
})
|
|
|
|
it('prices project-scope cold schema at the call model cache-write rate', () => {
|
|
const inventory = Array.from({ length: 4 }, (_, i) => `mcp__github__t${i}`)
|
|
const hotTurns = [makeTurn([
|
|
on(MODEL, makeCall({ tools: ['mcp__github__t0'], cacheCreation: 10_000 })),
|
|
on(MODEL, makeCall({ tools: ['mcp__github__t1'], cacheCreation: 10_000 })),
|
|
])]
|
|
const coldTurns = [makeTurn([on(MODEL, makeCall({ cacheCreation: 10_000 }))])]
|
|
const projects = [
|
|
projectNamed('api', [makeSession({ inventory, turns: hotTurns, mcpBreakdown: { github: { calls: 2 } } })]),
|
|
projectNamed('web', [makeSession({ inventory, turns: coldTurns, mcpBreakdown: { github: { calls: 0 } } })]),
|
|
projectNamed('docs', [makeSession({ inventory, turns: coldTurns, mcpBreakdown: { github: { calls: 0 } } })]),
|
|
]
|
|
const finding = detectMcpProfileAdvisor(projects, undefined, undefined, findingPricing(projects, BLENDED))!
|
|
expect(finding.tokensSaved).toBe(90)
|
|
// Two cold calls, each writing 4 tools x 9 tokens.
|
|
expect(finding.savingsUSD).toBeCloseTo(2 * 36 * 5e-6, 12)
|
|
})
|
|
})
|