diff --git a/app/renderer/App.test.tsx b/app/renderer/App.test.tsx index 01b8cca..42ed0f1 100644 --- a/app/renderer/App.test.tsx +++ b/app/renderer/App.test.tsx @@ -2,10 +2,11 @@ import { act, fireEvent, render, screen, waitFor } from '@testing-library/react' import { beforeEach, describe, expect, it, vi } from 'vitest' -import { App, overviewMemoKey } from './App' +import { App, overviewMemoKey, topCategoryByModel, usageSnapshotProps } from './App' +import { sanitizeProps } from '../electron/telemetry' import { __resetPolledMemo, hasPolledMemo, primePolledMemo } from './hooks/usePolled' import { setActiveCurrency } from './lib/format' -import type { DateRange, MenubarPayload, OptimizeJsonReport, SpendFlow } from './lib/types' +import type { DateRange, MenubarPayload, ModelReportRow, OptimizeJsonReport, SpendFlow } from './lib/types' const stored = new Map() vi.stubGlobal('localStorage', { @@ -624,3 +625,104 @@ describe('currency correctness', () => { expect(hasPolledMemo('sentinel-warmed-key')).toBe(false) }) }) + +describe('usage_snapshot telemetry props', () => { + // The renderer builds these props; the main-process sanitizer (sanitizeProps) + // is the last gate before the wire. Test the composition, which is what ships. + function enrichedPayload(): MenubarPayload { + const p = overviewPayload() + p.current.cost = 42 + p.current.topModels = [ + { name: 'claude-opus-4-8', cost: 30, savingsUSD: 0, savingsBaselineModel: '', calls: 400 }, + { name: 'M'.repeat(80), cost: 0.5, savingsUSD: 0, savingsBaselineModel: '', calls: 2 }, + ] + p.current.topActivities = [ + { name: 'coding', cost: 20, savingsUSD: 0, turns: 100, oneShotRate: 0.6123 }, + { name: 'debugging', cost: 10, savingsUSD: 0, turns: 40, oneShotRate: null }, + ] + p.current.mcpServers = [ + { name: 'context7', calls: 5 }, + { name: 'S'.repeat(80), calls: 250 }, + { name: 'shadcn', calls: 1500 }, + ] + p.current.skills = [ + { name: 'graphify', turns: 3, cost: 0 }, + { name: 'council', turns: 150, cost: 0 }, + ] + // A path-like project name that MUST NEVER reach telemetry: the snapshot never + // reads topProjects, and this guards against a future field accidentally doing so. + p.current.topProjects = [{ + name: '/Users/torukmakto/secret-client/private-repo', + cost: 42, savingsUSD: 0, sessions: 1, avgCostPerSession: 42, sessionDetails: [], + }] + return p + } + + it('includes MCP servers and skills as names + bucketed usage', () => { + const props = sanitizeProps(usageSnapshotProps(enrichedPayload())) + + const mcp = props.mcpServers as Array<{ name: string; callBucket: string }> + expect(mcp.map(m => [m.name.slice(0, 8), m.callBucket])).toEqual([ + ['context7', '1-10'], + ['SSSSSSSS', '100-1k'], + ['shadcn', '1k+'], + ]) + + const skills = props.skills as Array<{ name: string; callBucket: string }> + // Skills are measured in turns; buckets mirror the count scale. + expect(skills).toEqual([ + { name: 'graphify', callBucket: '1-10' }, + { name: 'council', callBucket: '100-1k' }, + ]) + }) + + it('truncates over-long names at the 64-char sanitizer cap', () => { + const props = sanitizeProps(usageSnapshotProps(enrichedPayload())) + const mcp = props.mcpServers as Array<{ name: string }> + const models = props.models as Array<{ name: string }> + expect(mcp[1]!.name.length).toBe(64) + expect(models[1]!.name.length).toBe(64) + }) + + it('never leaks a filesystem path or project name', () => { + const serialized = JSON.stringify(sanitizeProps(usageSnapshotProps(enrichedPayload()))) + expect(serialized).not.toContain('/Users/') + expect(serialized).not.toContain('secret-client') + expect(serialized).not.toContain('private-repo') + }) + + function modelRow(over: Partial & Pick): ModelReportRow { + return { + provider: 'claude', providerDisplayName: 'Claude', category: null, + inputTokens: 0, outputTokens: 0, cacheWriteTokens: 0, cacheReadTokens: 0, totalTokens: 0, + costUSD: 0, savingsUSD: 0, savingsBaselineModel: '', calls: 0, credits: null, ...over, + } + } + + it('crosses each top model with its dominant task category when the report joins', () => { + // The overview model name is the display/short name; for Claude that equals + // modelDisplayName, which is how the by-model report row joins back. + const rows = [ + modelRow({ model: 'claude-opus-4-20260101', modelDisplayName: 'claude-opus-4-8', topCategory: 'coding' }), + ] + const props = sanitizeProps(usageSnapshotProps(enrichedPayload(), topCategoryByModel(rows))) + const models = props.models as Array> + expect(models[0]).toEqual({ name: 'claude-opus-4-8', costBucket: '10-50', topCategory: 'coding' }) + // A model the report has no category for carries name + costBucket only, never a fabricated cross. + expect(Object.keys(models[1]!).sort()).toEqual(['costBucket', 'name']) + }) + + it('still emits a valid snapshot without topCategory when the by-model fetch fails', () => { + // The graceful-degradation path: usageSnapshotProps is called with no category map. + const props = sanitizeProps(usageSnapshotProps(enrichedPayload())) + const models = props.models as Array> + for (const m of models) expect(Object.keys(m).sort()).toEqual(['costBucket', 'name']) + // Everything else the snapshot carries is intact. + expect((props.mcpServers as unknown[]).length).toBe(3) + expect((props.skills as unknown[]).length).toBe(2) + expect(props.categories).toEqual([ + { name: 'coding', oneShotRate: 0.61 }, + { name: 'debugging', oneShotRate: -1 }, + ]) + }) +}) diff --git a/app/renderer/App.tsx b/app/renderer/App.tsx index 23e5f4c..8e48ba1 100644 --- a/app/renderer/App.tsx +++ b/app/renderer/App.tsx @@ -26,7 +26,7 @@ import { Compare } from './sections/Compare' import { Plans } from './sections/Plans' import { Settings, type SettingsPane } from './sections/Settings' import { SpendContent } from './sections/Spend' -import type { DateRange, MenubarPayload, Period, TelemetryStatus } from './lib/types' +import type { DateRange, MenubarPayload, ModelReportRow, Period, TelemetryStatus } from './lib/types' // Bucket raw dollar amounts before they leave the machine: telemetry carries // coarse ranges, never exact spend. @@ -39,21 +39,61 @@ function costBucket(usd: number): string { return '1k+' } +// Bucket occurrence counts (MCP-server / skill invocations) the same way costBucket +// coarsens dollars: telemetry carries usage magnitude, never an exact tally. +function countBucket(n: number): string { + if (n < 10) return '1-10' + if (n < 100) return '10-100' + if (n < 1000) return '100-1k' + return '1k+' +} + +/** Map each model to its dominant task category from the default models report. + * `topCategory` is computed only in that view (not `--by-task`). The overview's + * `topModels[].name` is the provider display name — for Claude that's exactly + * `modelDisplayName`, so we key on both it and the raw `model` id and take the + * highest-cost row per key (rows arrive cost-descending). */ +export function topCategoryByModel(rows: ModelReportRow[]): Map { + const map = new Map() + for (const row of rows) { + if (!row.topCategory) continue + if (!map.has(row.modelDisplayName)) map.set(row.modelDisplayName, row.topCategory) + if (!map.has(row.model)) map.set(row.model, row.topCategory) + } + return map +} + /** The once-per-day anonymous aggregate (main process dedups by calendar day). */ -function usageSnapshotProps(payload: MenubarPayload): Record { +export function usageSnapshotProps(payload: MenubarPayload, modelCategories?: Map): Record { return { period: payload.current.label, providerCount: Object.keys(payload.current.providers).length, costBucket: costBucket(payload.current.cost), - models: (payload.current.topModels ?? []).slice(0, 8).map(model => ({ - name: model.name, - costBucket: costBucket(model.cost), - })), + // Each top model with its coarse cost bucket, and — when the once-daily + // by-model report joins — its dominant task category (a single name string, + // never an array, so the sanitizer keeps it). This is the model x purpose cross. + models: (payload.current.topModels ?? []).slice(0, 8).map(model => { + const entry: Record = { name: model.name, costBucket: costBucket(model.cost) } + const topCategory = modelCategories?.get(model.name) + if (topCategory) entry.topCategory = topCategory + return entry + }), + // Aggregate task categories (the "purpose" dimension across all models). categories: (payload.current.topActivities ?? []).slice(0, 12).map(activity => ({ name: activity.name, // Task-completion signal: share of turns resolved in one shot, 2dp. oneShotRate: activity.oneShotRate == null ? -1 : Math.round(activity.oneShotRate * 100) / 100, })), + // MCP servers and skills by name + bucketed usage. Names are config identifiers + // (like model names), never args/paths/descriptions. Skills are measured in turns. + mcpServers: (payload.current.mcpServers ?? []).slice(0, 12).map(server => ({ + name: server.name, + callBucket: countBucket(server.calls), + })), + skills: (payload.current.skills ?? []).slice(0, 12).map(skill => ({ + name: skill.name, + callBucket: countBucket(skill.turns), + })), } } @@ -216,10 +256,25 @@ function AppMain() { // Once-per-day anonymous usage aggregate, only from the canonical view // (all providers, standard period, no config scope) so buckets are stable. + // Gated to the first qualifying render per calendar day so the extra by-model + // report fetch runs at most once/day, not on every poll (main also dedups the + // event). The fetch enriches each model with its dominant task category; if it + // fails we still emit the snapshot, just without the model x category cross. + const snapshotDayRef = useRef(null) useEffect(() => { if (!overview.data || provider !== 'all' || customRange || claudeConfigSource) return - trackEvent('usage_snapshot', usageSnapshotProps(overview.data)) - }, [overview.data, provider, customRange, claudeConfigSource, trackEvent]) + const today = localDateKey(new Date()) + if (snapshotDayRef.current === today) return + snapshotDayRef.current = today + const payload = overview.data + void (async () => { + let modelCategories: Map | undefined + try { + modelCategories = topCategoryByModel(await codeburn.getModels(period, 'all', false)) + } catch { /* degrade: emit the snapshot without per-model topCategory */ } + trackEvent('usage_snapshot', usageSnapshotProps(payload, modelCategories)) + })() + }, [overview.data, provider, customRange, claudeConfigSource, period, trackEvent]) useEffect(() => { let saved: string | null = null