Merge pull request #736 from getagentseal/feat/telemetry-usage-detail

feat: telemetry captures model x purpose, MCP servers, skills

app/renderer/App.test.tsx

@@ -2,10 +2,11 @@
 import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-import { App, overviewMemoKey } from './App'
+import { App, overviewMemoKey, topCategoryByModel, usageSnapshotProps } from './App'
+import { sanitizeProps } from '../electron/telemetry'
 import { __resetPolledMemo, hasPolledMemo, primePolledMemo } from './hooks/usePolled'
 import { setActiveCurrency } from './lib/format'
-import type { DateRange, MenubarPayload, OptimizeJsonReport, SpendFlow } from './lib/types'
+import type { DateRange, MenubarPayload, ModelReportRow, OptimizeJsonReport, SpendFlow } from './lib/types'
 
 const stored = new Map<string, string>()
 vi.stubGlobal('localStorage', {
@@ -624,3 +625,104 @@ describe('currency correctness', () => {
     expect(hasPolledMemo('sentinel-warmed-key')).toBe(false)
   })
 })
+
+describe('usage_snapshot telemetry props', () => {
+  // The renderer builds these props; the main-process sanitizer (sanitizeProps)
+  // is the last gate before the wire. Test the composition, which is what ships.
+  function enrichedPayload(): MenubarPayload {
+    const p = overviewPayload()
+    p.current.cost = 42
+    p.current.topModels = [
+      { name: 'claude-opus-4-8', cost: 30, savingsUSD: 0, savingsBaselineModel: '', calls: 400 },
+      { name: 'M'.repeat(80), cost: 0.5, savingsUSD: 0, savingsBaselineModel: '', calls: 2 },
+    ]
+    p.current.topActivities = [
+      { name: 'coding', cost: 20, savingsUSD: 0, turns: 100, oneShotRate: 0.6123 },
+      { name: 'debugging', cost: 10, savingsUSD: 0, turns: 40, oneShotRate: null },
+    ]
+    p.current.mcpServers = [
+      { name: 'context7', calls: 5 },
+      { name: 'S'.repeat(80), calls: 250 },
+      { name: 'shadcn', calls: 1500 },
+    ]
+    p.current.skills = [
+      { name: 'graphify', turns: 3, cost: 0 },
+      { name: 'council', turns: 150, cost: 0 },
+    ]
+    // A path-like project name that MUST NEVER reach telemetry: the snapshot never
+    // reads topProjects, and this guards against a future field accidentally doing so.
+    p.current.topProjects = [{
+      name: '/Users/torukmakto/secret-client/private-repo',
+      cost: 42, savingsUSD: 0, sessions: 1, avgCostPerSession: 42, sessionDetails: [],
+    }]
+    return p
+  }
+
+  it('includes MCP servers and skills as names + bucketed usage', () => {
+    const props = sanitizeProps(usageSnapshotProps(enrichedPayload()))
+
+    const mcp = props.mcpServers as Array<{ name: string; callBucket: string }>
+    expect(mcp.map(m => [m.name.slice(0, 8), m.callBucket])).toEqual([
+      ['context7', '1-10'],
+      ['SSSSSSSS', '100-1k'],
+      ['shadcn', '1k+'],
+    ])
+
+    const skills = props.skills as Array<{ name: string; callBucket: string }>
+    // Skills are measured in turns; buckets mirror the count scale.
+    expect(skills).toEqual([
+      { name: 'graphify', callBucket: '1-10' },
+      { name: 'council', callBucket: '100-1k' },
+    ])
+  })
+
+  it('truncates over-long names at the 64-char sanitizer cap', () => {
+    const props = sanitizeProps(usageSnapshotProps(enrichedPayload()))
+    const mcp = props.mcpServers as Array<{ name: string }>
+    const models = props.models as Array<{ name: string }>
+    expect(mcp[1]!.name.length).toBe(64)
+    expect(models[1]!.name.length).toBe(64)
+  })
+
+  it('never leaks a filesystem path or project name', () => {
+    const serialized = JSON.stringify(sanitizeProps(usageSnapshotProps(enrichedPayload())))
+    expect(serialized).not.toContain('/Users/')
+    expect(serialized).not.toContain('secret-client')
+    expect(serialized).not.toContain('private-repo')
+  })
+
+  function modelRow(over: Partial<ModelReportRow> & Pick<ModelReportRow, 'model' | 'modelDisplayName'>): ModelReportRow {
+    return {
+      provider: 'claude', providerDisplayName: 'Claude', category: null,
+      inputTokens: 0, outputTokens: 0, cacheWriteTokens: 0, cacheReadTokens: 0, totalTokens: 0,
+      costUSD: 0, savingsUSD: 0, savingsBaselineModel: '', calls: 0, credits: null, ...over,
+    }
+  }
+
+  it('crosses each top model with its dominant task category when the report joins', () => {
+    // The overview model name is the display/short name; for Claude that equals
+    // modelDisplayName, which is how the by-model report row joins back.
+    const rows = [
+      modelRow({ model: 'claude-opus-4-20260101', modelDisplayName: 'claude-opus-4-8', topCategory: 'coding' }),
+    ]
+    const props = sanitizeProps(usageSnapshotProps(enrichedPayload(), topCategoryByModel(rows)))
+    const models = props.models as Array<Record<string, unknown>>
+    expect(models[0]).toEqual({ name: 'claude-opus-4-8', costBucket: '10-50', topCategory: 'coding' })
+    // A model the report has no category for carries name + costBucket only, never a fabricated cross.
+    expect(Object.keys(models[1]!).sort()).toEqual(['costBucket', 'name'])
+  })
+
+  it('still emits a valid snapshot without topCategory when the by-model fetch fails', () => {
+    // The graceful-degradation path: usageSnapshotProps is called with no category map.
+    const props = sanitizeProps(usageSnapshotProps(enrichedPayload()))
+    const models = props.models as Array<Record<string, unknown>>
+    for (const m of models) expect(Object.keys(m).sort()).toEqual(['costBucket', 'name'])
+    // Everything else the snapshot carries is intact.
+    expect((props.mcpServers as unknown[]).length).toBe(3)
+    expect((props.skills as unknown[]).length).toBe(2)
+    expect(props.categories).toEqual([
+      { name: 'coding', oneShotRate: 0.61 },
+      { name: 'debugging', oneShotRate: -1 },
+    ])
+  })
+})

app/renderer/App.tsx

@@ -26,7 +26,7 @@ import { Compare } from './sections/Compare'
 import { Plans } from './sections/Plans'
 import { Settings, type SettingsPane } from './sections/Settings'
 import { SpendContent } from './sections/Spend'
-import type { DateRange, MenubarPayload, Period, TelemetryStatus } from './lib/types'
+import type { DateRange, MenubarPayload, ModelReportRow, Period, TelemetryStatus } from './lib/types'
 
 // Bucket raw dollar amounts before they leave the machine: telemetry carries
 // coarse ranges, never exact spend.
@@ -39,21 +39,61 @@ function costBucket(usd: number): string {
   return '1k+'
 }
 
+// Bucket occurrence counts (MCP-server / skill invocations) the same way costBucket
+// coarsens dollars: telemetry carries usage magnitude, never an exact tally.
+function countBucket(n: number): string {
+  if (n < 10) return '1-10'
+  if (n < 100) return '10-100'
+  if (n < 1000) return '100-1k'
+  return '1k+'
+}
+
+/** Map each model to its dominant task category from the default models report.
+ * `topCategory` is computed only in that view (not `--by-task`). The overview's
+ * `topModels[].name` is the provider display name — for Claude that's exactly
+ * `modelDisplayName`, so we key on both it and the raw `model` id and take the
+ * highest-cost row per key (rows arrive cost-descending). */
+export function topCategoryByModel(rows: ModelReportRow[]): Map<string, string> {
+  const map = new Map<string, string>()
+  for (const row of rows) {
+    if (!row.topCategory) continue
+    if (!map.has(row.modelDisplayName)) map.set(row.modelDisplayName, row.topCategory)
+    if (!map.has(row.model)) map.set(row.model, row.topCategory)
+  }
+  return map
+}
+
 /** The once-per-day anonymous aggregate (main process dedups by calendar day). */
-function usageSnapshotProps(payload: MenubarPayload): Record<string, unknown> {
+export function usageSnapshotProps(payload: MenubarPayload, modelCategories?: Map<string, string>): Record<string, unknown> {
   return {
     period: payload.current.label,
     providerCount: Object.keys(payload.current.providers).length,
     costBucket: costBucket(payload.current.cost),
-    models: (payload.current.topModels ?? []).slice(0, 8).map(model => ({
-      name: model.name,
-      costBucket: costBucket(model.cost),
-    })),
+    // Each top model with its coarse cost bucket, and — when the once-daily
+    // by-model report joins — its dominant task category (a single name string,
+    // never an array, so the sanitizer keeps it). This is the model x purpose cross.
+    models: (payload.current.topModels ?? []).slice(0, 8).map(model => {
+      const entry: Record<string, unknown> = { name: model.name, costBucket: costBucket(model.cost) }
+      const topCategory = modelCategories?.get(model.name)
+      if (topCategory) entry.topCategory = topCategory
+      return entry
+    }),
+    // Aggregate task categories (the "purpose" dimension across all models).
     categories: (payload.current.topActivities ?? []).slice(0, 12).map(activity => ({
       name: activity.name,
       // Task-completion signal: share of turns resolved in one shot, 2dp.
       oneShotRate: activity.oneShotRate == null ? -1 : Math.round(activity.oneShotRate * 100) / 100,
     })),
+    // MCP servers and skills by name + bucketed usage. Names are config identifiers
+    // (like model names), never args/paths/descriptions. Skills are measured in turns.
+    mcpServers: (payload.current.mcpServers ?? []).slice(0, 12).map(server => ({
+      name: server.name,
+      callBucket: countBucket(server.calls),
+    })),
+    skills: (payload.current.skills ?? []).slice(0, 12).map(skill => ({
+      name: skill.name,
+      callBucket: countBucket(skill.turns),
+    })),
   }
 }
 
@@ -216,10 +256,25 @@ function AppMain() {
 
   // Once-per-day anonymous usage aggregate, only from the canonical view
   // (all providers, standard period, no config scope) so buckets are stable.
+  // Gated to the first qualifying render per calendar day so the extra by-model
+  // report fetch runs at most once/day, not on every poll (main also dedups the
+  // event). The fetch enriches each model with its dominant task category; if it
+  // fails we still emit the snapshot, just without the model x category cross.
+  const snapshotDayRef = useRef<string | null>(null)
   useEffect(() => {
     if (!overview.data || provider !== 'all' || customRange || claudeConfigSource) return
-    trackEvent('usage_snapshot', usageSnapshotProps(overview.data))
-  }, [overview.data, provider, customRange, claudeConfigSource, trackEvent])
+    const today = localDateKey(new Date())
+    if (snapshotDayRef.current === today) return
+    snapshotDayRef.current = today
+    const payload = overview.data
+    void (async () => {
+      let modelCategories: Map<string, string> | undefined
+      try {
+        modelCategories = topCategoryByModel(await codeburn.getModels(period, 'all', false))
+      } catch { /* degrade: emit the snapshot without per-model topCategory */ }
+      trackEvent('usage_snapshot', usageSnapshotProps(payload, modelCategories))
+    })()
+  }, [overview.data, provider, customRange, claudeConfigSource, period, trackEvent])
 
   useEffect(() => {
     let saved: string | null = null