mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-10-21 05:23:19 +00:00
feat(cli): an anonymised daily telemetry snapshot inside the menubar payload
The desktop app and the Windows tray both send a once-a-day usage aggregate when the person has opted in. Each built its own, which meant two schemas to keep honest. The CLI now computes one snapshot at the end of the menubar payload and both apps send it verbatim. Everything is bucketed and name-only: cost and count buckets, up to eight models each with its top six task categories and their share of turns, twelve categories with their top models, providers, MCP servers, skills and tools by identifier with call buckets, a session count and median length bucket, and cache hit and retry tax shares to two decimals. Nothing derived from paths, project or branch names, or message text can enter it; the builder only emits primitives and capped containers. The raw model by task cross rides in through the options bag and never lands in the payload itself.
This commit is contained in:
parent
058bd57808
commit
e3dd333949
6 changed files with 432 additions and 1 deletions
|
|
@ -92,6 +92,7 @@ import { getShortModelName } from './models.js'
|
|||
import type { ReworkedFile } from './workflow-insights.js'
|
||||
import type { PrRow, BranchRow } from './sessions-report.js'
|
||||
import type { LiveSessionsBlock } from './live-sessions.js'
|
||||
import { buildTelemetrySnapshot, type TelemetrySnapshot, type TelemetrySnapshotInput } from './telemetry-snapshot.js'
|
||||
|
||||
const TOP_ACTIVITIES_LIMIT = 20
|
||||
const TOP_MODELS_LIMIT = 20
|
||||
|
|
@ -387,6 +388,12 @@ export type MenubarPayload = {
|
|||
currency: { code: string; symbol: string; rate: number }
|
||||
combined?: CombinedUsage
|
||||
claudeConfigs?: ClaudeConfigSelector
|
||||
/// Anonymised, fully bucketed daily aggregate for consent-gated product
|
||||
/// telemetry: the `usage_snapshot` event that the desktop app and the Windows
|
||||
/// tray both send verbatim, so the two surfaces cannot drift apart. Opaque to
|
||||
/// every other consumer. `null` when it could not be computed. See
|
||||
/// src/telemetry-snapshot.ts for the privacy contract it holds.
|
||||
telemetrySnapshot: TelemetrySnapshot | null
|
||||
}
|
||||
|
||||
function oneShotRateFor(editTurns: number, oneShotTurns: number): number | null {
|
||||
|
|
@ -553,6 +560,10 @@ export type BreakdownArrays = {
|
|||
/// menubar payload defaults to an empty savings block — keeping the
|
||||
/// schema stable for consumers that don't care about local savings.
|
||||
localModelSavings?: LocalModelSavings
|
||||
/// Inputs the telemetry snapshot needs that the payload itself does not
|
||||
/// carry: the per-(model, task category) turn cross and session wall-clock
|
||||
/// durations. The raw values never enter the payload, only their buckets.
|
||||
telemetry?: TelemetrySnapshotInput
|
||||
}
|
||||
|
||||
export function buildMenubarPayload(
|
||||
|
|
@ -613,7 +624,11 @@ export function buildMenubarPayload(
|
|||
const c = getCurrency()
|
||||
return { code: c.code, symbol: c.symbol, rate: c.rate }
|
||||
})(),
|
||||
telemetrySnapshot: null,
|
||||
}
|
||||
// Derived from the payload that was just assembled, so the snapshot can never
|
||||
// report something the payload does not already say.
|
||||
payload.telemetrySnapshot = buildTelemetrySnapshot(payload, breakdowns?.telemetry)
|
||||
if (claudeConfigs && claudeConfigs.options.length > 1) {
|
||||
payload.claudeConfigs = claudeConfigs
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1842,7 +1842,9 @@ const STATUS_SNAPSHOT_FILE = 'status-snapshot'
|
|||
// resolution time at which its corpus scan began. The latter is the ordering
|
||||
// fence for competing writers: max(file mtime) is not monotonic because
|
||||
// deleting the newest file legitimately makes it decrease.
|
||||
const STATUS_SNAPSHOT_VERSION = 3
|
||||
//
|
||||
// v4: the payload carries `telemetrySnapshot`, which a v3 record predates.
|
||||
const STATUS_SNAPSHOT_VERSION = 4
|
||||
|
||||
type StatusSnapshotRecord = {
|
||||
version: number
|
||||
|
|
|
|||
BIN
src/telemetry-snapshot.ts
Normal file
BIN
src/telemetry-snapshot.ts
Normal file
Binary file not shown.
|
|
@ -12,6 +12,7 @@ import { stat } from 'node:fs/promises'
|
|||
import { aggregateProjectsIntoDays, buildPeriodDataFromDays, dateKeyInTz } from './day-aggregator.js'
|
||||
import { aggregateModelEfficiency } from './model-efficiency.js'
|
||||
import { aggregateModels } from './models-report.js'
|
||||
import { aggregateModelTaskTurns, sessionDurationMinutes } from './telemetry-snapshot.js'
|
||||
import { scanUserCorrections, medianTimeToFirstEditMs, aggregateFileChurn, computePricingCoverage } from './workflow-insights.js'
|
||||
import { buildPrAttribution, aggregateByBranch } from './sessions-report.js'
|
||||
import { scanAndDetect } from './optimize.js'
|
||||
|
|
@ -1155,6 +1156,12 @@ export async function buildMenubarPayloadForRange(periodInfo: PeriodInfo, opts:
|
|||
})),
|
||||
]
|
||||
return {
|
||||
// Telemetry-only inputs: the model x task cross and session lengths, which
|
||||
// the payload itself does not carry. Bucketed inside the snapshot builder.
|
||||
telemetry: {
|
||||
modelTasks: aggregateModelTaskTurns(scanProjects),
|
||||
sessionMinutes: sessionDurationMinutes(scanProjects),
|
||||
},
|
||||
tools: Object.entries(toolMap).sort(([, a], [, b]) => b - a).slice(0, 10).map(([name, calls]) => ({ name, calls })),
|
||||
skills: Object.entries(skillMap).sort(([, a], [, b]) => b.cost - a.cost).slice(0, 10).map(([name, d]) => ({ name, ...d })),
|
||||
subagents: subagents.sort((a, b) => b.cost - a.cost || (b.totalTokens ?? 0) - (a.totalTokens ?? 0)),
|
||||
|
|
|
|||
|
|
@ -124,6 +124,19 @@ describe('codeburn status --format menubar-json', () => {
|
|||
|
||||
const history = payload['history'] as { daily: unknown[] }
|
||||
expect(Array.isArray(history.daily)).toBe(true)
|
||||
|
||||
// The consent-gated telemetry aggregate the desktop app and the tray both
|
||||
// send verbatim. Bucketed and name-only, so it must never echo a cost.
|
||||
const snapshot = payload['telemetrySnapshot'] as Record<string, unknown>
|
||||
expect(snapshot).toBeTruthy()
|
||||
expect(snapshot['schema']).toBe(2)
|
||||
// The same period label the payload shows, date suffix and all: the event
|
||||
// it rides on is already stamped with the calendar day.
|
||||
expect(snapshot['period']).toBe(current['label'])
|
||||
expect(snapshot['costBucket']).toBe('<1')
|
||||
expect((snapshot['models'] as Array<{ name: string; tasks: unknown[] }>)[0]!.tasks.length).toBeGreaterThan(0)
|
||||
expect(JSON.stringify(snapshot)).not.toContain('myapp')
|
||||
expect(JSON.stringify(snapshot)).not.toContain(String(current['cost']))
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true })
|
||||
}
|
||||
|
|
|
|||
394
tests/telemetry-snapshot.test.ts
Normal file
394
tests/telemetry-snapshot.test.ts
Normal file
|
|
@ -0,0 +1,394 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { sanitizeProps } from '../app/electron/telemetry.js'
|
||||
import { buildMenubarPayload, type MenubarPayload, type PeriodData } from '../src/menubar-json.js'
|
||||
import {
|
||||
aggregateModelTaskTurns,
|
||||
buildTelemetrySnapshot,
|
||||
costBucket,
|
||||
countBucket,
|
||||
minutesBucket,
|
||||
sessionDurationMinutes,
|
||||
TELEMETRY_SNAPSHOT_SCHEMA,
|
||||
type ModelTaskTurns,
|
||||
} from '../src/telemetry-snapshot.js'
|
||||
import type { ClassifiedTurn, ParsedApiCall, ProjectSummary, SessionSummary, TaskCategory } from '../src/types.js'
|
||||
|
||||
function call(model: string, costUSD = 1): ParsedApiCall {
|
||||
return {
|
||||
provider: 'claude',
|
||||
model,
|
||||
usage: {
|
||||
inputTokens: 100,
|
||||
outputTokens: 50,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
},
|
||||
costUSD,
|
||||
tools: ['Edit'],
|
||||
mcpTools: [],
|
||||
skills: [],
|
||||
hasAgentSpawn: false,
|
||||
hasPlanMode: false,
|
||||
speed: 'standard',
|
||||
timestamp: '2026-05-05T00:00:00Z',
|
||||
bashCommands: [],
|
||||
deduplicationKey: `${model}-${costUSD}-${Math.random()}`,
|
||||
}
|
||||
}
|
||||
|
||||
function turn(
|
||||
model: string,
|
||||
category: TaskCategory,
|
||||
opts: { hasEdits?: boolean; retries?: number; calls?: ParsedApiCall[] } = {},
|
||||
): ClassifiedTurn {
|
||||
return {
|
||||
userMessage: 'refactor the parser in src/secret-project/parser.ts',
|
||||
assistantCalls: opts.calls ?? [call(model)],
|
||||
timestamp: '2026-05-05T00:00:00Z',
|
||||
sessionId: 's1',
|
||||
category,
|
||||
retries: opts.retries ?? 0,
|
||||
hasEdits: opts.hasEdits ?? true,
|
||||
}
|
||||
}
|
||||
|
||||
function project(turns: ClassifiedTurn[], span: { first: string; last: string } = { first: '2026-05-05T00:00:00Z', last: '2026-05-05T00:30:00Z' }): ProjectSummary {
|
||||
const session = {
|
||||
sessionId: 's1',
|
||||
project: 'app',
|
||||
firstTimestamp: span.first,
|
||||
lastTimestamp: span.last,
|
||||
totalCostUSD: 0,
|
||||
totalInputTokens: 0,
|
||||
totalOutputTokens: 0,
|
||||
totalCacheReadTokens: 0,
|
||||
totalCacheWriteTokens: 0,
|
||||
apiCalls: turns.length,
|
||||
turns,
|
||||
modelBreakdown: {},
|
||||
toolBreakdown: {},
|
||||
mcpBreakdown: {},
|
||||
bashBreakdown: {},
|
||||
categoryBreakdown: {} as SessionSummary['categoryBreakdown'],
|
||||
skillBreakdown: {},
|
||||
} as unknown as SessionSummary
|
||||
return { project: 'app', projectPath: '/app', sessions: [session], totalCostUSD: 0, totalApiCalls: turns.length }
|
||||
}
|
||||
|
||||
/// A payload with something in every block the snapshot reads, plus identifying
|
||||
/// strings in the blocks it must never read.
|
||||
function richPayload(over: Partial<PeriodData> = {}, breakdowns: Parameters<typeof buildMenubarPayload>[6] = {}): MenubarPayload {
|
||||
const period = {
|
||||
label: '30 Days',
|
||||
cost: 312.5,
|
||||
savingsUSD: 0,
|
||||
calls: 4200,
|
||||
sessions: 140,
|
||||
inputTokens: 1000,
|
||||
outputTokens: 500,
|
||||
cacheReadTokens: 3000,
|
||||
cacheWriteTokens: 200,
|
||||
categories: [
|
||||
{ name: 'Coding', cost: 200, savingsUSD: 0, turns: 900, editTurns: 400, oneShotTurns: 300 },
|
||||
{ name: 'Debugging', cost: 80, savingsUSD: 0, turns: 300, editTurns: 100, oneShotTurns: 40 },
|
||||
],
|
||||
models: [
|
||||
{ name: 'claude-sonnet-4-5', cost: 200, savingsUSD: 0, calls: 3000 },
|
||||
{ name: 'claude-opus-4-6', cost: 112.5, savingsUSD: 0, calls: 1200 },
|
||||
],
|
||||
projects: [{ name: 'acme-secret-service', cost: 300, savingsUSD: 0, sessions: 40 }],
|
||||
topSessions: [{ project: 'acme-secret-service', cost: 30, savingsUSD: 0, calls: 12, date: '2026-05-05' }],
|
||||
topReworkedFiles: [{ path: 'billing-secrets.ts', sessions: 3, edits: 12 }],
|
||||
byBranch: [{ branch: 'feat/unreleased-thing', cost: 12, calls: 3, sessions: 1 }],
|
||||
...over,
|
||||
} as PeriodData
|
||||
return buildMenubarPayload(
|
||||
period,
|
||||
[
|
||||
{ name: 'claude', displayName: 'Claude', cost: 300, calls: 4000, hasUsage: true },
|
||||
{ name: 'codex', displayName: 'Codex', cost: 12.5, calls: 200, hasUsage: true },
|
||||
],
|
||||
null,
|
||||
undefined,
|
||||
{ totalUSD: 31.25, retries: 40, editTurns: 500, byModel: [] },
|
||||
undefined,
|
||||
{
|
||||
mcpServers: [{ name: 'context7', calls: 40 }],
|
||||
skills: [{ name: 'code-review', turns: 12, cost: 3 }],
|
||||
tools: [{ name: 'Edit', calls: 900 }],
|
||||
...breakdowns,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
describe('telemetry snapshot buckets', () => {
|
||||
it('coarsens dollars at the same boundaries the desktop uses', () => {
|
||||
expect(costBucket(0)).toBe('<1')
|
||||
expect(costBucket(0.99)).toBe('<1')
|
||||
expect(costBucket(1)).toBe('1-10')
|
||||
expect(costBucket(9.99)).toBe('1-10')
|
||||
expect(costBucket(10)).toBe('10-50')
|
||||
expect(costBucket(50)).toBe('50-200')
|
||||
expect(costBucket(200)).toBe('200-1k')
|
||||
expect(costBucket(1000)).toBe('1k+')
|
||||
expect(costBucket(Number.NaN)).toBe('<1')
|
||||
})
|
||||
|
||||
it('coarsens counts and durations', () => {
|
||||
expect(countBucket(0)).toBe('1-10')
|
||||
expect(countBucket(9)).toBe('1-10')
|
||||
expect(countBucket(10)).toBe('10-100')
|
||||
expect(countBucket(100)).toBe('100-1k')
|
||||
expect(countBucket(1000)).toBe('1k+')
|
||||
expect(minutesBucket(4.9)).toBe('<5')
|
||||
expect(minutesBucket(5)).toBe('5-15')
|
||||
expect(minutesBucket(15)).toBe('15-60')
|
||||
expect(minutesBucket(60)).toBe('60-240')
|
||||
expect(minutesBucket(240)).toBe('240+')
|
||||
})
|
||||
})
|
||||
|
||||
describe('aggregateModelTaskTurns', () => {
|
||||
it('crosses each turn primary model with its task category', () => {
|
||||
const rows = aggregateModelTaskTurns([project([
|
||||
turn('claude-sonnet-4-5', 'coding', { hasEdits: true, retries: 0 }),
|
||||
turn('claude-sonnet-4-5', 'coding', { hasEdits: true, retries: 2 }),
|
||||
turn('claude-sonnet-4-5', 'debugging', { hasEdits: false }),
|
||||
turn('claude-opus-4-6', 'planning', { hasEdits: false }),
|
||||
])])
|
||||
|
||||
expect(rows).toContainEqual({ model: 'Sonnet 4.5', category: 'Coding', turns: 2, editTurns: 2, oneShotTurns: 1 })
|
||||
expect(rows).toContainEqual({ model: 'Sonnet 4.5', category: 'Debugging', turns: 1, editTurns: 0, oneShotTurns: 0 })
|
||||
expect(rows.find(row => row.category === 'Planning')?.turns).toBe(1)
|
||||
})
|
||||
|
||||
it('attributes a mixed turn to its first non-synthetic model', () => {
|
||||
const rows = aggregateModelTaskTurns([project([
|
||||
turn('x', 'coding', { calls: [call('<synthetic>'), call('claude-opus-4-6')] }),
|
||||
])])
|
||||
|
||||
expect(rows).toHaveLength(1)
|
||||
expect(rows[0]!.model).not.toBe('<synthetic>')
|
||||
})
|
||||
})
|
||||
|
||||
describe('sessionDurationMinutes', () => {
|
||||
it('measures wall-clock length and skips unusable timestamps', () => {
|
||||
const good = project([turn('claude-sonnet-4-5', 'coding')], { first: '2026-05-05T00:00:00Z', last: '2026-05-05T00:45:00Z' })
|
||||
const backwards = project([turn('claude-sonnet-4-5', 'coding')], { first: '2026-05-05T01:00:00Z', last: '2026-05-05T00:00:00Z' })
|
||||
const unparseable = project([turn('claude-sonnet-4-5', 'coding')], { first: 'not-a-date', last: 'nor-this' })
|
||||
|
||||
expect(sessionDurationMinutes([good])).toEqual([45])
|
||||
expect(sessionDurationMinutes([backwards, unparseable])).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('buildTelemetrySnapshot', () => {
|
||||
const modelTasks: ModelTaskTurns[] = [
|
||||
{ model: 'Sonnet 4.5', category: 'Coding', turns: 600, editTurns: 300, oneShotTurns: 210 },
|
||||
{ model: 'Sonnet 4.5', category: 'Debugging', turns: 200, editTurns: 100, oneShotTurns: 40 },
|
||||
{ model: 'Opus 4.6', category: 'Coding', turns: 100, editTurns: 50, oneShotTurns: 25 },
|
||||
]
|
||||
|
||||
it('reports the schema, period and bucketed totals', () => {
|
||||
const snapshot = buildTelemetrySnapshot(richPayload())
|
||||
|
||||
expect(snapshot.schema).toBe(TELEMETRY_SNAPSHOT_SCHEMA)
|
||||
expect(snapshot.schema).toBe(2)
|
||||
expect(snapshot.period).toBe('30 Days')
|
||||
expect(snapshot.providerCount).toBe(2)
|
||||
expect(snapshot.costBucket).toBe('200-1k')
|
||||
expect(snapshot.sessions.countBucket).toBe('100-1k')
|
||||
expect(snapshot.providers).toEqual([
|
||||
{ name: 'claude', costBucket: '200-1k' },
|
||||
{ name: 'codex', costBucket: '10-50' },
|
||||
])
|
||||
})
|
||||
|
||||
it('crosses models with their task categories and shares', () => {
|
||||
const payload = richPayload()
|
||||
payload.current.topModels = [
|
||||
{ name: 'Sonnet 4.5', cost: 200, savingsUSD: 0, savingsBaselineModel: '', calls: 3000 },
|
||||
{ name: 'Opus 4.6', cost: 112.5, savingsUSD: 0, savingsBaselineModel: '', calls: 1200 },
|
||||
]
|
||||
const snapshot = buildTelemetrySnapshot(payload, { modelTasks })
|
||||
|
||||
expect(snapshot.models[0]).toEqual({
|
||||
name: 'Sonnet 4.5',
|
||||
costBucket: '200-1k',
|
||||
turnBucket: '100-1k',
|
||||
oneShotRate: 0.63,
|
||||
tasks: [
|
||||
{ name: 'Coding', turnBucket: '100-1k', share: 0.75 },
|
||||
{ name: 'Debugging', turnBucket: '100-1k', share: 0.25 },
|
||||
],
|
||||
})
|
||||
expect(snapshot.models[1]!.tasks).toEqual([{ name: 'Coding', turnBucket: '100-1k', share: 1 }])
|
||||
})
|
||||
|
||||
it('names the top models per category and reports one-shot rates to 2dp', () => {
|
||||
const payload = richPayload()
|
||||
payload.current.topModels = [
|
||||
{ name: 'Sonnet 4.5', cost: 200, savingsUSD: 0, savingsBaselineModel: '', calls: 3000 },
|
||||
{ name: 'Opus 4.6', cost: 112.5, savingsUSD: 0, savingsBaselineModel: '', calls: 1200 },
|
||||
]
|
||||
const snapshot = buildTelemetrySnapshot(payload, { modelTasks })
|
||||
|
||||
expect(snapshot.categories[0]).toEqual({
|
||||
name: 'Coding',
|
||||
turnBucket: '100-1k',
|
||||
oneShotRate: 0.75,
|
||||
topModels: ['Sonnet 4.5', 'Opus 4.6'],
|
||||
})
|
||||
expect(snapshot.categories[1]!.topModels).toEqual(['Sonnet 4.5'])
|
||||
})
|
||||
|
||||
it('marks a rate it cannot compute as -1 rather than zero', () => {
|
||||
const payload = richPayload({
|
||||
categories: [{ name: 'Conversation', cost: 1, savingsUSD: 0, turns: 5, editTurns: 0, oneShotTurns: 0 }],
|
||||
})
|
||||
const snapshot = buildTelemetrySnapshot(payload)
|
||||
|
||||
expect(snapshot.categories[0]!.oneShotRate).toBe(-1)
|
||||
expect(snapshot.models.every(model => model.oneShotRate === -1)).toBe(true)
|
||||
})
|
||||
|
||||
it('buckets the median session length only when durations are known', () => {
|
||||
const payload = richPayload()
|
||||
|
||||
expect(buildTelemetrySnapshot(payload).sessions.medianMinutesBucket).toBeUndefined()
|
||||
expect(buildTelemetrySnapshot(payload, { sessionMinutes: [2, 20, 400] }).sessions.medianMinutesBucket).toBe('15-60')
|
||||
})
|
||||
|
||||
it('reports cache hit and retry tax as shares, not dollars', () => {
|
||||
const snapshot = buildTelemetrySnapshot(richPayload())
|
||||
|
||||
expect(snapshot.efficiency.cacheHitRate).toBe(0.75)
|
||||
expect(snapshot.efficiency.retryTaxShare).toBe(0.1)
|
||||
})
|
||||
|
||||
it('omits efficiency figures that have no denominator', () => {
|
||||
const snapshot = buildTelemetrySnapshot(richPayload({
|
||||
cost: 0,
|
||||
inputTokens: 0,
|
||||
cacheReadTokens: 0,
|
||||
}))
|
||||
|
||||
expect(snapshot.efficiency).toEqual({})
|
||||
})
|
||||
|
||||
it('caps every array', () => {
|
||||
const many = (n: number, make: (i: number) => unknown) => Array.from({ length: n }, (_, i) => make(i))
|
||||
const payload = richPayload({
|
||||
categories: many(20, i => ({ name: `Cat ${i}`, cost: 20 - i, savingsUSD: 0, turns: 10, editTurns: 5, oneShotTurns: 1 })) as PeriodData['categories'],
|
||||
}, {
|
||||
mcpServers: many(20, i => ({ name: `server-${i}`, calls: 5 })) as { name: string; calls: number }[],
|
||||
skills: many(20, i => ({ name: `skill-${i}`, turns: 5, cost: 1 })) as { name: string; turns: number; cost: number }[],
|
||||
tools: many(20, i => ({ name: `tool-${i}`, calls: 5 })) as { name: string; calls: number }[],
|
||||
})
|
||||
payload.current.topModels = many(20, i => ({ name: `model-${i}`, cost: 20 - i, savingsUSD: 0, savingsBaselineModel: '', calls: 10 })) as MenubarPayload['current']['topModels']
|
||||
payload.current.providers = Object.fromEntries(many(20, i => [`provider-${i}`, 20 - i]) as [string, number][])
|
||||
const crowdedModel: ModelTaskTurns[] = many(10, i => ({ model: 'model-0', category: `Cat ${i}`, turns: 10 - i, editTurns: 1, oneShotTurns: 1 })) as ModelTaskTurns[]
|
||||
|
||||
const snapshot = buildTelemetrySnapshot(payload, { modelTasks: crowdedModel })
|
||||
|
||||
expect(snapshot.models).toHaveLength(8)
|
||||
expect(snapshot.models[0]!.tasks).toHaveLength(6)
|
||||
expect(snapshot.categories).toHaveLength(12)
|
||||
expect(snapshot.providers).toHaveLength(8)
|
||||
expect(snapshot.mcpServers).toHaveLength(12)
|
||||
expect(snapshot.skills).toHaveLength(12)
|
||||
expect(snapshot.tools).toHaveLength(12)
|
||||
})
|
||||
|
||||
it('caps the models named under one category at three', () => {
|
||||
const payload = richPayload()
|
||||
payload.current.topModels = Array.from({ length: 5 }, (_, i) => ({ name: `m${i}`, cost: 5 - i, savingsUSD: 0, savingsBaselineModel: '', calls: 1 }))
|
||||
const snapshot = buildTelemetrySnapshot(payload, {
|
||||
modelTasks: Array.from({ length: 5 }, (_, i) => ({ model: `m${i}`, category: 'Coding', turns: 5 - i, editTurns: 1, oneShotTurns: 1 })),
|
||||
})
|
||||
|
||||
expect(snapshot.categories[0]!.topModels).toEqual(['m0', 'm1', 'm2'])
|
||||
})
|
||||
|
||||
it('carries no path, project, branch, file or prompt text', () => {
|
||||
const payload = richPayload()
|
||||
const serialized = JSON.stringify(buildTelemetrySnapshot(payload, {
|
||||
modelTasks,
|
||||
sessionMinutes: [30],
|
||||
}))
|
||||
|
||||
for (const forbidden of ['acme-secret-service', 'billing-secrets.ts', 'feat/unreleased-thing', '/app', 'secret-project']) {
|
||||
expect(serialized).not.toContain(forbidden)
|
||||
}
|
||||
// And no exact magnitudes: the raw cost, session count and retry tax dollars.
|
||||
for (const exact of ['312.5', '4200', '140', '31.25']) {
|
||||
expect(serialized).not.toContain(exact)
|
||||
}
|
||||
})
|
||||
|
||||
it('emits only strings, finite numbers and capped containers', () => {
|
||||
const snapshot = buildTelemetrySnapshot(richPayload(), { modelTasks, sessionMinutes: [30] })
|
||||
|
||||
const walk = (value: unknown, depth: number): void => {
|
||||
// props -> models[] -> model -> tasks[] -> task -> leaf is the deepest
|
||||
// shape the snapshot emits, and the sanitizer's own cap allows no more.
|
||||
expect(depth).toBeLessThanOrEqual(5)
|
||||
if (Array.isArray(value)) {
|
||||
for (const entry of value) walk(entry, depth + 1)
|
||||
return
|
||||
}
|
||||
if (value !== null && typeof value === 'object') {
|
||||
for (const entry of Object.values(value)) walk(entry, depth + 1)
|
||||
return
|
||||
}
|
||||
expect(['string', 'number', 'boolean']).toContain(typeof value)
|
||||
if (typeof value === 'number') expect(Number.isFinite(value)).toBe(true)
|
||||
if (typeof value === 'string') expect(value.length).toBeLessThanOrEqual(64)
|
||||
}
|
||||
walk(snapshot, 0)
|
||||
})
|
||||
|
||||
it('survives the desktop whitelist sanitizer unchanged', () => {
|
||||
const snapshot = buildTelemetrySnapshot(richPayload(), { modelTasks, sessionMinutes: [30] })
|
||||
|
||||
expect(sanitizeProps(snapshot)).toEqual(JSON.parse(JSON.stringify(snapshot)))
|
||||
})
|
||||
|
||||
it('truncates a name longer than the sanitizer would keep', () => {
|
||||
const payload = richPayload()
|
||||
payload.current.topModels = [{ name: 'm'.repeat(200), cost: 1, savingsUSD: 0, savingsBaselineModel: '', calls: 1 }]
|
||||
const snapshot = buildTelemetrySnapshot(payload)
|
||||
|
||||
expect(snapshot.models[0]!.name).toHaveLength(64)
|
||||
})
|
||||
})
|
||||
|
||||
describe('buildMenubarPayload: telemetrySnapshot', () => {
|
||||
it('always carries a snapshot built from the payload it just assembled', () => {
|
||||
const payload = richPayload()
|
||||
|
||||
expect(payload.telemetrySnapshot).not.toBeNull()
|
||||
expect(payload.telemetrySnapshot!.schema).toBe(2)
|
||||
expect(payload.telemetrySnapshot!.period).toBe(payload.current.label)
|
||||
expect(payload.telemetrySnapshot!.costBucket).toBe('200-1k')
|
||||
})
|
||||
|
||||
it('folds the telemetry-only breakdown inputs into the snapshot', () => {
|
||||
const payload = richPayload({}, {
|
||||
telemetry: {
|
||||
modelTasks: [{ model: 'Sonnet 4.5', category: 'Coding', turns: 600, editTurns: 300, oneShotTurns: 210 }],
|
||||
sessionMinutes: [30],
|
||||
},
|
||||
})
|
||||
|
||||
expect(payload.telemetrySnapshot!.sessions.medianMinutesBucket).toBe('15-60')
|
||||
// The cross reached the snapshot, and the raw counts stayed out of the payload.
|
||||
expect(payload.telemetrySnapshot!.categories[0]!.topModels).toEqual(['Sonnet 4.5'])
|
||||
expect(JSON.stringify(payload.current)).not.toContain('oneShotTurns')
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue