diff --git a/dash/src/App.tsx b/dash/src/App.tsx index 92fe1ae..6b1e5a7 100644 --- a/dash/src/App.tsx +++ b/dash/src/App.tsx @@ -83,9 +83,6 @@ function DeviceView({ payload, isRemote, unit }: { payload?: Payload; isRemote: const toolBars: BarItem[] = c ? Object.entries(c.providers).filter(([, v]) => v > 0).sort((a, b) => b[1] - a[1]).map(([k, v]) => ({ name: k, value: v, display: usd(v) })) : [] - const modelBars: BarItem[] = c - ? c.topModels.filter((m) => m.cost > 0).slice(0, 8).map((m) => ({ name: m.name, value: m.cost, display: usd(m.cost) })) - : [] const activityBars: BarItem[] = c ? c.topActivities.filter((a) => a.cost > 0).map((a) => ({ name: a.name, value: a.cost, display: usd(a.cost) })) : [] @@ -136,7 +133,37 @@ function DeviceView({ payload, isRemote, unit }: { payload?: Payload; isRemote: - + m.cost > 0).slice(0, 8).map((m) => ({ + name: m.name, + cost: usd(m.cost), + calls: fmtNum(m.calls), + savings: usd(m.savingsUSD), + }))} + /> + + + +
+ + ({ + name: m.name, + costPerEdit: usd(m.costPerEdit), + oneShot: `${Math.round(m.oneShotRate * 100)}%`, + }))} + />
@@ -152,11 +179,13 @@ function DeviceView({ payload, isRemote, unit }: { payload?: Payload; isRemote: { key: 'name', label: 'Project' }, { key: 'cost', label: 'Cost', num: true }, { key: 'sessions', label: 'Sessions', num: true }, + { key: 'avgCost', label: 'Avg/session', num: true }, ]} rows={(c?.topProjects ?? []).slice(0, 10).map((p) => ({ name: p.name, cost: usd(p.cost), sessions: fmtNum(p.sessions), + avgCost: usd(p.avgCostPerSession), }))} /> )} diff --git a/src/menubar-json.ts b/src/menubar-json.ts index c59bc17..e696572 100644 --- a/src/menubar-json.ts +++ b/src/menubar-json.ts @@ -32,6 +32,16 @@ export type PeriodData = { projects?: Array<{ name: string; cost: number; savingsUSD: number; sessions: number; sessionDetails?: Array<{ cost: number; savingsUSD: number; calls: number; inputTokens: number; outputTokens: number; date: string; models: Array<{ name: string; cost: number; savingsUSD: number }> }> }> modelEfficiency?: Array<{ name: string; costPerEdit: number | null; oneShotRate: number | null }> topSessions?: Array<{ project: string; cost: number; savingsUSD: number; calls: number; date: string }> + /// Workflow-intelligence rollups (issue: workflow intelligence). Optional so + /// the day-aggregator PeriodData path (which has no per-turn data) can omit + /// them; the fresh-parse payload path always sets them. + workflow?: { corrections: number; correctionRate: number | null; medianTimeToFirstEditMs: number | null } + /// Files most reworked by edit-family calls, relative to project root, ranked + /// by distinct sessions then edits. Full (top 15) list; the payload basenames + /// and trims it. + topReworkedFiles?: ReworkedFile[] + /// Share (0-1) of cost-bearing calls that resolved a price. + pricingCoverage?: number } export type ProviderCost = { @@ -44,6 +54,7 @@ export type ProviderCost = { import type { OptimizeResult } from './optimize.js' import { getCurrency } from './currency.js' import type { GranularHistory } from './granular-history.js' +import type { ReworkedFile } from './workflow-insights.js' const TOP_ACTIVITIES_LIMIT = 20 const TOP_MODELS_LIMIT = 20 @@ -53,6 +64,7 @@ const SYNTHETIC_MODEL_NAME = '' const TOP_PROJECTS_LIMIT = 5 const TOP_SESSIONS_LIMIT = 3 const MODEL_EFFICIENCY_LIMIT = 5 +const TOP_REWORKED_FILES_LIMIT = 8 export type DailyModelBreakdown = { name: string @@ -215,6 +227,21 @@ export type MenubarPayload = { calls: number date: string }> + /// Workflow-intelligence rollup for the period. `unansweredSessions` is + /// add-only and optional: sessions that ended with no assistant reply are + /// not computable from the session cache (the parser drops a trailing + /// unanswered user turn), so it stays unset until a parse-time capture lands. + workflow: { + corrections: number + correctionRate: number | null + medianTimeToFirstEditMs: number | null + unansweredSessions?: number + } + /// Files most reworked by edit-family calls (top 8). Path is basename-only + /// for privacy; distinct sessions and total edit calls per file. + topReworkedFiles: Array<{ path: string; sessions: number; edits: number }> + /// Share (0-1) of cost-bearing calls that resolved a price. + pricingCoverage: number retryTax: { totalUSD: number retries: number @@ -377,6 +404,22 @@ function buildModelEfficiency(models: PeriodData['modelEfficiency']): MenubarPay .map(m => ({ name: m.name, costPerEdit: m.costPerEdit, oneShotRate: m.oneShotRate })) } +function buildWorkflow(workflow: PeriodData['workflow']): MenubarPayload['current']['workflow'] { + return { + corrections: workflow?.corrections ?? 0, + correctionRate: workflow?.correctionRate ?? null, + medianTimeToFirstEditMs: workflow?.medianTimeToFirstEditMs ?? null, + } +} + +function buildTopReworkedFiles(files: PeriodData['topReworkedFiles']): MenubarPayload['current']['topReworkedFiles'] { + return (files ?? []) + .slice(0, TOP_REWORKED_FILES_LIMIT) + // Basename only: the menubar/web payload can leave the machine, so drop the + // directory path and keep just the file name. + .map(f => ({ path: f.path.split('/').pop() || f.path, sessions: f.sessions, edits: f.edits })) +} + function buildTopSessions(sessions: PeriodData['topSessions']): MenubarPayload['current']['topSessions'] { return (sessions ?? []) .sort((a, b) => (b.cost + b.savingsUSD) - (a.cost + a.savingsUSD)) @@ -432,6 +475,9 @@ export function buildMenubarPayload( topProjects: buildTopProjects(current.projects ?? []), modelEfficiency: buildModelEfficiency(current.modelEfficiency ?? []), topSessions: buildTopSessions(current.topSessions ?? []), + workflow: buildWorkflow(current.workflow), + topReworkedFiles: buildTopReworkedFiles(current.topReworkedFiles), + pricingCoverage: current.pricingCoverage ?? 1, retryTax: retryTax ?? { totalUSD: 0, retries: 0, editTurns: 0, byModel: [] }, routingWaste: routingWaste ?? { totalSavingsUSD: 0, baselineModel: '', baselineCostPerEdit: 0, byModel: [] }, tools: breakdowns?.tools ?? [], diff --git a/src/optimize.ts b/src/optimize.ts index 7d281bd..7900998 100644 --- a/src/optimize.ts +++ b/src/optimize.ts @@ -11,6 +11,7 @@ import type { DateRange, ProjectSummary, SessionSummary } from './types.js' import { formatCost } from './currency.js' import { formatTokens } from './format.js' import { recommendModelDefault, type ModelDefaultRecommendation } from './act/model-defaults.js' +import { aggregateFileChurn, buildCoachingNotes, scanUserCorrections, medianTimeToFirstEditMs, worstOneShotCategory, type ReworkedFile } from './workflow-insights.js' // ============================================================================ // Display constants @@ -339,6 +340,10 @@ export type OptimizeJsonReport = { estimatedSavingsUSD: number fix: WasteAction }> + /// Files most reworked by edit-family calls, relative to project root (top 15). + topReworkedFiles: ReworkedFile[] + /// 1-3 templated one-liners keyed on the strongest workflow signals. + coachingNotes: string[] modelRecommendations?: Array } @@ -3130,6 +3135,28 @@ function renderFinding(n: number, f: WasteFinding, costRate: number): string[] { return lines } +function renderWorkflowSection(reworkedFiles: ReworkedFile[], coachingNotes: string[]): string[] { + const lines: string[] = [] + if (reworkedFiles.length > 0) { + lines.push(chalk.bold.hex(ORANGE)(' Top reworked files')) + lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH))) + for (const f of reworkedFiles) { + const meta = chalk.dim(`${f.sessions} session${f.sessions === 1 ? '' : 's'}, ${f.edits} edit${f.edits === 1 ? '' : 's'}`) + lines.push(` ${f.path} ${meta}`) + } + lines.push('') + } + if (coachingNotes.length > 0) { + lines.push(chalk.bold.hex(ORANGE)(' Coaching notes')) + lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH))) + for (const note of coachingNotes) { + lines.push(wrap('- ' + note, PANEL_WIDTH, ' ')) + } + lines.push('') + } + return lines +} + function renderOptimize( findings: WasteFinding[], costRate: number, @@ -3139,6 +3166,8 @@ function renderOptimize( callCount: number, healthScore: number, healthGrade: HealthGrade, + reworkedFiles: ReworkedFile[], + coachingNotes: string[], appliedHeader?: string, previouslyApplied?: Record, modelRecommendations?: ModelDefaultRecommendation[], @@ -3165,6 +3194,7 @@ function renderOptimize( lines.push(chalk.dim(' token waste: junk directory reads, duplicate file reads, unused')) lines.push(chalk.dim(' agents/skills/MCP servers, bloated CLAUDE.md, and more.')) lines.push('') + lines.push(...renderWorkflowSection(reworkedFiles, coachingNotes)) return lines.join('\n') } @@ -3187,7 +3217,9 @@ function renderOptimize( lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH))) lines.push(chalk.dim(' Estimates only.')) lines.push('') - + + lines.push(...renderWorkflowSection(reworkedFiles, coachingNotes)) + if (modelRecommendations && modelRecommendations.length > 0) { lines.push(chalk.bold.hex(ORANGE)(' Model defaults recommendation')) lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH))) @@ -3203,6 +3235,22 @@ function renderOptimize( return lines.join('\n') } +// File churn + coaching notes for the workflow-intelligence section. Both are +// derived from the same parsed projects the detectors already ran on, so this +// adds no extra file I/O. +function buildWorkflowReport(projects: ProjectSummary[]): { topReworkedFiles: ReworkedFile[]; coachingNotes: string[] } { + const topReworkedFiles = aggregateFileChurn(projects) + const corrections = scanUserCorrections(projects) + const coachingNotes = buildCoachingNotes({ + worstOneShot: worstOneShotCategory(projects), + corrections: corrections.corrections, + correctionRate: corrections.correctionRate, + topReworkedFile: topReworkedFiles[0] ?? null, + medianTimeToFirstEditMs: medianTimeToFirstEditMs(projects), + }) + return { topReworkedFiles, coachingNotes } +} + export async function runOptimize( projects: ProjectSummary[], periodLabel: string, @@ -3230,7 +3278,8 @@ export async function runOptimize( return } - const output = renderOptimize(findings, costRate, periodLabel, periodCost, sessions.length, callCount, healthScore, healthGrade, opts.appliedHeader, opts.previouslyApplied, result.modelRecommendations) + const { topReworkedFiles, coachingNotes } = buildWorkflowReport(projects) + const output = renderOptimize(findings, costRate, periodLabel, periodCost, sessions.length, callCount, healthScore, healthGrade, topReworkedFiles, coachingNotes, opts.appliedHeader, opts.previouslyApplied, result.modelRecommendations) console.log(output) } @@ -3277,6 +3326,7 @@ export function buildOptimizeJsonReport( estimatedSavingsUSD: f.tokensSaved * result.costRate, fix: f.fix, })), + ...buildWorkflowReport(projects), modelRecommendations: result.modelRecommendations, } } diff --git a/src/usage-aggregator.ts b/src/usage-aggregator.ts index 13ad0f7..75fa6cc 100644 --- a/src/usage-aggregator.ts +++ b/src/usage-aggregator.ts @@ -9,6 +9,7 @@ import { stat } from 'node:fs/promises' import { aggregateProjectsIntoDays, buildPeriodDataFromDays } from './day-aggregator.js' import { aggregateModelEfficiency } from './model-efficiency.js' import { aggregateModels } from './models-report.js' +import { scanUserCorrections, medianTimeToFirstEditMs, aggregateFileChurn, computePricingCoverage } from './workflow-insights.js' import { scanAndDetect } from './optimize.js' import { getDaysInRange, ensureCacheHydrated, loadDailyCache, emptyCache, BACKFILL_DAYS, toDateString, type DailyCache, type DailyEntry } from './daily-cache.js' import { buildGranularHistory } from './granular-history.js' @@ -42,6 +43,13 @@ export function buildPeriodData(label: string, projects: ProjectSummary[]): Peri } } + const unpricedModels = findUnpricedModels(Object.entries(modelTotals) + .map(([model, d]) => ({ model, calls: d.calls, cost: d.cost, tokens: d.tokens }))) + const costBearingCalls = Object.entries(modelTotals) + .reduce((s, [model, d]) => s + (model === '' ? 0 : d.calls), 0) + const unpricedCalls = unpricedModels.reduce((s, m) => s + m.calls, 0) + const corrections = scanUserCorrections(projects) + return { label, cost: projects.reduce((s, p) => s + p.totalCostUSD, 0), @@ -56,8 +64,14 @@ export function buildPeriodData(label: string, projects: ProjectSummary[]): Peri models: Object.entries(modelTotals) .sort(([, a], [, b]) => b.cost - a.cost) .map(([name, d]) => ({ name, calls: d.calls, cost: d.cost, savingsUSD: d.savingsUSD, estimatedCostUSD: d.estimatedCostUSD })), - unpricedModels: findUnpricedModels(Object.entries(modelTotals) - .map(([model, d]) => ({ model, calls: d.calls, cost: d.cost, tokens: d.tokens }))), + unpricedModels, + workflow: { + corrections: corrections.corrections, + correctionRate: corrections.correctionRate, + medianTimeToFirstEditMs: medianTimeToFirstEditMs(projects), + }, + topReworkedFiles: aggregateFileChurn(projects), + pricingCoverage: computePricingCoverage(costBearingCalls, unpricedCalls), } } @@ -305,6 +319,14 @@ export async function buildMenubarPayloadForRange(periodInfo: PeriodInfo, opts: // detection, both derivable solely from surviving sessions. currentData.estimatedCostUSD = scanData.estimatedCostUSD currentData.unpricedModels = scanData.unpricedModels + // Workflow-intelligence metrics are session-derived by nature (they need + // turn text, timestamps, and tool sequences that day entries don't + // carry), so like the fields above they come from the scan. Note + // pricingCoverage therefore describes surviving-session calls, a smaller + // population than the carried headline cost. + currentData.workflow = scanData.workflow + currentData.topReworkedFiles = scanData.topReworkedFiles + currentData.pricingCoverage = scanData.pricingCoverage // Sessions: the cache buckets a session on its START day, the scan // counts it on any day it was ACTIVE. Each undercounts differently // (day-filtered views miss midnight-spanners; the scan misses expired diff --git a/src/workflow-insights.ts b/src/workflow-insights.ts new file mode 100644 index 0000000..370cec2 --- /dev/null +++ b/src/workflow-insights.ts @@ -0,0 +1,225 @@ +import { homedir } from 'os' + +import { EDIT_TOOLS } from './classifier.js' +import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory } from './types.js' + +// User-side mirror of compare-stats.ts scanSelfCorrections (which scans the +// assistant's own apologies). These match a *user* follow-up telling the +// assistant it got something wrong. Deliberately conservative: bare "wrong" or +// "undo" are excluded because they show up in ordinary task requests ("fix the +// wrong output", "undo the migration"). Every pattern requires a correction +// context so praise like "you were right" or "that's right" never trips it. +export const USER_CORRECTION_PATTERNS: RegExp[] = [ + /\bthat'?s (?:not|n'?t) (?:what|right|correct|it)\b/i, + /\bthat'?s (?:wrong|incorrect)\b/i, + /\bthat is (?:wrong|incorrect|not right)\b/i, + /\bnot what I (?:meant|wanted|asked|said)\b/i, + /\bno,? I (?:meant|wanted|said|asked for)\b/i, + /\byou (?:missed|forgot|misunderstood|broke)\b/i, + /\brevert (?:that|it|this|the|your)\b/i, + /\bundo (?:that|it|this|your|the last|the change)\b/i, + /\bwrong (?:file|approach|place|method|function|answer|way|direction)\b/i, + /\bstill (?:wrong|broken|failing|not working)\b/i, +] + +function matchesCorrection(text: string): boolean { + return USER_CORRECTION_PATTERNS.some(p => p.test(text)) +} + +export type UserCorrectionStats = { + corrections: number + /// Turns carrying a real user prompt (non-empty userMessage). The denominator + /// for correctionRate — continuation turns with no fresh prompt are excluded. + userTurns: number + correctionRate: number | null +} + +export function scanUserCorrections(projects: ProjectSummary[]): UserCorrectionStats { + let corrections = 0 + let userTurns = 0 + for (const project of projects) { + for (const session of project.sessions) { + for (const turn of session.turns) { + const msg = turn.userMessage + if (!msg || !msg.trim()) continue + userTurns++ + if (matchesCorrection(msg)) corrections++ + } + } + } + return { corrections, userTurns, correctionRate: userTurns > 0 ? corrections / userTurns : null } +} + +function callHasEditTools(tools: string[]): boolean { + return tools.some(t => EDIT_TOOLS.has(t)) +} + +/// Per-session ms from the session's first turn to the first assistant call +/// that used an edit-family tool. Returns null for sessions that never edited, +/// so those are excluded from the median rather than counted as zero. +export function sessionTimeToFirstEditMs(session: ProjectSummary['sessions'][number]): number | null { + const startMs = Date.parse(session.turns[0]?.timestamp ?? '') + if (Number.isNaN(startMs)) return null + for (const turn of session.turns) { + for (const call of turn.assistantCalls) { + if (!callHasEditTools(call.tools)) continue + const editMs = Date.parse(call.timestamp) + if (Number.isNaN(editMs)) continue + // Clamp: out-of-order timestamps across resumed transcripts would + // otherwise pull the median negative. + return Math.max(0, editMs - startMs) + } + } + return null +} + +export function medianTimeToFirstEditMs(projects: ProjectSummary[]): number | null { + const samples: number[] = [] + for (const project of projects) { + for (const session of project.sessions) { + const ms = sessionTimeToFirstEditMs(session) + if (ms !== null) samples.push(ms) + } + } + if (samples.length === 0) return null + samples.sort((a, b) => a - b) + const mid = Math.floor(samples.length / 2) + return samples.length % 2 === 0 ? (samples[mid - 1]! + samples[mid]!) / 2 : samples[mid]! +} + +export type ReworkedFile = { + path: string + sessions: number + edits: number +} + +function relativizePath(absPath: string, projectPath: string): string { + if (projectPath && (absPath === projectPath || absPath.startsWith(projectPath + '/'))) { + return absPath.slice(projectPath.length + 1) || absPath + } + const home = homedir() + if (absPath === home || absPath.startsWith(home + '/')) return '~' + absPath.slice(home.length) + return absPath +} + +/// Ranks the files touched most by edit-family tool calls. File paths come from +/// each call's toolSequence, which the parser/cache retain per edit tool_use +/// (see parser.ts and session-cache CachedCall.toolSequence). Rank by distinct +/// sessions first (a file reworked across many sessions is the real churn +/// signal), then total edit calls. +export function aggregateFileChurn(projects: ProjectSummary[], limit = 15): ReworkedFile[] { + type Acc = { path: string; sessions: Set; edits: number } + const byPath = new Map() + + for (const project of projects) { + for (const session of project.sessions) { + for (const turn of session.turns) { + for (const call of turn.assistantCalls) { + if (!call.toolSequence) continue + for (const step of call.toolSequence) { + for (const tc of step) { + if (!EDIT_TOOLS.has(tc.tool) || !tc.file) continue + let acc = byPath.get(tc.file) + if (!acc) { + acc = { path: relativizePath(tc.file, project.projectPath), sessions: new Set(), edits: 0 } + byPath.set(tc.file, acc) + } + acc.sessions.add(session.sessionId) + acc.edits++ + } + } + } + } + } + } + + return [...byPath.values()] + .map(a => ({ path: a.path, sessions: a.sessions.size, edits: a.edits })) + .sort((a, b) => b.sessions - a.sessions || b.edits - a.edits || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) + .slice(0, limit) +} + +/// Share (0-1) of cost-bearing calls that resolved a price. `unpricedCalls` are +/// the calls of models with usage but no pricing table entry (findUnpricedModels). +/// Returns 1 when there is nothing to price (no coverage gap to report). +export function computePricingCoverage(totalCostBearingCalls: number, unpricedCalls: number): number { + if (totalCostBearingCalls <= 0) return 1 + const priced = Math.max(0, totalCostBearingCalls - unpricedCalls) + return priced / totalCostBearingCalls +} + +export type CategoryOneShot = { category: string; rate: number; editTurns: number } + +/// The task category with the weakest one-shot rate (over enough edit turns to +/// trust), used by the coaching notes. Rate is a percentage (0-100), matching +/// model-efficiency and the report's category one-shot figures. +export function worstOneShotCategory(projects: ProjectSummary[], minEditTurns = 5): CategoryOneShot | null { + const acc = new Map() + for (const project of projects) { + for (const session of project.sessions) { + for (const [cat, d] of Object.entries(session.categoryBreakdown)) { + const e = acc.get(cat) ?? { editTurns: 0, oneShotTurns: 0 } + e.editTurns += d.editTurns + e.oneShotTurns += d.oneShotTurns + acc.set(cat, e) + } + } + } + let worst: CategoryOneShot | null = null + for (const [cat, d] of acc) { + if (d.editTurns < minEditTurns) continue + const rate = (d.oneShotTurns / d.editTurns) * 100 + if (!worst || rate < worst.rate) { + worst = { category: CATEGORY_LABELS[cat as TaskCategory] ?? cat, rate, editTurns: d.editTurns } + } + } + return worst +} + +// Coaching-note thresholds. Kept conservative so a note only fires on a signal +// strong enough to act on. +const ONE_SHOT_LOW_PERCENT = 60 +const CORRECTION_HIGH_RATE = 0.15 +const CORRECTION_MIN_COUNT = 3 +const CHURN_MIN_SESSIONS = 3 +const TTFE_SLOW_MS = 5 * 60 * 1000 + +export type CoachingInput = { + worstOneShot?: CategoryOneShot | null + corrections?: number + correctionRate?: number | null + topReworkedFile?: ReworkedFile | null + medianTimeToFirstEditMs?: number | null +} + +function formatDurationShort(ms: number): string { + if (ms >= 60_000) return `${Math.round(ms / 60_000)}m` + return `${Math.round(ms / 1000)}s` +} + +/// 1-3 templated one-liners keyed on the strongest workflow signals. Pure +/// templating on already-computed numbers. Copy is dry and specific and uses no +/// em-dashes (UI copy house style). +export function buildCoachingNotes(input: CoachingInput): string[] { + const notes: string[] = [] + + const ws = input.worstOneShot + if (ws && ws.editTurns >= 5 && ws.rate < ONE_SHOT_LOW_PERCENT) { + notes.push(`One-shot rate on ${ws.category} is ${Math.round(ws.rate)}% over ${ws.editTurns} edit turns. Add the constraints up front or split the work into smaller edits.`) + } + + if (input.correctionRate != null && input.correctionRate >= CORRECTION_HIGH_RATE && (input.corrections ?? 0) >= CORRECTION_MIN_COUNT) { + notes.push(`You corrected the assistant on ${Math.round(input.correctionRate * 100)}% of prompts (${input.corrections} times). State the requirements in the first message to cut the back and forth.`) + } + + const cf = input.topReworkedFile + if (cf && cf.sessions >= CHURN_MIN_SESSIONS) { + notes.push(`${cf.path} was reworked across ${cf.sessions} sessions (${cf.edits} edits). A focused pass on it may cost less than the repeated churn.`) + } + + if (input.medianTimeToFirstEditMs != null && input.medianTimeToFirstEditMs >= TTFE_SLOW_MS) { + notes.push(`Median time to first edit is ${formatDurationShort(input.medianTimeToFirstEditMs)}. Point the assistant at the target file to cut the exploration before it starts editing.`) + } + + return notes.slice(0, 3) +} diff --git a/tests/menubar-carried-headline.test.ts b/tests/menubar-carried-headline.test.ts index 4b8af48..93dc81d 100644 --- a/tests/menubar-carried-headline.test.ts +++ b/tests/menubar-carried-headline.test.ts @@ -103,6 +103,15 @@ describe('carried history reaches the user-visible headline', () => { expect(payload.current.topModels.some(m => m.name === 'Opus 4.8' && m.cost === 100)).toBe(true) const projects = payload.current.topProjects expect(projects.some(p => p.name === 'proj-x' && p.cost === 100 && p.sessions === 3)).toBe(true) + + // Scan-derived workflow-intelligence fields must be PROPAGATED onto the + // cache-authoritative headline, not silently dropped by the selective + // merge (the regression that made them dead on the default path). With no + // surviving sessions they are the empty-scan values, but they must exist. + expect(payload.current.workflow).toBeDefined() + expect(payload.current.workflow?.corrections).toBe(0) + expect(Array.isArray(payload.current.topReworkedFiles)).toBe(true) + expect(typeof payload.current.pricingCoverage).toBe('number') }) it('merges live today data on top of carried history without double counting', async () => { diff --git a/tests/workflow-insights.test.ts b/tests/workflow-insights.test.ts new file mode 100644 index 0000000..e566e69 --- /dev/null +++ b/tests/workflow-insights.test.ts @@ -0,0 +1,288 @@ +import { describe, expect, it } from 'vitest' + +import { + scanUserCorrections, + medianTimeToFirstEditMs, + aggregateFileChurn, + computePricingCoverage, + worstOneShotCategory, + buildCoachingNotes, + USER_CORRECTION_PATTERNS, +} from '../src/workflow-insights.js' +import type { ClassifiedTurn, ParsedApiCall, ProjectSummary, SessionSummary, TaskCategory, ToolCall } from '../src/types.js' + +function call(opts: { tools?: string[]; timestamp?: string; toolSequence?: ToolCall[][]; model?: string; costUSD?: number } = {}): ParsedApiCall { + return { + provider: 'claude', + model: opts.model ?? 'claude-sonnet-4-5', + usage: { + inputTokens: 0, outputTokens: 0, cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, cachedInputTokens: 0, reasoningTokens: 0, webSearchRequests: 0, + }, + costUSD: opts.costUSD ?? 0, + tools: opts.tools ?? [], + mcpTools: [], + skills: [], + subagentTypes: [], + hasAgentSpawn: false, + hasPlanMode: false, + speed: 'standard', + timestamp: opts.timestamp ?? '2026-05-05T00:00:00.000Z', + bashCommands: [], + deduplicationKey: 'k', + ...(opts.toolSequence ? { toolSequence: opts.toolSequence } : {}), + } +} + +function turn(opts: { userMessage?: string; calls?: ParsedApiCall[]; timestamp?: string; sessionId?: string } = {}): ClassifiedTurn { + return { + userMessage: opts.userMessage ?? '', + assistantCalls: opts.calls ?? [], + timestamp: opts.timestamp ?? '2026-05-05T00:00:00.000Z', + sessionId: opts.sessionId ?? 's1', + category: 'coding', + retries: 0, + hasEdits: (opts.calls ?? []).some(c => c.tools.some(t => t === 'Edit' || t === 'Write')), + } +} + +function session(sessionId: string, turns: ClassifiedTurn[], categoryBreakdown: SessionSummary['categoryBreakdown'] = {} as SessionSummary['categoryBreakdown']): SessionSummary { + return { + sessionId, + project: 'app', + firstTimestamp: turns[0]?.timestamp ?? '2026-05-05T00:00:00.000Z', + lastTimestamp: '2026-05-05T01:00:00.000Z', + totalCostUSD: 0, totalInputTokens: 0, totalOutputTokens: 0, totalReasoningTokens: 0, + totalCacheReadTokens: 0, totalCacheWriteTokens: 0, totalSavingsUSD: 0, + apiCalls: turns.reduce((s, t) => s + t.assistantCalls.length, 0), + turns, + modelBreakdown: {}, toolBreakdown: {}, mcpBreakdown: {}, bashBreakdown: {}, + categoryBreakdown, + skillBreakdown: {}, subagentBreakdown: {}, + } +} + +function project(sessions: SessionSummary[], projectPath = '/home/u/app'): ProjectSummary { + return { + project: 'app', projectPath, sessions, + totalCostUSD: 0, totalSavingsUSD: 0, totalApiCalls: 0, totalProxiedCostUSD: 0, + } +} + +function cat(editTurns: number, oneShotTurns: number): SessionSummary['categoryBreakdown'][TaskCategory] { + return { turns: editTurns, costUSD: 0, savingsUSD: 0, retries: 0, editTurns, oneShotTurns } +} + +describe('scanUserCorrections', () => { + it('counts turns whose user message signals a correction', () => { + const p = project([session('s1', [ + turn({ userMessage: "no, I meant the other file" }), + turn({ userMessage: "that's not what I asked for" }), + turn({ userMessage: "you missed the edge case" }), + turn({ userMessage: "revert that change" }), + turn({ userMessage: "add a new feature please" }), + ])]) + const r = scanUserCorrections([p]) + expect(r.corrections).toBe(4) + expect(r.userTurns).toBe(5) + expect(r.correctionRate).toBeCloseTo(0.8) + }) + + it('does not flag praise or ordinary requests (false-positive guards)', () => { + const phrases = [ + 'you were right about that', + "you're right, thanks", + "that's right, ship it", + 'looks correct to me', + 'undo the migration when done', + 'the build is failing, can you fix it', + 'what went wrong here', + ] + const p = project([session('s1', phrases.map(m => turn({ userMessage: m })))]) + const r = scanUserCorrections([p]) + expect(r.corrections).toBe(0) + }) + + it('ignores continuation turns with no fresh prompt', () => { + const p = project([session('s1', [ + turn({ userMessage: '' }), + turn({ userMessage: ' ' }), + turn({ userMessage: 'that is wrong' }), + ])]) + const r = scanUserCorrections([p]) + expect(r.userTurns).toBe(1) + expect(r.corrections).toBe(1) + expect(r.correctionRate).toBe(1) + }) + + it('returns a null rate for empty input', () => { + const r = scanUserCorrections([]) + expect(r).toEqual({ corrections: 0, userTurns: 0, correctionRate: null }) + }) + + it('every pattern is case-insensitive', () => { + for (const re of USER_CORRECTION_PATTERNS) expect(re.flags).toContain('i') + }) +}) + +describe('medianTimeToFirstEditMs', () => { + const edit = (ts: string) => call({ tools: ['Edit'], timestamp: ts }) + + it('measures from the first turn to the first edit call', () => { + const s = session('s1', [ + turn({ timestamp: '2026-05-05T00:00:00.000Z', calls: [call({ tools: ['Read'], timestamp: '2026-05-05T00:00:10.000Z' })] }), + turn({ timestamp: '2026-05-05T00:01:00.000Z', calls: [edit('2026-05-05T00:00:30.000Z')] }), + ]) + expect(medianTimeToFirstEditMs([project([s])])).toBe(30_000) + }) + + it('excludes sessions with no edits (not counted as zero)', () => { + const withEdit = session('s1', [turn({ timestamp: '2026-05-05T00:00:00.000Z', calls: [edit('2026-05-05T00:00:20.000Z')] })]) + const noEdit = session('s2', [turn({ timestamp: '2026-05-05T00:00:00.000Z', calls: [call({ tools: ['Read'], timestamp: '2026-05-05T00:05:00.000Z' })] })]) + expect(medianTimeToFirstEditMs([project([withEdit, noEdit])])).toBe(20_000) + }) + + it('takes the median across sessions (even count averages the middle two)', () => { + const mk = (id: string, secs: number) => session(id, [turn({ timestamp: '2026-05-05T00:00:00.000Z', calls: [edit(new Date(Date.parse('2026-05-05T00:00:00.000Z') + secs * 1000).toISOString())] })]) + // deltas: 10, 20, 30, 40 -> median (20+30)/2 = 25s + expect(medianTimeToFirstEditMs([project([mk('a', 10), mk('b', 20), mk('c', 30), mk('d', 40)])])).toBe(25_000) + }) + + it('clamps out-of-order timestamps to zero', () => { + const s = session('s1', [turn({ timestamp: '2026-05-05T00:01:00.000Z', calls: [edit('2026-05-05T00:00:00.000Z')] })]) + expect(medianTimeToFirstEditMs([project([s])])).toBe(0) + }) + + it('returns null when no session has an edit', () => { + const s = session('s1', [turn({ timestamp: '2026-05-05T00:00:00.000Z', calls: [call({ tools: ['Read'] })] })]) + expect(medianTimeToFirstEditMs([project([s])])).toBeNull() + }) +}) + +describe('aggregateFileChurn', () => { + const editSeq = (file: string): ToolCall[][] => [[{ tool: 'Edit', file }]] + + it('ranks by distinct sessions then total edits, and relativizes paths', () => { + const s1 = session('s1', [ + turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: editSeq('/home/u/app/src/a.ts') })] }), + turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: editSeq('/home/u/app/src/a.ts') })] }), + turn({ sessionId: 's1', calls: [call({ tools: ['Write'], toolSequence: [[{ tool: 'Write', file: '/home/u/app/src/b.ts' }]] })] }), + ]) + const s2 = session('s2', [ + turn({ sessionId: 's2', calls: [call({ tools: ['Edit'], toolSequence: editSeq('/home/u/app/src/a.ts') })] }), + ]) + const files = aggregateFileChurn([project([s1, s2])]) + expect(files[0]).toEqual({ path: 'src/a.ts', sessions: 2, edits: 3 }) + expect(files[1]).toEqual({ path: 'src/b.ts', sessions: 1, edits: 1 }) + }) + + it('breaks a session-count tie by edits then path', () => { + const s = session('s1', [ + turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: editSeq('/home/u/app/z.ts') })] }), + turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: editSeq('/home/u/app/z.ts') })] }), + turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: editSeq('/home/u/app/a.ts') })] }), + ]) + const files = aggregateFileChurn([project([s])]) + // both touched by 1 session; z.ts has more edits so it ranks first + expect(files.map(f => f.path)).toEqual(['z.ts', 'a.ts']) + }) + + it('ignores non-edit tools and calls without a file path', () => { + const s = session('s1', [ + turn({ sessionId: 's1', calls: [call({ tools: ['Read'], toolSequence: [[{ tool: 'Read', file: '/home/u/app/r.ts' }]] })] }), + turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: [[{ tool: 'Edit' }]] })] }), + ]) + expect(aggregateFileChurn([project([s])])).toEqual([]) + }) + + it('respects the limit', () => { + const turns = Array.from({ length: 5 }, (_, i) => + turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: editSeq(`/home/u/app/f${i}.ts`) })] })) + const files = aggregateFileChurn([project([session('s1', turns)])], 3) + expect(files).toHaveLength(3) + }) +}) + +describe('computePricingCoverage', () => { + it('is 1 when everything is priced', () => { + expect(computePricingCoverage(100, 0)).toBe(1) + }) + it('is the priced share when some calls are unpriced', () => { + expect(computePricingCoverage(100, 25)).toBe(0.75) + }) + it('is 1 when there are no cost-bearing calls', () => { + expect(computePricingCoverage(0, 0)).toBe(1) + }) + it('clamps to 0 if unpriced somehow exceeds the total', () => { + expect(computePricingCoverage(10, 20)).toBe(0) + }) +}) + +describe('worstOneShotCategory', () => { + it('picks the lowest one-shot rate above the edit-turn floor', () => { + const s = session('s1', [], { + feature: cat(10, 3), // 30% + debugging: cat(8, 6), // 75% + } as SessionSummary['categoryBreakdown']) + const worst = worstOneShotCategory([project([s])]) + expect(worst?.category).toBe('Feature Dev') + expect(worst?.rate).toBe(30) + expect(worst?.editTurns).toBe(10) + }) + + it('excludes categories below the edit-turn floor', () => { + const s = session('s1', [], { + feature: cat(2, 0), // 0% but only 2 edit turns + debugging: cat(10, 8), // 80% + } as SessionSummary['categoryBreakdown']) + const worst = worstOneShotCategory([project([s])]) + expect(worst?.category).toBe('Debugging') + }) + + it('returns null when nothing qualifies', () => { + const s = session('s1', [], { feature: cat(1, 0) } as SessionSummary['categoryBreakdown']) + expect(worstOneShotCategory([project([s])])).toBeNull() + }) +}) + +describe('buildCoachingNotes', () => { + it('emits a note per strong signal and never uses em-dashes', () => { + const notes = buildCoachingNotes({ + worstOneShot: { category: 'Feature Dev', rate: 40, editTurns: 12 }, + corrections: 6, + correctionRate: 0.2, + topReworkedFile: { path: 'src/a.ts', sessions: 4, edits: 20 }, + medianTimeToFirstEditMs: 6 * 60 * 1000, + }) + expect(notes.length).toBeGreaterThan(0) + expect(notes.length).toBeLessThanOrEqual(3) + for (const n of notes) expect(n).not.toContain('—') + }) + + it('caps at 3 notes even when all four signals fire', () => { + const notes = buildCoachingNotes({ + worstOneShot: { category: 'Feature Dev', rate: 40, editTurns: 12 }, + corrections: 6, + correctionRate: 0.2, + topReworkedFile: { path: 'src/a.ts', sessions: 4, edits: 20 }, + medianTimeToFirstEditMs: 6 * 60 * 1000, + }) + expect(notes).toHaveLength(3) + }) + + it('stays silent when every signal is below threshold', () => { + const notes = buildCoachingNotes({ + worstOneShot: { category: 'Feature Dev', rate: 90, editTurns: 12 }, + corrections: 1, + correctionRate: 0.02, + topReworkedFile: { path: 'src/a.ts', sessions: 1, edits: 2 }, + medianTimeToFirstEditMs: 10 * 1000, + }) + expect(notes).toEqual([]) + }) + + it('requires both a high rate and a minimum count for the correction note', () => { + const highRateLowCount = buildCoachingNotes({ correctionRate: 0.5, corrections: 1 }) + expect(highRateLowCount).toEqual([]) + }) +})