diff --git a/src/act/model-defaults.ts b/src/act/model-defaults.ts index 5103a982..537f6a0e 100644 --- a/src/act/model-defaults.ts +++ b/src/act/model-defaults.ts @@ -161,6 +161,7 @@ export async function buildApplyModelDefaultPlan(recommendation: ModelDefaultRec capturedAt: new Date().toISOString(), estimatedTokens: 0, sessions: recommendation.currentEditTurns + recommendation.candidateEditTurns, + candidateModel: recommendation.candidateModel, metrics: { [recommendation.candidateModel]: recommendation.candidateOneShotRate, [recommendation.currentModel]: recommendation.currentOneShotRate, diff --git a/src/act/report.ts b/src/act/report.ts index 72e350a9..95ec6e8c 100644 --- a/src/act/report.ts +++ b/src/act/report.ts @@ -1,4 +1,5 @@ import { existsSync } from 'fs' +import { dirname } from 'node:path' import type { DateRange, ProjectSummary, SessionSummary } from '../types.js' import type { ActionBaseline, ActionKind, ActionRecord } from './types.js' import type { FindingPlan } from './plans.js' @@ -124,6 +125,26 @@ function sessionsInWindow(projects: ProjectSummary[], start: Date, end: Date): S return out } +function projectPathKey(path: string): string { + const normalized = path.trim().replace(/\\/g, '/').replace(/\/+$/, '') + return normalized.toLowerCase() +} + +function modelDefaultSessionsInWindow( + rec: ActionRecord, projects: ProjectSummary[], start: Date, end: Date, +): { projectFound: boolean; sessions: SessionSummary[] } { + const settingsPath = rec.changes[0]?.path + if (!settingsPath) return { projectFound: false, sessions: [] } + const normalizedSettingsPath = settingsPath.replace(/\\/g, '/') + const targetProjectPath = dirname(dirname(normalizedSettingsPath)) + const targetKey = projectPathKey(targetProjectPath) + const targetProjects = projects.filter(project => projectPathKey(project.projectPath) === targetKey) + return { + projectFound: targetProjects.length > 0, + sessions: sessionsInWindow(targetProjects, start, end), + } +} + function countToolCalls(sessions: SessionSummary[], names: ReadonlySet): number { let n = 0 for (const s of sessions) { @@ -286,12 +307,16 @@ async function guardRow( async function modelDefaultRow( base: ActReportRow, rec: ActionRecord, sessions: SessionSummary[], - baseline: ActionBaseline, afterStart: Date, now: Date, + baseline: ActionBaseline, afterStart: Date, now: Date, projectFound: boolean, ): Promise { + if (!projectFound) { + return { ...base, note: 'not measurable: project not found in current data (path may have changed)' } + } const models = Object.keys(baseline.metrics) if (models.length < 2) return { ...base, note: 'not measurable: invalid baseline' } - const candidateModel = models[0]! - const preApplyRate = baseline.metrics[candidateModel]! + const candidateModel = baseline.candidateModel ?? models[0]! + const preApplyRate = baseline.metrics[candidateModel] + if (preApplyRate === undefined) return { ...base, note: 'not measurable: invalid baseline' } const mockProject: ProjectSummary = { project: 'mock', @@ -326,12 +351,15 @@ async function modelDefaultRow( ...base, status: 'measured', realizedTokens: 0, - confidence: 'normal', + confidence: confidenceFor(sessions.length, baseline, afterStart, now), note: `correlation, not attribution: one-shot rate ${(preApplyRate * 100).toFixed(1)}% -> ${(postApplyRate * 100).toFixed(1)}%` } } -async function computeRow(rec: ActionRecord, sessions: SessionSummary[], afterStart: Date, now: Date, opts: ActReportOptions): Promise { +async function computeRow( + rec: ActionRecord, sessions: SessionSummary[], afterStart: Date, now: Date, + opts: ActReportOptions, modelDefaultProjectFound = true, +): Promise { const estimatedAtApply = rec.baseline?.estimatedTokens ?? 0 const base: ActReportRow = { id: rec.id, @@ -354,7 +382,7 @@ async function computeRow(rec: ActionRecord, sessions: SessionSummary[], afterSt if (rec.kind === 'claude-md-rule') return readEditRow(base, sessions, baseline, afterStart, now) if (rec.kind === 'shell-config') return { ...base, note: 'not measurable: bash result token sizes are not retained in the summary' } if (rec.kind === 'guard-install') return guardRow(base, afterStart, now, baseline, opts) - if (rec.kind === 'model-default') return modelDefaultRow(base, rec, sessions, baseline, afterStart, now) + if (rec.kind === 'model-default') return modelDefaultRow(base, rec, sessions, baseline, afterStart, now, modelDefaultProjectFound) return { ...base, note: 'not measurable: kind is not tracked by act report' } } @@ -409,7 +437,11 @@ export async function computeActReport(opts: ActReportOptions = {}): Promise r.status === 'measured' && isTokenKind(r.kind)) @@ -453,6 +485,7 @@ function realizedCell(r: ActReportRow): string { if (r.status === 'reverted') return 'reverted' if (r.status === 'not-measurable') return 'not measurable' if (r.correlation) return `abandoned ${r.correlation.abandonedPctThen}% -> ${r.correlation.abandonedPctNow}% (corr.)` + if (r.kind === 'model-default') return 'correlation' return formatTokens(r.realizedTokens) } diff --git a/src/act/types.ts b/src/act/types.ts index 1f0baf67..043908e6 100644 --- a/src/act/types.ts +++ b/src/act/types.ts @@ -33,6 +33,8 @@ export type ActionBaseline = { estimatedTokens: number sessions: number metrics: Record + // model-default only: identifies the candidate independently of metrics key order. + candidateModel?: string } export type ActionRecord = { diff --git a/tests/act-report.test.ts b/tests/act-report.test.ts index ae0664c9..d690f138 100644 --- a/tests/act-report.test.ts +++ b/tests/act-report.test.ts @@ -11,7 +11,7 @@ import { renderActReport, } from '../src/act/report.js' import type { ActionRecord } from '../src/act/types.js' -import type { ProjectSummary } from '../src/types.js' +import type { ClassifiedTurn, ProjectSummary } from '../src/types.js' type Session = ProjectSummary['sessions'][number] @@ -58,10 +58,13 @@ function makeSession(id: string, firstTimestamp: string, over: Partial } } -function projectOf(sessions: Session[]): ProjectSummary { +function projectOf( + sessions: Session[], + over: { project?: string; projectPath?: string } = {}, +): ProjectSummary { return { - project: 'app', - projectPath: '/tmp/app', + project: over.project ?? 'app', + projectPath: over.projectPath ?? '/tmp/app', sessions, totalCostUSD: sessions.reduce((s, x) => s + x.totalCostUSD, 0), totalSavingsUSD: 0, @@ -89,6 +92,79 @@ function mcpRecord(over: Partial = {}): ActionRecord { } } +function modelEditTurns(model: string, editTurns: number, oneShotTurns: number): ClassifiedTurn[] { + return Array.from({ length: editTurns }, (_, i) => ({ + userMessage: 'edit the code', + timestamp: daysAgo(5), + sessionId: `model-${model}-${i}`, + category: 'coding', + retries: i < oneShotTurns ? 0 : 1, + hasEdits: true, + assistantCalls: [{ + provider: 'claude', + model, + usage: { + inputTokens: 100, + outputTokens: 50, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + }, + costUSD: 1, + tools: ['Edit'], + mcpTools: [], + skills: [], + subagentTypes: [], + hasAgentSpawn: false, + hasPlanMode: false, + speed: 'standard', + timestamp: daysAgo(5), + bashCommands: [], + deduplicationKey: `model-${model}-${i}`, + }], + })) +} + +function modelProject( + project: string, projectPath: string, model: string, editTurns: number, oneShotTurns: number, +): ProjectSummary { + const turns = modelEditTurns(model, editTurns, oneShotTurns) + const session = makeSession(`model-${project}`, daysAgo(5), { + project, + apiCalls: turns.length, + turns, + }) + return projectOf([session], { project, projectPath }) +} + +function modelDefaultRecord(over: Partial = {}): ActionRecord { + const at = daysAgo(10) + return { + id: 'md1', + at, + kind: 'model-default', + findingId: 'model-default:app', + description: 'Set Claude Code default model to candidate-model for app', + changes: [{ + path: '/tmp/app/.claude/settings.json', + backup: null, + op: 'edit', + afterHash: '', + }], + status: 'applied', + baseline: { + windowDays: 30, + capturedAt: at, + estimatedTokens: 0, + sessions: 60, + metrics: { 'candidate-model': 0.9, 'current-model': 0.95 }, + }, + ...over, + } +} + const load = (projects: ProjectSummary[]) => async () => projects describe('mcp realized delta', () => { @@ -132,6 +208,81 @@ describe('mcp realized delta', () => { }) }) +describe('model-default quality tripwire', () => { + it('fires for a >5pp same-project regression even when another project would mask it globally', async () => { + const actionsDir = await writeJournal([modelDefaultRecord()]) + const target = modelProject('app', '/tmp/app', 'candidate-model', 20, 10) + const masking = modelProject('other', '/tmp/other', 'candidate-model', 80, 80) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([target, masking]) }) + + const row = report.rows[0]! + expect(row.status).toBe('measured') + // Scope-fix pin: pre-fix global aggregation reports 90.0% instead of 50.0%. + expect(row.note).toBe('quality regression, consider undo: one-shot rate 90.0% -> 50.0%') + expect(row.confidence).toBe('low') + }) + + it('reports correlation without a regression and uses the labeled candidate model', async () => { + const rec = modelDefaultRecord({ + baseline: { + windowDays: 30, + capturedAt: daysAgo(10), + estimatedTokens: 0, + sessions: 60, + candidateModel: 'candidate-model', + metrics: { 'current-model': 0.95, 'candidate-model': 0.75 }, + }, + }) + const actionsDir = await writeJournal([rec]) + const target = modelProject('app', '/tmp/app', 'candidate-model', 20, 16) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([target]) }) + + const row = report.rows[0]! + expect(row.status).toBe('measured') + expect(row.confidence).toBe('low') + expect(row.note).toBe('correlation, not attribution: one-shot rate 75.0% -> 80.0%') + expect(renderActReport(report)).toMatch(/Set Claude Code default model to candidate-model for app\s+│\s+-\s+│\s+correlation\s+│/) + }) + + it('is not measurable with fewer than 20 candidate edit turns in the target project', async () => { + const actionsDir = await writeJournal([modelDefaultRecord()]) + const target = modelProject('app', '/tmp/app', 'candidate-model', 19, 19) + const unrelated = modelProject('other', '/tmp/other', 'candidate-model', 50, 50) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([target, unrelated]) }) + + const row = report.rows[0]! + expect(row.status).toBe('not-measurable') + expect(row.note).toBe('not measurable: < 20 edit turns for candidate-model since apply') + }) + + it('reports a missing scoped project separately from insufficient edit turns', async () => { + const actionsDir = await writeJournal([modelDefaultRecord()]) + const unrelated = modelProject('other', '/tmp/other', 'candidate-model', 50, 50) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([unrelated]) }) + + const row = report.rows[0]! + expect(row.status).toBe('not-measurable') + expect(row.note).toBe('not measurable: project not found in current data (path may have changed)') + }) + + it('matches a backslash-separated journal path to a forward-slash project path', async () => { + const rec = modelDefaultRecord({ + changes: [{ + path: 'C:\\work\\app\\.claude\\settings.json', + backup: null, + op: 'edit', + afterHash: '', + }], + }) + const actionsDir = await writeJournal([rec]) + const target = modelProject('app', 'C:/work/app', 'candidate-model', 20, 10) + const masking = modelProject('other', 'C:/work/other', 'candidate-model', 80, 80) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([target, masking]) }) + + expect(report.rows[0]!.note).toBe('quality regression, consider undo: one-shot rate 90.0% -> 50.0%') + }) +}) + describe('confidence markers', () => { it('marks low when fewer than 20 post-window sessions', async () => { const actionsDir = await writeJournal([mcpRecord()])