Merge pull request #206 from voidborne-d/fix/skill-subcategory-203
Some checks are pending
CI / semgrep (push) Waiting to run

fix(classifier): surface skill name as subCategory for general turns (#203)
This commit is contained in:
Resham Joshi 2026-05-03 16:42:47 -07:00 committed by GitHub
commit cf8c2aa493
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 188 additions and 5 deletions

View file

@ -53,6 +53,10 @@ function getAllTools(turn: ParsedTurn): string[] {
return turn.assistantCalls.flatMap(c => c.tools)
}
function getAllSkills(turn: ParsedTurn): string[] {
return turn.assistantCalls.flatMap(c => c.skills ?? [])
}
function classifyByToolPattern(turn: ParsedTurn): TaskCategory | null {
const tools = getAllTools(turn)
if (tools.length === 0) return null
@ -159,5 +163,12 @@ export function classifyTurn(turn: ParsedTurn): ClassifiedTurn {
}
}
return { ...turn, category, retries: countRetries(turn), hasEdits: turnHasEdits(turn) }
const result: ClassifiedTurn = { ...turn, category, retries: countRetries(turn), hasEdits: turnHasEdits(turn) }
if (category === 'general') {
const skills = getAllSkills(turn)
if (skills.length > 0) result.subCategory = skills[0]
}
return result
}

View file

@ -353,8 +353,11 @@ function ModelBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw:
)
}
const SKILL_SUB_ROWS_LIMIT = 5
function ActivityBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw: number; bw: number }) {
const categoryTotals: Record<string, { turns: number; costUSD: number; editTurns: number; oneShotTurns: number }> = {}
const skillTotals: Record<string, { turns: number; costUSD: number; editTurns: number; oneShotTurns: number }> = {}
for (const project of projects) {
for (const session of project.sessions) {
for (const [cat, data] of Object.entries(session.categoryBreakdown)) {
@ -364,24 +367,47 @@ function ActivityBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; p
categoryTotals[cat].editTurns += data.editTurns
categoryTotals[cat].oneShotTurns += data.oneShotTurns
}
for (const [skill, data] of Object.entries(session.skillBreakdown ?? {})) {
if (!skillTotals[skill]) skillTotals[skill] = { turns: 0, costUSD: 0, editTurns: 0, oneShotTurns: 0 }
skillTotals[skill].turns += data.turns
skillTotals[skill].costUSD += data.costUSD
skillTotals[skill].editTurns += data.editTurns
skillTotals[skill].oneShotTurns += data.oneShotTurns
}
}
}
const sorted = Object.entries(categoryTotals).sort(([, a], [, b]) => b.costUSD - a.costUSD)
const sortedSkills = Object.entries(skillTotals).sort(([, a], [, b]) => b.costUSD - a.costUSD).slice(0, SKILL_SUB_ROWS_LIMIT)
const maxCost = sorted[0]?.[1]?.costUSD ?? 0
return (
<Panel title="By Activity" color={PANEL_COLORS.activity} width={pw}>
<Text dimColor wrap="truncate-end">{''.padEnd(bw + 14)}{'cost'.padStart(8)}{'turns'.padStart(6)}{'1-shot'.padStart(7)}</Text>
{sorted.map(([cat, data]) => {
{sorted.flatMap(([cat, data]) => {
const oneShotPct = data.editTurns > 0 ? Math.round((data.oneShotTurns / data.editTurns) * 100) + '%' : '-'
return (
const rows = [
<Text key={cat} wrap="truncate-end">
<HBar value={data.costUSD} max={maxCost} width={bw} />
<Text color={CATEGORY_COLORS[cat as TaskCategory] ?? '#666666'}> {fit(CATEGORY_LABELS[cat as TaskCategory] ?? cat, 13)}</Text>
<Text color={GOLD}>{formatCost(data.costUSD).padStart(8)}</Text>
<Text>{String(data.turns).padStart(6)}</Text>
<Text color={data.editTurns === 0 ? DIM : oneShotPct === '100%' ? '#5BF58C' : ORANGE}>{String(oneShotPct).padStart(7)}</Text>
</Text>
)
</Text>,
]
if (cat === 'general' && sortedSkills.length > 0) {
for (const [skill, sd] of sortedSkills) {
const subPct = sd.editTurns > 0 ? Math.round((sd.oneShotTurns / sd.editTurns) * 100) + '%' : '-'
rows.push(
<Text key={`${cat}:${skill}`} wrap="truncate-end" dimColor>
<HBar value={sd.costUSD} max={maxCost} width={bw} />
<Text> {fit(` /${skill}`, 13)}</Text>
<Text>{formatCost(sd.costUSD).padStart(8)}</Text>
<Text>{String(sd.turns).padStart(6)}</Text>
<Text>{String(subPct).padStart(7)}</Text>
</Text>,
)
}
}
return rows
})}
</Panel>
)

View file

@ -44,6 +44,17 @@ function extractMcpTools(tools: string[]): string[] {
return tools.filter(t => t.startsWith('mcp__'))
}
function extractSkillNames(content: ContentBlock[]): string[] {
return content
.filter((b): b is ToolUseBlock => b.type === 'tool_use' && b.name === 'Skill')
.map(b => {
const input = (b.input ?? {}) as Record<string, unknown>
const raw = input['skill'] ?? input['name']
return typeof raw === 'string' ? raw.trim() : ''
})
.filter(name => name.length > 0)
}
function extractCoreTools(tools: string[]): string[] {
return tools.filter(t => !t.startsWith('mcp__'))
}
@ -93,6 +104,7 @@ function parseApiCall(entry: JournalEntry): ParsedApiCall | null {
}
const tools = extractToolNames(msg.content ?? [])
const skills = extractSkillNames(msg.content ?? [])
const costUSD = calculateCost(
msg.model,
tokens.inputTokens,
@ -112,6 +124,7 @@ function parseApiCall(entry: JournalEntry): ParsedApiCall | null {
costUSD,
tools,
mcpTools: extractMcpTools(tools),
skills,
hasAgentSpawn: tools.includes('Agent'),
hasPlanMode: tools.includes('EnterPlanMode'),
speed: usage.speed ?? 'standard',
@ -200,6 +213,7 @@ function buildSessionSummary(
const mcpBreakdown: SessionSummary['mcpBreakdown'] = Object.create(null)
const bashBreakdown: SessionSummary['bashBreakdown'] = Object.create(null)
const categoryBreakdown: SessionSummary['categoryBreakdown'] = Object.create(null)
const skillBreakdown: SessionSummary['skillBreakdown'] = Object.create(null)
let totalCost = 0
let totalInput = 0
@ -224,6 +238,19 @@ function buildSessionSummary(
if (turn.retries === 0) categoryBreakdown[turn.category].oneShotTurns++
}
if (turn.subCategory) {
const skillKey = turn.subCategory
if (!skillBreakdown[skillKey]) {
skillBreakdown[skillKey] = { turns: 0, costUSD: 0, editTurns: 0, oneShotTurns: 0 }
}
skillBreakdown[skillKey].turns++
skillBreakdown[skillKey].costUSD += turnCost
if (turn.hasEdits) {
skillBreakdown[skillKey].editTurns++
if (turn.retries === 0) skillBreakdown[skillKey].oneShotTurns++
}
}
for (const call of turn.assistantCalls) {
totalCost += call.costUSD
totalInput += call.usage.inputTokens
@ -283,6 +310,7 @@ function buildSessionSummary(
mcpBreakdown,
bashBreakdown,
categoryBreakdown,
skillBreakdown,
}
}
@ -402,6 +430,7 @@ function providerCallToTurn(call: ParsedProviderCall): ParsedTurn {
costUSD: call.costUSD,
tools,
mcpTools: extractMcpTools(tools),
skills: [],
hasAgentSpawn: tools.includes('Agent'),
hasPlanMode: tools.includes('EnterPlanMode'),
speed: call.speed,

View file

@ -72,6 +72,7 @@ export type ParsedApiCall = {
costUSD: number
tools: string[]
mcpTools: string[]
skills: string[]
hasAgentSpawn: boolean
hasPlanMode: boolean
speed: 'standard' | 'fast'
@ -97,6 +98,7 @@ export type TaskCategory =
export type ClassifiedTurn = ParsedTurn & {
category: TaskCategory
subCategory?: string
retries: number
hasEdits: boolean
}
@ -118,6 +120,7 @@ export type SessionSummary = {
mcpBreakdown: Record<string, { calls: number }>
bashBreakdown: Record<string, { calls: number }>
categoryBreakdown: Record<TaskCategory, { turns: number; costUSD: number; retries: number; editTurns: number; oneShotTurns: number }>
skillBreakdown: Record<string, { turns: number; costUSD: number; editTurns: number; oneShotTurns: number }>
}
export type ProjectSummary = {

103
tests/classifier.test.ts Normal file
View file

@ -0,0 +1,103 @@
import { describe, it, expect } from 'vitest'
import { classifyTurn } from '../src/classifier.js'
import type { ParsedApiCall, ParsedTurn } from '../src/types.js'
function makeCall(opts: Partial<ParsedApiCall> & { tools?: string[]; skills?: string[] }): ParsedApiCall {
const tools = opts.tools ?? []
return {
provider: 'claude',
model: 'Opus 4.7',
usage: {
inputTokens: 0,
outputTokens: 0,
cacheCreationInputTokens: 0,
cacheReadInputTokens: 0,
cachedInputTokens: 0,
reasoningTokens: 0,
webSearchRequests: 0,
},
costUSD: 0,
tools,
mcpTools: tools.filter(t => t.startsWith('mcp__')),
skills: opts.skills ?? [],
hasAgentSpawn: tools.includes('Agent'),
hasPlanMode: tools.includes('EnterPlanMode'),
speed: 'standard',
timestamp: '2026-05-04T00:00:00Z',
bashCommands: [],
deduplicationKey: 'k',
...opts,
}
}
function makeTurn(calls: ParsedApiCall[], userMessage = ''): ParsedTurn {
return {
userMessage,
assistantCalls: calls,
timestamp: '2026-05-04T00:00:00Z',
sessionId: 's1',
}
}
describe('classifyTurn — Skill subCategory', () => {
it('attaches subCategory when a Skill tool fires alone (input.skill)', () => {
const turn = makeTurn([makeCall({ tools: ['Skill'], skills: ['init'] })])
const c = classifyTurn(turn)
expect(c.category).toBe('general')
expect(c.subCategory).toBe('init')
})
it('attaches subCategory when skill identifier comes via input.name (extracted upstream)', () => {
const turn = makeTurn([makeCall({ tools: ['Skill'], skills: ['atelier'] })])
const c = classifyTurn(turn)
expect(c.category).toBe('general')
expect(c.subCategory).toBe('atelier')
})
it('uses the first skill identifier when a single turn invokes multiple skills', () => {
const turn = makeTurn([makeCall({ tools: ['Skill', 'Skill'], skills: ['review', 'security-review'] })])
const c = classifyTurn(turn)
expect(c.category).toBe('general')
expect(c.subCategory).toBe('review')
})
it('aggregates skills across multiple assistant calls in the same turn', () => {
const turn = makeTurn([
makeCall({ tools: ['Skill'], skills: ['claude-api'] }),
makeCall({ tools: ['Skill'], skills: ['init'] }),
])
const c = classifyTurn(turn)
expect(c.category).toBe('general')
expect(c.subCategory).toBe('claude-api')
})
it('does not attach subCategory when the Skill tool fires but no skill name was extracted', () => {
const turn = makeTurn([makeCall({ tools: ['Skill'], skills: [] })])
const c = classifyTurn(turn)
expect(c.category).toBe('general')
expect(c.subCategory).toBeUndefined()
})
it('does not attach subCategory when category is not general (e.g. Skill alongside Edit promotes to coding)', () => {
const turn = makeTurn([makeCall({ tools: ['Skill', 'Edit'], skills: ['init'] })])
const c = classifyTurn(turn)
expect(c.category).toBe('coding')
expect(c.subCategory).toBeUndefined()
})
it('does not attach subCategory for non-Skill general turns', () => {
const turn = makeTurn([makeCall({ tools: [] })], 'just chatting')
const c = classifyTurn(turn)
expect(c.subCategory).toBeUndefined()
})
it('tolerates missing skills field on legacy ParsedApiCall shape', () => {
const baseCall = makeCall({ tools: ['Skill'], skills: ['init'] })
const legacyCall = { ...baseCall } as unknown as ParsedApiCall & { skills?: string[] }
delete (legacyCall as { skills?: string[] }).skills
const c = classifyTurn(makeTurn([legacyCall]))
expect(c.category).toBe('general')
expect(c.subCategory).toBeUndefined()
})
})

View file

@ -28,6 +28,7 @@ function makeTurn(model: string, cost: number, opts: { hasEdits?: boolean; retri
costUSD: cost,
tools: defaultTools,
mcpTools: [],
skills: [],
hasAgentSpawn: opts.hasAgentSpawn ?? false,
hasPlanMode: opts.hasPlanMode ?? false,
speed: opts.speed ?? 'standard' as const,
@ -56,6 +57,7 @@ function makeProject(turns: ClassifiedTurn[]): ProjectSummary {
mcpBreakdown: {},
bashBreakdown: {},
categoryBreakdown: {} as SessionSummary['categoryBreakdown'],
skillBreakdown: {} as SessionSummary['skillBreakdown'],
}
return {
project: 'test-project',

View file

@ -37,6 +37,7 @@ function makeSession(id: string, cost: number, timestamp = '2026-04-14T10:00:00Z
mcpBreakdown: {},
bashBreakdown: {},
categoryBreakdown: { ...EMPTY_CATEGORY_BREAKDOWN },
skillBreakdown: {},
}
}

View file

@ -29,6 +29,7 @@ function makeCall(timestamp: string, costUSD: number, model = 'Opus 4.7', provid
costUSD,
tools: [],
mcpTools: [],
skills: [],
hasAgentSpawn: false,
hasPlanMode: false,
speed: 'standard' as const,
@ -72,6 +73,7 @@ describe('aggregateProjectsIntoDays', () => {
mcpBreakdown: {},
bashBreakdown: {},
categoryBreakdown: {} as never,
skillBreakdown: {} as never,
}],
}),
]
@ -114,6 +116,7 @@ describe('aggregateProjectsIntoDays', () => {
mcpBreakdown: {},
bashBreakdown: {},
categoryBreakdown: {} as never,
skillBreakdown: {} as never,
}],
}),
]
@ -143,6 +146,7 @@ describe('aggregateProjectsIntoDays', () => {
turns: [],
modelBreakdown: {}, toolBreakdown: {}, mcpBreakdown: {}, bashBreakdown: {},
categoryBreakdown: {} as never,
skillBreakdown: {} as never,
}],
}),
]
@ -175,6 +179,7 @@ describe('aggregateProjectsIntoDays', () => {
],
modelBreakdown: {}, toolBreakdown: {}, mcpBreakdown: {}, bashBreakdown: {},
categoryBreakdown: {} as never,
skillBreakdown: {} as never,
}],
}),
]
@ -290,6 +295,7 @@ describe('buildPeriodDataFromDays', () => {
}],
modelBreakdown: {}, toolBreakdown: {}, mcpBreakdown: {}, bashBreakdown: {},
categoryBreakdown: {} as never,
skillBreakdown: {} as never,
}],
}),
]

View file

@ -56,6 +56,7 @@ function makeProject(projectPath: string): ProjectSummary {
costUSD: 1.23,
tools: ['Read'],
mcpTools: [],
skills: [],
hasAgentSpawn: false,
hasPlanMode: false,
speed: 'standard',
@ -103,6 +104,7 @@ function makeProject(projectPath: string): ProjectSummary {
brainstorming: { turns: 0, costUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
general: { turns: 0, costUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
},
skillBreakdown: {},
},
],
totalCostUSD: 1.23,