mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-05-17 03:56:45 +00:00
Merge pull request #206 from voidborne-d/fix/skill-subcategory-203
Some checks are pending
CI / semgrep (push) Waiting to run
Some checks are pending
CI / semgrep (push) Waiting to run
fix(classifier): surface skill name as subCategory for general turns (#203)
This commit is contained in:
commit
cf8c2aa493
9 changed files with 188 additions and 5 deletions
|
|
@ -53,6 +53,10 @@ function getAllTools(turn: ParsedTurn): string[] {
|
|||
return turn.assistantCalls.flatMap(c => c.tools)
|
||||
}
|
||||
|
||||
function getAllSkills(turn: ParsedTurn): string[] {
|
||||
return turn.assistantCalls.flatMap(c => c.skills ?? [])
|
||||
}
|
||||
|
||||
function classifyByToolPattern(turn: ParsedTurn): TaskCategory | null {
|
||||
const tools = getAllTools(turn)
|
||||
if (tools.length === 0) return null
|
||||
|
|
@ -159,5 +163,12 @@ export function classifyTurn(turn: ParsedTurn): ClassifiedTurn {
|
|||
}
|
||||
}
|
||||
|
||||
return { ...turn, category, retries: countRetries(turn), hasEdits: turnHasEdits(turn) }
|
||||
const result: ClassifiedTurn = { ...turn, category, retries: countRetries(turn), hasEdits: turnHasEdits(turn) }
|
||||
|
||||
if (category === 'general') {
|
||||
const skills = getAllSkills(turn)
|
||||
if (skills.length > 0) result.subCategory = skills[0]
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
|
|
|||
|
|
@ -353,8 +353,11 @@ function ModelBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw:
|
|||
)
|
||||
}
|
||||
|
||||
const SKILL_SUB_ROWS_LIMIT = 5
|
||||
|
||||
function ActivityBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw: number; bw: number }) {
|
||||
const categoryTotals: Record<string, { turns: number; costUSD: number; editTurns: number; oneShotTurns: number }> = {}
|
||||
const skillTotals: Record<string, { turns: number; costUSD: number; editTurns: number; oneShotTurns: number }> = {}
|
||||
for (const project of projects) {
|
||||
for (const session of project.sessions) {
|
||||
for (const [cat, data] of Object.entries(session.categoryBreakdown)) {
|
||||
|
|
@ -364,24 +367,47 @@ function ActivityBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; p
|
|||
categoryTotals[cat].editTurns += data.editTurns
|
||||
categoryTotals[cat].oneShotTurns += data.oneShotTurns
|
||||
}
|
||||
for (const [skill, data] of Object.entries(session.skillBreakdown ?? {})) {
|
||||
if (!skillTotals[skill]) skillTotals[skill] = { turns: 0, costUSD: 0, editTurns: 0, oneShotTurns: 0 }
|
||||
skillTotals[skill].turns += data.turns
|
||||
skillTotals[skill].costUSD += data.costUSD
|
||||
skillTotals[skill].editTurns += data.editTurns
|
||||
skillTotals[skill].oneShotTurns += data.oneShotTurns
|
||||
}
|
||||
}
|
||||
}
|
||||
const sorted = Object.entries(categoryTotals).sort(([, a], [, b]) => b.costUSD - a.costUSD)
|
||||
const sortedSkills = Object.entries(skillTotals).sort(([, a], [, b]) => b.costUSD - a.costUSD).slice(0, SKILL_SUB_ROWS_LIMIT)
|
||||
const maxCost = sorted[0]?.[1]?.costUSD ?? 0
|
||||
return (
|
||||
<Panel title="By Activity" color={PANEL_COLORS.activity} width={pw}>
|
||||
<Text dimColor wrap="truncate-end">{''.padEnd(bw + 14)}{'cost'.padStart(8)}{'turns'.padStart(6)}{'1-shot'.padStart(7)}</Text>
|
||||
{sorted.map(([cat, data]) => {
|
||||
{sorted.flatMap(([cat, data]) => {
|
||||
const oneShotPct = data.editTurns > 0 ? Math.round((data.oneShotTurns / data.editTurns) * 100) + '%' : '-'
|
||||
return (
|
||||
const rows = [
|
||||
<Text key={cat} wrap="truncate-end">
|
||||
<HBar value={data.costUSD} max={maxCost} width={bw} />
|
||||
<Text color={CATEGORY_COLORS[cat as TaskCategory] ?? '#666666'}> {fit(CATEGORY_LABELS[cat as TaskCategory] ?? cat, 13)}</Text>
|
||||
<Text color={GOLD}>{formatCost(data.costUSD).padStart(8)}</Text>
|
||||
<Text>{String(data.turns).padStart(6)}</Text>
|
||||
<Text color={data.editTurns === 0 ? DIM : oneShotPct === '100%' ? '#5BF58C' : ORANGE}>{String(oneShotPct).padStart(7)}</Text>
|
||||
</Text>
|
||||
)
|
||||
</Text>,
|
||||
]
|
||||
if (cat === 'general' && sortedSkills.length > 0) {
|
||||
for (const [skill, sd] of sortedSkills) {
|
||||
const subPct = sd.editTurns > 0 ? Math.round((sd.oneShotTurns / sd.editTurns) * 100) + '%' : '-'
|
||||
rows.push(
|
||||
<Text key={`${cat}:${skill}`} wrap="truncate-end" dimColor>
|
||||
<HBar value={sd.costUSD} max={maxCost} width={bw} />
|
||||
<Text> {fit(` /${skill}`, 13)}</Text>
|
||||
<Text>{formatCost(sd.costUSD).padStart(8)}</Text>
|
||||
<Text>{String(sd.turns).padStart(6)}</Text>
|
||||
<Text>{String(subPct).padStart(7)}</Text>
|
||||
</Text>,
|
||||
)
|
||||
}
|
||||
}
|
||||
return rows
|
||||
})}
|
||||
</Panel>
|
||||
)
|
||||
|
|
|
|||
|
|
@ -44,6 +44,17 @@ function extractMcpTools(tools: string[]): string[] {
|
|||
return tools.filter(t => t.startsWith('mcp__'))
|
||||
}
|
||||
|
||||
function extractSkillNames(content: ContentBlock[]): string[] {
|
||||
return content
|
||||
.filter((b): b is ToolUseBlock => b.type === 'tool_use' && b.name === 'Skill')
|
||||
.map(b => {
|
||||
const input = (b.input ?? {}) as Record<string, unknown>
|
||||
const raw = input['skill'] ?? input['name']
|
||||
return typeof raw === 'string' ? raw.trim() : ''
|
||||
})
|
||||
.filter(name => name.length > 0)
|
||||
}
|
||||
|
||||
function extractCoreTools(tools: string[]): string[] {
|
||||
return tools.filter(t => !t.startsWith('mcp__'))
|
||||
}
|
||||
|
|
@ -93,6 +104,7 @@ function parseApiCall(entry: JournalEntry): ParsedApiCall | null {
|
|||
}
|
||||
|
||||
const tools = extractToolNames(msg.content ?? [])
|
||||
const skills = extractSkillNames(msg.content ?? [])
|
||||
const costUSD = calculateCost(
|
||||
msg.model,
|
||||
tokens.inputTokens,
|
||||
|
|
@ -112,6 +124,7 @@ function parseApiCall(entry: JournalEntry): ParsedApiCall | null {
|
|||
costUSD,
|
||||
tools,
|
||||
mcpTools: extractMcpTools(tools),
|
||||
skills,
|
||||
hasAgentSpawn: tools.includes('Agent'),
|
||||
hasPlanMode: tools.includes('EnterPlanMode'),
|
||||
speed: usage.speed ?? 'standard',
|
||||
|
|
@ -200,6 +213,7 @@ function buildSessionSummary(
|
|||
const mcpBreakdown: SessionSummary['mcpBreakdown'] = Object.create(null)
|
||||
const bashBreakdown: SessionSummary['bashBreakdown'] = Object.create(null)
|
||||
const categoryBreakdown: SessionSummary['categoryBreakdown'] = Object.create(null)
|
||||
const skillBreakdown: SessionSummary['skillBreakdown'] = Object.create(null)
|
||||
|
||||
let totalCost = 0
|
||||
let totalInput = 0
|
||||
|
|
@ -224,6 +238,19 @@ function buildSessionSummary(
|
|||
if (turn.retries === 0) categoryBreakdown[turn.category].oneShotTurns++
|
||||
}
|
||||
|
||||
if (turn.subCategory) {
|
||||
const skillKey = turn.subCategory
|
||||
if (!skillBreakdown[skillKey]) {
|
||||
skillBreakdown[skillKey] = { turns: 0, costUSD: 0, editTurns: 0, oneShotTurns: 0 }
|
||||
}
|
||||
skillBreakdown[skillKey].turns++
|
||||
skillBreakdown[skillKey].costUSD += turnCost
|
||||
if (turn.hasEdits) {
|
||||
skillBreakdown[skillKey].editTurns++
|
||||
if (turn.retries === 0) skillBreakdown[skillKey].oneShotTurns++
|
||||
}
|
||||
}
|
||||
|
||||
for (const call of turn.assistantCalls) {
|
||||
totalCost += call.costUSD
|
||||
totalInput += call.usage.inputTokens
|
||||
|
|
@ -283,6 +310,7 @@ function buildSessionSummary(
|
|||
mcpBreakdown,
|
||||
bashBreakdown,
|
||||
categoryBreakdown,
|
||||
skillBreakdown,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -402,6 +430,7 @@ function providerCallToTurn(call: ParsedProviderCall): ParsedTurn {
|
|||
costUSD: call.costUSD,
|
||||
tools,
|
||||
mcpTools: extractMcpTools(tools),
|
||||
skills: [],
|
||||
hasAgentSpawn: tools.includes('Agent'),
|
||||
hasPlanMode: tools.includes('EnterPlanMode'),
|
||||
speed: call.speed,
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ export type ParsedApiCall = {
|
|||
costUSD: number
|
||||
tools: string[]
|
||||
mcpTools: string[]
|
||||
skills: string[]
|
||||
hasAgentSpawn: boolean
|
||||
hasPlanMode: boolean
|
||||
speed: 'standard' | 'fast'
|
||||
|
|
@ -97,6 +98,7 @@ export type TaskCategory =
|
|||
|
||||
export type ClassifiedTurn = ParsedTurn & {
|
||||
category: TaskCategory
|
||||
subCategory?: string
|
||||
retries: number
|
||||
hasEdits: boolean
|
||||
}
|
||||
|
|
@ -118,6 +120,7 @@ export type SessionSummary = {
|
|||
mcpBreakdown: Record<string, { calls: number }>
|
||||
bashBreakdown: Record<string, { calls: number }>
|
||||
categoryBreakdown: Record<TaskCategory, { turns: number; costUSD: number; retries: number; editTurns: number; oneShotTurns: number }>
|
||||
skillBreakdown: Record<string, { turns: number; costUSD: number; editTurns: number; oneShotTurns: number }>
|
||||
}
|
||||
|
||||
export type ProjectSummary = {
|
||||
|
|
|
|||
103
tests/classifier.test.ts
Normal file
103
tests/classifier.test.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
import { describe, it, expect } from 'vitest'
|
||||
|
||||
import { classifyTurn } from '../src/classifier.js'
|
||||
import type { ParsedApiCall, ParsedTurn } from '../src/types.js'
|
||||
|
||||
function makeCall(opts: Partial<ParsedApiCall> & { tools?: string[]; skills?: string[] }): ParsedApiCall {
|
||||
const tools = opts.tools ?? []
|
||||
return {
|
||||
provider: 'claude',
|
||||
model: 'Opus 4.7',
|
||||
usage: {
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
},
|
||||
costUSD: 0,
|
||||
tools,
|
||||
mcpTools: tools.filter(t => t.startsWith('mcp__')),
|
||||
skills: opts.skills ?? [],
|
||||
hasAgentSpawn: tools.includes('Agent'),
|
||||
hasPlanMode: tools.includes('EnterPlanMode'),
|
||||
speed: 'standard',
|
||||
timestamp: '2026-05-04T00:00:00Z',
|
||||
bashCommands: [],
|
||||
deduplicationKey: 'k',
|
||||
...opts,
|
||||
}
|
||||
}
|
||||
|
||||
function makeTurn(calls: ParsedApiCall[], userMessage = ''): ParsedTurn {
|
||||
return {
|
||||
userMessage,
|
||||
assistantCalls: calls,
|
||||
timestamp: '2026-05-04T00:00:00Z',
|
||||
sessionId: 's1',
|
||||
}
|
||||
}
|
||||
|
||||
describe('classifyTurn — Skill subCategory', () => {
|
||||
it('attaches subCategory when a Skill tool fires alone (input.skill)', () => {
|
||||
const turn = makeTurn([makeCall({ tools: ['Skill'], skills: ['init'] })])
|
||||
const c = classifyTurn(turn)
|
||||
expect(c.category).toBe('general')
|
||||
expect(c.subCategory).toBe('init')
|
||||
})
|
||||
|
||||
it('attaches subCategory when skill identifier comes via input.name (extracted upstream)', () => {
|
||||
const turn = makeTurn([makeCall({ tools: ['Skill'], skills: ['atelier'] })])
|
||||
const c = classifyTurn(turn)
|
||||
expect(c.category).toBe('general')
|
||||
expect(c.subCategory).toBe('atelier')
|
||||
})
|
||||
|
||||
it('uses the first skill identifier when a single turn invokes multiple skills', () => {
|
||||
const turn = makeTurn([makeCall({ tools: ['Skill', 'Skill'], skills: ['review', 'security-review'] })])
|
||||
const c = classifyTurn(turn)
|
||||
expect(c.category).toBe('general')
|
||||
expect(c.subCategory).toBe('review')
|
||||
})
|
||||
|
||||
it('aggregates skills across multiple assistant calls in the same turn', () => {
|
||||
const turn = makeTurn([
|
||||
makeCall({ tools: ['Skill'], skills: ['claude-api'] }),
|
||||
makeCall({ tools: ['Skill'], skills: ['init'] }),
|
||||
])
|
||||
const c = classifyTurn(turn)
|
||||
expect(c.category).toBe('general')
|
||||
expect(c.subCategory).toBe('claude-api')
|
||||
})
|
||||
|
||||
it('does not attach subCategory when the Skill tool fires but no skill name was extracted', () => {
|
||||
const turn = makeTurn([makeCall({ tools: ['Skill'], skills: [] })])
|
||||
const c = classifyTurn(turn)
|
||||
expect(c.category).toBe('general')
|
||||
expect(c.subCategory).toBeUndefined()
|
||||
})
|
||||
|
||||
it('does not attach subCategory when category is not general (e.g. Skill alongside Edit promotes to coding)', () => {
|
||||
const turn = makeTurn([makeCall({ tools: ['Skill', 'Edit'], skills: ['init'] })])
|
||||
const c = classifyTurn(turn)
|
||||
expect(c.category).toBe('coding')
|
||||
expect(c.subCategory).toBeUndefined()
|
||||
})
|
||||
|
||||
it('does not attach subCategory for non-Skill general turns', () => {
|
||||
const turn = makeTurn([makeCall({ tools: [] })], 'just chatting')
|
||||
const c = classifyTurn(turn)
|
||||
expect(c.subCategory).toBeUndefined()
|
||||
})
|
||||
|
||||
it('tolerates missing skills field on legacy ParsedApiCall shape', () => {
|
||||
const baseCall = makeCall({ tools: ['Skill'], skills: ['init'] })
|
||||
const legacyCall = { ...baseCall } as unknown as ParsedApiCall & { skills?: string[] }
|
||||
delete (legacyCall as { skills?: string[] }).skills
|
||||
const c = classifyTurn(makeTurn([legacyCall]))
|
||||
expect(c.category).toBe('general')
|
||||
expect(c.subCategory).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
|
@ -28,6 +28,7 @@ function makeTurn(model: string, cost: number, opts: { hasEdits?: boolean; retri
|
|||
costUSD: cost,
|
||||
tools: defaultTools,
|
||||
mcpTools: [],
|
||||
skills: [],
|
||||
hasAgentSpawn: opts.hasAgentSpawn ?? false,
|
||||
hasPlanMode: opts.hasPlanMode ?? false,
|
||||
speed: opts.speed ?? 'standard' as const,
|
||||
|
|
@ -56,6 +57,7 @@ function makeProject(turns: ClassifiedTurn[]): ProjectSummary {
|
|||
mcpBreakdown: {},
|
||||
bashBreakdown: {},
|
||||
categoryBreakdown: {} as SessionSummary['categoryBreakdown'],
|
||||
skillBreakdown: {} as SessionSummary['skillBreakdown'],
|
||||
}
|
||||
return {
|
||||
project: 'test-project',
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ function makeSession(id: string, cost: number, timestamp = '2026-04-14T10:00:00Z
|
|||
mcpBreakdown: {},
|
||||
bashBreakdown: {},
|
||||
categoryBreakdown: { ...EMPTY_CATEGORY_BREAKDOWN },
|
||||
skillBreakdown: {},
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ function makeCall(timestamp: string, costUSD: number, model = 'Opus 4.7', provid
|
|||
costUSD,
|
||||
tools: [],
|
||||
mcpTools: [],
|
||||
skills: [],
|
||||
hasAgentSpawn: false,
|
||||
hasPlanMode: false,
|
||||
speed: 'standard' as const,
|
||||
|
|
@ -72,6 +73,7 @@ describe('aggregateProjectsIntoDays', () => {
|
|||
mcpBreakdown: {},
|
||||
bashBreakdown: {},
|
||||
categoryBreakdown: {} as never,
|
||||
skillBreakdown: {} as never,
|
||||
}],
|
||||
}),
|
||||
]
|
||||
|
|
@ -114,6 +116,7 @@ describe('aggregateProjectsIntoDays', () => {
|
|||
mcpBreakdown: {},
|
||||
bashBreakdown: {},
|
||||
categoryBreakdown: {} as never,
|
||||
skillBreakdown: {} as never,
|
||||
}],
|
||||
}),
|
||||
]
|
||||
|
|
@ -143,6 +146,7 @@ describe('aggregateProjectsIntoDays', () => {
|
|||
turns: [],
|
||||
modelBreakdown: {}, toolBreakdown: {}, mcpBreakdown: {}, bashBreakdown: {},
|
||||
categoryBreakdown: {} as never,
|
||||
skillBreakdown: {} as never,
|
||||
}],
|
||||
}),
|
||||
]
|
||||
|
|
@ -175,6 +179,7 @@ describe('aggregateProjectsIntoDays', () => {
|
|||
],
|
||||
modelBreakdown: {}, toolBreakdown: {}, mcpBreakdown: {}, bashBreakdown: {},
|
||||
categoryBreakdown: {} as never,
|
||||
skillBreakdown: {} as never,
|
||||
}],
|
||||
}),
|
||||
]
|
||||
|
|
@ -290,6 +295,7 @@ describe('buildPeriodDataFromDays', () => {
|
|||
}],
|
||||
modelBreakdown: {}, toolBreakdown: {}, mcpBreakdown: {}, bashBreakdown: {},
|
||||
categoryBreakdown: {} as never,
|
||||
skillBreakdown: {} as never,
|
||||
}],
|
||||
}),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ function makeProject(projectPath: string): ProjectSummary {
|
|||
costUSD: 1.23,
|
||||
tools: ['Read'],
|
||||
mcpTools: [],
|
||||
skills: [],
|
||||
hasAgentSpawn: false,
|
||||
hasPlanMode: false,
|
||||
speed: 'standard',
|
||||
|
|
@ -103,6 +104,7 @@ function makeProject(projectPath: string): ProjectSummary {
|
|||
brainstorming: { turns: 0, costUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
general: { turns: 0, costUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
},
|
||||
skillBreakdown: {},
|
||||
},
|
||||
],
|
||||
totalCostUSD: 1.23,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue