From 7a878f4d19e3873cb3fc14338dbdf8d402a23bec Mon Sep 17 00:00:00 2001 From: Resham Joshi <65915470+iamtoruk@users.noreply.github.com> Date: Sat, 9 May 2026 22:48:11 -0700 Subject: [PATCH] Classifier: feature verb wins over debug keyword (part of #196) (#289) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Messages like "add error handling", "create an issue tracker", or "implement the 404 page" were landing in the Debugging bucket because the classifier checked DEBUG_KEYWORDS (which matches `error`, `issue`, `404`) before FEATURE_KEYWORDS in both `refineByKeywords` (tool-bearing turns) and `classifyConversation` (chat-only turns). The position of the matched word in the sentence is a much stronger intent signal than the order of the checks in code, so we now pick whichever pattern matches earliest. The new helper `firstMatchingCategory` runs each candidate regex once with `RegExp.exec` and keeps the match with the lowest `index`. Ties (rare in practice — same start position) break by the order the candidates were listed, which is `refactoring > feature > debugging` for coding turns. That ordering preserves existing behavior for plain bug reports (e.g. "login is broken, traceback below") while flipping mislabeled feature work to its correct category. 8 regression tests in `tests/classifier.test.ts` cover the mislabel cases from #196 plus tie-break / chat-only cases. Full suite: 45 files / 609 tests, all green. Closes the activity-misattribution half of #196. The Cursor provider attribution half (single 'cursor' project for all sessions) is addressed in a separate PR. --- CHANGELOG.md | 11 +++++++++ src/classifier.ts | 44 ++++++++++++++++++++++++++++++----- tests/classifier.test.ts | 50 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1be6fa..167a271 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,17 @@ period-level `activities[]` rollup so a consumer can sum across days and reconcile. Closes #279. +### Fixed (CLI) +- **Activity classifier no longer mislabels feature work as debugging.** + Messages like "add error handling", "create an issue tracker", or + "implement the 404 page" used to land in the Debugging bucket because + the classifier checked the debug-keyword regex (which matches `error`, + `issue`, `404`) before the feature regex. Now the keyword that appears + earliest in the user message wins, so "add" beats "error", "create" + beats "issue", etc. A real bug report ("login is broken, traceback + below") still classifies as debugging because the debug word leads. + Fixes the activity-misattribution half of #196. + ### Changed (CLI) - **`optimize` suggestions now declare their destination.** Every paste-style fix carries an explicit destination — `claude-md` (permanent project rule), diff --git a/src/classifier.ts b/src/classifier.ts index 28076c8..9a5de49 100644 --- a/src/classifier.ts +++ b/src/classifier.ts @@ -93,12 +93,38 @@ function classifyByToolPattern(turn: ParsedTurn): TaskCategory | null { return null } +/// Picks the category whose keyword pattern matches earliest in the message. +/// On a tie (same start index) the candidate listed first in `candidates` wins, +/// so callers control tie-break priority by ordering. Returns null when no +/// pattern matches. The first-match heuristic fixes the long-standing problem +/// where "add error handling" was tagged Debugging because the DEBUG regex was +/// checked before FEATURE; now FEATURE wins because "add" appears before +/// "error". Issue #196. +function firstMatchingCategory( + text: string, + candidates: ReadonlyArray<{ regex: RegExp; category: TaskCategory }>, +): TaskCategory | null { + let best: { index: number; order: number; category: TaskCategory } | null = null + for (let i = 0; i < candidates.length; i++) { + const c = candidates[i]! + const m = c.regex.exec(text) + if (!m) continue + if (!best || m.index < best.index || (m.index === best.index && i < best.order)) { + best = { index: m.index, order: i, category: c.category } + } + } + return best?.category ?? null +} + function refineByKeywords(category: TaskCategory, userMessage: string): TaskCategory { if (category === 'coding') { - if (DEBUG_KEYWORDS.test(userMessage)) return 'debugging' - if (REFACTOR_KEYWORDS.test(userMessage)) return 'refactoring' - if (FEATURE_KEYWORDS.test(userMessage)) return 'feature' - return 'coding' + // Tie-break order (when two keywords match at the same index): refactoring + // first because its words are the most specific, then feature, then debug. + return firstMatchingCategory(userMessage, [ + { regex: REFACTOR_KEYWORDS, category: 'refactoring' }, + { regex: FEATURE_KEYWORDS, category: 'feature' }, + { regex: DEBUG_KEYWORDS, category: 'debugging' }, + ]) ?? 'coding' } if (category === 'exploration') { @@ -113,8 +139,14 @@ function refineByKeywords(category: TaskCategory, userMessage: string): TaskCate function classifyConversation(userMessage: string): TaskCategory { if (BRAINSTORM_KEYWORDS.test(userMessage)) return 'brainstorming' if (RESEARCH_KEYWORDS.test(userMessage)) return 'exploration' - if (DEBUG_KEYWORDS.test(userMessage)) return 'debugging' - if (FEATURE_KEYWORDS.test(userMessage)) return 'feature' + // Same first-match-wins logic as refineByKeywords so a chat-only message + // starting with a feature verb does not flip to debugging because of an + // incidental "error" or "fix" word later in the same sentence. + const debugOrFeature = firstMatchingCategory(userMessage, [ + { regex: FEATURE_KEYWORDS, category: 'feature' }, + { regex: DEBUG_KEYWORDS, category: 'debugging' }, + ]) + if (debugOrFeature) return debugOrFeature if (FILE_PATTERNS.test(userMessage)) return 'coding' if (SCRIPT_PATTERNS.test(userMessage)) return 'coding' if (URL_PATTERN.test(userMessage)) return 'exploration' diff --git a/tests/classifier.test.ts b/tests/classifier.test.ts index ab322bb..6a02d64 100644 --- a/tests/classifier.test.ts +++ b/tests/classifier.test.ts @@ -101,3 +101,53 @@ describe('classifyTurn — Skill subCategory', () => { expect(c.subCategory).toBeUndefined() }) }) + +// Regression coverage for issue #196: feature verbs that lead a message +// were previously hijacked into 'debugging' just because the message contained +// an incidental "error" / "fix" / "issue" word later in the same sentence. +// Now whichever keyword pattern matches earliest wins. +describe('classifyTurn — feature vs debugging precedence (#196)', () => { + function codingTurn(userMessage: string): ParsedTurn { + return makeTurn([makeCall({ tools: ['Edit'] })], userMessage) + } + + it('classifies "add error handling" as feature, not debugging', () => { + const c = classifyTurn(codingTurn('add error handling to the auth module')) + expect(c.category).toBe('feature') + }) + + it('classifies "create an issue tracker" as feature, not debugging', () => { + const c = classifyTurn(codingTurn('create an issue tracker page in the dashboard')) + expect(c.category).toBe('feature') + }) + + it('classifies "implement the 404 page" as feature, not debugging', () => { + const c = classifyTurn(codingTurn('implement the 404 page with a friendly redirect')) + expect(c.category).toBe('feature') + }) + + it('still classifies "fix the layout for the new feature" as debugging', () => { + const c = classifyTurn(codingTurn('fix the layout for the new feature')) + expect(c.category).toBe('debugging') + }) + + it('still classifies a plain bug report as debugging', () => { + const c = classifyTurn(codingTurn('login is broken, traceback below')) + expect(c.category).toBe('debugging') + }) + + it('classifies "refactor the error handling" as refactoring', () => { + const c = classifyTurn(codingTurn('refactor the error handling so it is cleaner')) + expect(c.category).toBe('refactoring') + }) + + it('chat-only message starting with "add" stays feature even with "fix" later', () => { + const c = classifyTurn(makeTurn([], 'add a setting page; we will fix the styles after')) + expect(c.category).toBe('feature') + }) + + it('chat-only message starting with "fix" stays debugging even with "add" later', () => { + const c = classifyTurn(makeTurn([], 'fix the bug introduced when we added the new flag')) + expect(c.category).toBe('debugging') + }) +})