From e5966915e461dc9ec7c388b4dbdd19a8dbd62d4e Mon Sep 17 00:00:00 2001 From: Lena <241371603+zelenenka@users.noreply.github.com> Date: Tue, 16 Jun 2026 17:17:51 +0200 Subject: [PATCH] Duplicate Bot: Switch to structured tool output for Claude (#59432) Rather than continuing to try to persuade Claude to pretty please only output json, or to parse json out of Claude's prose, switch to forced tool calls with a json schema and hope this works better. And bump the version of the bot to 4 since the model upgrade which was shipped earlier today is evidently behaving differently enough. Release Notes: - N/A --- .../github-check-new-issue-for-duplicates.py | 184 +++++++++++------- ...ithub-track-duplicate-bot-effectiveness.py | 1 + 2 files changed, 115 insertions(+), 70 deletions(-) diff --git a/script/github-check-new-issue-for-duplicates.py b/script/github-check-new-issue-for-duplicates.py index b03f429b7f5..2589425d709 100644 --- a/script/github-check-new-issue-for-duplicates.py +++ b/script/github-check-new-issue-for-duplicates.py @@ -40,6 +40,7 @@ REPO_OWNER = "zed-industries" REPO_NAME = "zed" TRACKING_ISSUE_NUMBER = 46355 STAFF_TEAM_SLUG = "staff" +CLAUDE_MODEL = "claude-sonnet-4-6" # area prefixes to collapse in taxonomy (show summary instead of all sub-labels) PREFIXES_TO_COLLAPSE = ["languages", "parity", "tooling"] @@ -170,8 +171,8 @@ No action needed. A maintainer will review this shortly. return "\n\n".join(sections) -def call_claude(api_key, system, user_content, max_tokens=1024): - """Send a message to Claude and return the text response. Raises on non-2xx status.""" +def _claude_request(api_key, payload): + """POST to the Claude Messages API, raise on non-2xx, log token usage, return parsed data.""" response = requests.post( "https://api.anthropic.com/v1/messages", headers={ @@ -179,19 +180,23 @@ def call_claude(api_key, system, user_content, max_tokens=1024): "anthropic-version": "2023-06-01", "content-type": "application/json", }, - json={ - "model": "claude-sonnet-4-6", - "max_tokens": max_tokens, - "temperature": 0.0, - "system": system, - "messages": [{"role": "user", "content": user_content}], - }, + json={"model": CLAUDE_MODEL, "temperature": 0.0, **payload}, ) response.raise_for_status() data = response.json() usage = data.get("usage", {}) log(f" Token usage - Input: {usage.get('input_tokens', 'N/A')}, Output: {usage.get('output_tokens', 'N/A')}") + return data + + +def call_claude(api_key, system_prompt, user_content, max_tokens=1024): + """Send a message to Claude and return the text response. Raises on non-2xx status.""" + data = _claude_request(api_key, { + "max_tokens": max_tokens, + "system": system_prompt, + "messages": [{"role": "user", "content": user_content}], + }) content = data.get("content", []) if content and content[0].get("type") == "text": @@ -199,6 +204,30 @@ def call_claude(api_key, system, user_content, max_tokens=1024): return "" +def call_claude_tool(api_key, system_prompt, user_content, tool, max_tokens=1024): + """Call Claude, forcing it to invoke `tool`, and return the structured input dict. + + Forcing a tool call makes the API emit schema-shaped JSON via its tool-use mechanism + instead of free-form text we'd have to parse out of prose or markdown fences. Raises on + non-2xx status, or if no tool_use block is returned. + """ + data = _claude_request(api_key, { + "max_tokens": max_tokens, + "system": system_prompt, + "messages": [{"role": "user", "content": user_content}], + "tools": [tool], + "tool_choice": {"type": "tool", "name": tool["name"]}, + }) + + if data.get("stop_reason") == "max_tokens": + log(" Warning: response hit max_tokens; structured output may be truncated") + + for block in data.get("content", []): + if block.get("type") == "tool_use": + return block.get("input") or {} + raise ValueError(f"Claude returned no tool_use block for tool '{tool['name']}'") + + def fetch_issue(issue_number: int): """Fetch issue from GitHub and return as a dict.""" log(f"Fetching issue #{issue_number}") @@ -622,33 +651,12 @@ Worth surfacing — strict examples: Count: typically 0 or 1. Never more than 2 unless there's an obvious cluster of identical "not_planned" reports. 0 is a normal outcome. -# Output format +# Output -Output only valid JSON (no markdown code blocks): -{ - "likely_duplicates": [ - { - "number": 12345, - "shared_root_cause": "The specific bug/root cause shared by both issues", - "explanation": "Brief explanation with concrete evidence from both issues" - } - ], - "possible_duplicates": [ - { - "number": 12345, - "shared_root_cause": "The specific bug/root cause shared by both issues", - "explanation": "Brief explanation with concrete evidence from both issues" - } - ], - "related_closed_issues": [ - { - "number": 12345, - "explanation": "Brief explanation of why this is useful triage context" - } - ] -} - -Return empty arrays where nothing relevant is found.""" +Report your verdict by calling the report_duplicate_analysis tool. Fill the "reasoning" +field first with a brief scratchpad weighing the strongest candidates and whether they +share a root cause, then fill each bucket. Use empty arrays where nothing relevant is +found.""" user_content = f"""## New Issue #{issue['number']} **Title:** {issue['title']} @@ -659,19 +667,48 @@ Return empty arrays where nothing relevant is found.""" ## Existing Issues to Compare {json.dumps(candidates, indent=2)}""" - response = call_claude(anthropic_key, system_prompt, user_content, max_tokens=2048) + duplicate_match_schema = { + "type": "object", + "properties": { + "number": {"type": "integer", "description": "The candidate issue number"}, + "shared_root_cause": {"type": "string", "description": "The specific bug/root cause shared by both issues"}, + "explanation": {"type": "string", "description": "Brief explanation with concrete evidence from both issues"}, + }, + "required": ["number", "shared_root_cause", "explanation"], + } + analysis_tool = { + "name": "report_duplicate_analysis", + "description": "Report the duplicate analysis for the new issue.", + "input_schema": { + "type": "object", + "properties": { + "reasoning": { + "type": "string", + "description": "A brief scratchpad (at most 2-3 sentences) weighing the strongest " + "candidates and whether they share a root cause. Be terse.", + "maxLength": 700, + }, + "likely_duplicates": {"type": "array", "items": duplicate_match_schema}, + "possible_duplicates": {"type": "array", "items": duplicate_match_schema}, + "related_closed_issues": { + "type": "array", + "items": { + "type": "object", + "properties": { + "number": {"type": "integer", "description": "The candidate issue number"}, + "explanation": {"type": "string", "description": "Brief explanation of why this is useful triage context"}, + }, + "required": ["number", "explanation"], + }, + }, + }, + "required": ["reasoning", "likely_duplicates", "possible_duplicates", "related_closed_issues"], + }, + } - # Claude sometimes wraps JSON in a ```json ... ``` fence despite the prompt forbidding it - fence = re.match(r"^\s*```(?:json)?\s*\n?(.*?)\n?```\s*$", response, re.DOTALL) - if fence: - response = fence.group(1) - - try: - data = json.loads(response) - except json.JSONDecodeError as e: - log(f" Failed to parse Claude response as JSON: {e}") - log(f" Raw response:\n{response}") - sys.exit(1) + data = call_claude_tool(anthropic_key, system_prompt, user_content, analysis_tool, max_tokens=3072) + if data.get("reasoning"): + log(f" Reasoning: {data['reasoning']}") likely = data.get("likely_duplicates", []) possible = data.get("possible_duplicates", []) @@ -749,16 +786,32 @@ Return "omit" if ANY of the following apply (in observed practice, almost everyt 7. Label or single-keyword overlap. Only connection is a shared area:* label or one shared keyword. Omit. -Output only valid JSON (no markdown code blocks): -{ - "verdict": "include" | "omit", - "rule_violated": null | 1 | 2 | 3 | 4 | 5 | 6 | 7, - "rationale": "one concise sentence explaining the verdict" -} +Report your decision by calling the report_critique_verdict tool. Fill "rationale" first +(one concise sentence), then "verdict". When "verdict" is "include", "rule_violated" must be +null. When "verdict" is "omit", set "rule_violated" to the most relevant rule number, or +null if the candidate is simply too unrelated for any rule to specifically apply.""" -When "verdict" is "include", "rule_violated" must be null. -When "verdict" is "omit", "rule_violated" should be the most relevant rule number, or null -if the candidate is simply too unrelated for any rule to specifically apply.""" + +CRITIQUE_VERDICT_TOOL = { + "name": "report_critique_verdict", + "description": "Report whether the closed candidate is worth surfacing to a triager.", + "input_schema": { + "type": "object", + "properties": { + "rationale": { + "type": "string", + "description": "One concise sentence justifying the verdict, grounded in the candidate's actual text.", + "maxLength": 400, + }, + "verdict": {"type": "string", "enum": ["include", "omit"]}, + "rule_violated": { + "type": ["integer", "null"], + "description": "The most relevant omit-rule number (1-7), or null when including.", + }, + }, + "required": ["rationale", "verdict"], + }, +} def critique_closed_candidates(anthropic_key, issue, proposed, search_results): @@ -802,21 +855,12 @@ def critique_closed_candidates(anthropic_key, issue, proposed, search_results): log(f" Critique: evaluating #{number}") try: - response = call_claude(anthropic_key, CRITIQUE_SYSTEM_PROMPT, user_content, max_tokens=300) - except requests.RequestException as e: + verdict_data = call_claude_tool( + anthropic_key, CRITIQUE_SYSTEM_PROMPT, user_content, CRITIQUE_VERDICT_TOOL, max_tokens=600 + ) + except (requests.RequestException, ValueError) as e: # If the critique call fails, prefer omitting the candidate over posting noise. - log(f" Critique: API call failed for #{number} ({e}); omitting candidate") - continue - - fence = re.match(r"^\s*```(?:json)?\s*\n?(.*?)\n?```\s*$", response, re.DOTALL) - if fence: - response = fence.group(1) - - try: - verdict_data = json.loads(response) - except json.JSONDecodeError as e: - log(f" Critique: failed to parse verdict for #{number} ({e}); omitting candidate") - log(f" Raw response: {response}") + log(f" Critique: verdict call failed for #{number} ({e}); omitting candidate") continue verdict = verdict_data.get("verdict") diff --git a/script/github-track-duplicate-bot-effectiveness.py b/script/github-track-duplicate-bot-effectiveness.py index 02be1b9e137..ee435c43e32 100644 --- a/script/github-track-duplicate-bot-effectiveness.py +++ b/script/github-track-duplicate-bot-effectiveness.py @@ -54,6 +54,7 @@ TRANSIENT_HTTP_STATUSES = {429, 500, 502, 503, 504} # keep track of (e.g. the prompt gets a rewrite or the model gets swapped). # Newest first, please. The datetime is for the deployment time (merge to main). BOT_VERSION_TIMELINE = [ + ("v4", datetime(2026, 6, 16, 12, 43, tzinfo=timezone.utc)), ("v3", datetime(2026, 5, 25, 14, 30, tzinfo=timezone.utc)), ("v2", datetime(2026, 2, 26, 14, 9, tzinfo=timezone.utc)), ("v1", datetime(2026, 2, 18, tzinfo=timezone.utc)),