From b7b1d1a2c78baae9d51a326c9221ee2ffb60726e Mon Sep 17 00:00:00 2001 From: Lena <241371603+zelenenka@users.noreply.github.com> Date: Fri, 29 May 2026 15:28:35 +0200 Subject: [PATCH] Duplicate Bot: Reduce noise (#58074) Release Notes: - N/A --- .../github-check-new-issue-for-duplicates.py | 247 +++++++++++++++++- ...ithub-track-duplicate-bot-effectiveness.py | 38 ++- 2 files changed, 263 insertions(+), 22 deletions(-) diff --git a/script/github-check-new-issue-for-duplicates.py b/script/github-check-new-issue-for-duplicates.py index 023449bb27d..8a4eb8ec22a 100644 --- a/script/github-check-new-issue-for-duplicates.py +++ b/script/github-check-new-issue-for-duplicates.py @@ -146,11 +146,15 @@ No action needed. A maintainer will review this shortly. ] parts.append("**Possibly related open issues:**\n\n" + "\n".join(lines)) if related_closed_issues: + # state_reason is shown only for "duplicate" (the close type is otherwise + # already visible from GitHub's icon next to the issue number on render). lines = [ - f"- #{m['number']} (closed as {m['state_reason']}) — {m['explanation']}" + f"- #{m['number']}" + f"{' (closed as duplicate)' if m['state_reason'] == 'duplicate' else ''}" + f" — {m['explanation']}" for m in related_closed_issues ] - parts.append("**Recently closed, possibly related:**\n\n" + "\n".join(lines)) + parts.append("**Recently closed, possibly the same bug:**\n\n" + "\n".join(lines)) body = "\n\n".join(parts) sections.append(f"""
Additional recent context for triagers @@ -280,6 +284,12 @@ def detect_areas(anthropic_key, issue, area_labels): system_prompt = """You analyze GitHub issues to identify which area labels apply. +Decide the area from the user's stated symptom and reproduction steps. Issue bodies routinely +contain pasted log output, crash dumps, stack traces, settings files, and template headers like +"Attach Zed log file" or "Relevant Zed settings" — these are evidence about the symptom and +should not push you toward labels like "logging" or "settings" unless the bug itself is about +how that subsystem works. + Respond with ONLY a comma-separated list of matching area names. No prose, no explanation, no markdown, no preamble — just the names. @@ -500,8 +510,14 @@ def analyze_duplicates(anthropic_key, issue, magnets, search_results): return [], [], [] log("Analyzing candidates with Claude") + log(f" Candidate pool: {len(top_magnets)} magnets, {len(open_results)} open search results, " + f"{len(closed_results)} closed search results (will pass {min(len(closed_results), 5)} closed)") enrich_magnets(top_magnets) + closed_candidates_for_claude = closed_results[:5] + if closed_candidates_for_claude: + log(f" Closed candidates given to proposer: {[r['number'] for r in closed_candidates_for_claude]}") + candidates = [ {"number": m["number"], "title": m["title"], "body_preview": m["body_preview"], "state": "open", "state_reason": None, "source": "known_duplicate_magnet"} @@ -509,7 +525,7 @@ def analyze_duplicates(anthropic_key, issue, magnets, search_results): ] + [ {"number": r["number"], "title": r["title"], "body_preview": r["body_preview"], "state": r["state"], "state_reason": r["state_reason"], "source": "search_result"} - for r in open_results[:10] + closed_results[:5] + for r in open_results[:10] + closed_candidates_for_claude ] system_prompt = """You analyze GitHub issues to (a) identify duplicates among OPEN candidates @@ -548,17 +564,63 @@ Examples of things that are NOT duplicates: For OPEN duplicates (either bucket), false positives are MUCH worse than false negatives — they waste the time of both the issue author and the maintainers. When in doubt, omit. -# (b) Related closed issues — CLOSED candidates only +# (b) Closed candidates that may be the same bug — CLOSED candidates only -The goal is to give triagers extra context, NOT to claim a duplicate. The bar is lower than for -duplicates: include a closed candidate if a triager would plausibly want to see it when reviewing -the new issue. Examples worth surfacing: -- A recently fixed (state_reason "completed") issue describing the same symptom — triager may ask - the reporter to retest on the latest build. -- A cluster of similar issues closed as "not_planned" — signals a known limitation or design choice. -- A previously triaged duplicate (state_reason "duplicate") in the same code area. +The goal is NOT a "related reading" list. The goal is to surface closed issues where the +new issue is plausibly the SAME bug — a duplicate that just happens to be filed against a +closed predecessor instead of an open one. Empty is preferable to weak filler — triagers +lose trust in this section quickly if it's stretched. The same false-positives-are-worse +asymmetry as for duplicates applies here. -Include at most 5 closed candidates, prioritized by relevance. +The bar: a triager reading this should be able to act — ask the reporter to retest a fix, +point at a known design decision that already declined this request, or point at the +canonical bug this is a duplicate of. "Useful context" or "shared area" is NOT a reason +to include. + +Omit a candidate if ANY of these apply (in observed practice, almost everything does): + +1. Self-contradiction. If you find yourself writing "while focused on X rather than Y", + "although this is about A, the new issue is about B", "this issue focuses on... rather + than...", or any acknowledgment that the candidate isn't on the same topic — STOP. + You've already made the case for omitting it. + +2. Fabricated specifics. Every concrete claim about the candidate (its trigger, its scope, + its conditions) must be visible in the candidate's title or body preview. Specifics + like "when X happens", "under Y conditions", "specifically affecting Z" that aren't + supported by the candidate's actual text mean you're inventing details to fit the new + issue. Omit. + +3. Weasel phrases. Paraphrases of these all indicate you don't have a real claim: + "may indicate similar...", "could provide context for...", "shows / demonstrates recent + attention to...", "indicates the team has considered...", "demonstrates a pattern + of...", "may provide useful context...". STOP and omit. + +4. Retest by default. The "reporter may need to retest on the latest build" framing only + applies when the candidate's symptom is literally the same as the new issue's. It is + NOT a default justification for "this was a recent fix in roughly the same area." + +5. Same area / feature, different mechanism. Examples to omit: + - "ARM compile failure" alongside "ARM runtime perf" — same area, different mechanism. + - "Worktree path bug" alongside "worktree display label confusion" — same feature, + unrelated. + +6. Vague catch-all candidate. A closed issue like "Zed is slow" / "performance" / "agent + panel UX" that could be cited next to almost any new bug is filler. If you'd reuse the + same closed issue across many unrelated new issues, omit. + +7. Label or single-keyword overlap. A closed issue whose only connection is a shared + area:* label or one shared keyword is not relevant. + +Worth surfacing — strict examples: +- A recently fixed ("completed") issue with the SAME specific trigger as the new issue — + triager can ask the reporter to retest on the latest build. +- A cluster of "not_planned" closures about the EXACT same request — known design choice + the triager can point to. +- A previously triaged "duplicate" pointing at the same canonical issue, or sharing the + same specific mechanism. + +Count: typically 0 or 1. Never more than 2 unless there's an obvious cluster of identical +"not_planned" reports. 0 is a normal outcome. # Output format @@ -614,10 +676,164 @@ Return empty arrays where nothing relevant is found.""" likely = data.get("likely_duplicates", []) possible = data.get("possible_duplicates", []) closed = data.get("related_closed_issues", []) + + # Claude occasionally places a closed candidate in the duplicate buckets, or vice + # versa. Enforce that each match lives in the bucket consistent with the canonical + # state of the candidate we passed in. + candidate_states = {c["number"]: c["state"] for c in candidates} + + def filter_by_state(items, expected_state, label): + kept, wrong = [], [] + for m in items: + (kept if candidate_states.get(m["number"]) == expected_state else wrong).append(m) + if wrong: + log(f" Dropped {len(wrong)} from {label} with wrong/unknown state: {[m['number'] for m in wrong]}") + return kept + + likely = filter_by_state(likely, "open", "likely_duplicates") + possible = filter_by_state(possible, "open", "possible_duplicates") + closed = filter_by_state(closed, "closed", "related_closed_issues") + + # Avoid showing the same issue in both the user-facing alert and the triage section. + likely_numbers = {m["number"] for m in likely} + overlap = [m["number"] for m in possible if m["number"] in likely_numbers] + if overlap: + log(f" Dropped {len(overlap)} from possible_duplicates already in likely_duplicates: {overlap}") + possible = [m for m in possible if m["number"] not in likely_numbers] + log(f" Found {len(likely) + len(possible) + len(closed)} potential matches") return likely, possible, closed +CRITIQUE_SYSTEM_PROMPT = """You are evaluating ONE recently closed GitHub issue to decide whether a triager looking +at a brand-new bug report would find it useful to be told about that closed issue. + +There is no slate to fill. There is no quota. You will be shown exactly one candidate. +The default verdict is OMIT. Zero is the expected outcome for most candidates. + +A candidate is worth surfacing ONLY if the new issue is plausibly the SAME BUG as the +closed one — a duplicate that happens to be filed against a closed predecessor. Concretely, +the legitimate cases are exactly three: + +- The candidate was closed as "completed" (a fix shipped) AND the new issue has the same + specific trigger / symptom. The triager will ask the reporter to retest. +- The candidate was closed as "not_planned" AND the new issue is the EXACT same request + (a feature decision the team already declined). The triager will point at it. +- The candidate was closed as "duplicate" AND it pointed at the same canonical bug the new + issue describes, or it shares the same specific mechanism. + +"Same broad area", "similar-sounding symptom", or "recent attention to this subsystem" are +NOT reasons to include. Omit them. + +Return "omit" if ANY of the following apply (in observed practice, almost everything does): + +1. Self-contradiction. If your reasoning includes "while focused on X rather than Y", + "although this is about A, the new issue is about B", "this issue focuses on... rather + than...", or any acknowledgment the candidate is on a different topic — you've already + decided to omit. +2. Fabricated specifics. Every concrete claim about the candidate (its trigger, scope, + conditions) must be visible in the candidate's title or body preview. If you find + yourself describing the candidate using details that aren't in its text, you're + inventing details to fit the new issue. Omit. +3. Weasel phrases. Paraphrases of "may indicate similar...", "could provide context + for...", "shows / demonstrates recent attention to...", "indicates the team has + considered...", "demonstrates a pattern of...", "may provide useful context..." — + these mean you don't have a real claim. Omit. +4. Retest by default. The "reporter may need to retest on the latest build" framing only + applies when the closed issue's symptom is LITERALLY the same as the new issue's. "This + was a recent fix in roughly the same area" is not enough. +5. Same area / feature, different mechanism. Same area label but different bug, different + code path, different trigger. Omit. +6. Vague catch-all candidate. A closed issue like "Zed is slow" / "performance" / "agent + panel UX" that you could cite next to many unrelated new bugs. Omit. +7. Label or single-keyword overlap. Only connection is a shared area:* label or one shared + keyword. Omit. + +Output only valid JSON (no markdown code blocks): +{ + "verdict": "include" | "omit", + "rule_violated": null | 1 | 2 | 3 | 4 | 5 | 6 | 7, + "rationale": "one concise sentence explaining the verdict" +} + +When "verdict" is "include", "rule_violated" must be null. +When "verdict" is "omit", "rule_violated" should be the most relevant rule number, or null +if the candidate is simply too unrelated for any rule to specifically apply.""" + + +def critique_closed_candidates(anthropic_key, issue, proposed, search_results): + """Run a strict per-candidate critique pass over the proposer's closed candidates. + + For each proposed match, call Claude with only the new issue and that single candidate + (blind to the proposer's rationale) and ask for a yes/no verdict. Default is omit. + Returns the subset of `proposed` that passes critique. + """ + if not proposed: + log(" Critique: proposer surfaced 0 closed candidates; skipping") + return [] + + log(f" Critique: proposer surfaced {len(proposed)} closed candidate(s): " + f"{[m['number'] for m in proposed]}") + + results_by_number = {r["number"]: r for r in search_results} + kept = [] + for match in proposed: + number = match["number"] + candidate = results_by_number.get(number) + if candidate is None: + # Should not happen — analyze_duplicates only emits numbers from candidates it + # was given — but be defensive rather than crash the bot. + log(f" Critique: dropping #{number} — candidate context not found") + continue + + state_reason = candidate.get("state_reason") or "unknown" + user_content = f"""## New Issue #{issue['number']} +**Title:** {issue['title']} + +**Body:** +{issue['body'][:3000]} + +## Closed Candidate #{number} +**Title:** {candidate.get('title', '')} +**State reason:** {state_reason} + +**Body preview:** +{candidate.get('body_preview', '')}""" + + log(f" Critique: evaluating #{number}") + try: + response = call_claude(anthropic_key, CRITIQUE_SYSTEM_PROMPT, user_content, max_tokens=300) + except requests.RequestException as e: + # If the critique call fails, prefer omitting the candidate over posting noise. + log(f" Critique: API call failed for #{number} ({e}); omitting candidate") + continue + + fence = re.match(r"^\s*```(?:json)?\s*\n?(.*?)\n?```\s*$", response, re.DOTALL) + if fence: + response = fence.group(1) + + try: + verdict_data = json.loads(response) + except json.JSONDecodeError as e: + log(f" Critique: failed to parse verdict for #{number} ({e}); omitting candidate") + log(f" Raw response: {response}") + continue + + verdict = verdict_data.get("verdict") + rule = verdict_data.get("rule_violated") + rationale = verdict_data.get("rationale", "") + + if verdict == "include": + log(f" Critique: keeping #{number} — {rationale}") + kept.append(match) + else: + rule_str = f"rule {rule}" if rule else "no specific rule" + log(f" Critique: omitting #{number} ({rule_str}) — {rationale}") + + log(f" Critique: kept {len(kept)} of {len(proposed)} closed candidates") + return kept + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Identify potential duplicate issues") parser.add_argument("issue_number", type=int, help="Issue number to analyze") @@ -658,6 +874,13 @@ if __name__ == "__main__": anthropic_key, issue, relevant_magnets, search_results ) + # second-pass critique: prompt iteration on the proposer hit a ceiling around 30% noise. + # Re-evaluate each proposed closed candidate in isolation with a stricter prompt that + # has no slate to fill and is blind to the proposer's rationale. + related_closed_issues = critique_closed_candidates( + anthropic_key, issue, related_closed_issues, search_results + ) + # resolve close reason from our search results (the source of truth) so we don't depend # on Claude to faithfully echo it back results_by_number = {r["number"]: r for r in search_results} diff --git a/script/github-track-duplicate-bot-effectiveness.py b/script/github-track-duplicate-bot-effectiveness.py index 06dcae67cd8..02be1b9e137 100644 --- a/script/github-track-duplicate-bot-effectiveness.py +++ b/script/github-track-duplicate-bot-effectiveness.py @@ -24,6 +24,7 @@ import functools import os import re import sys +import time from datetime import datetime, timezone import requests @@ -47,6 +48,8 @@ BOT_START_DATE = "2026-02-18" NEEDS_TRIAGE_LABEL = "state:needs triage" DEFAULT_PROJECT_NUMBER = 76 VALID_CLOSED_AS_VALUES = {"duplicate", "not_planned", "completed"} +# HTTP statuses we'll retry on for GET requests +TRANSIENT_HTTP_STATUSES = {429, 500, 502, 503, 504} # Add a new tuple when you deploy a new version of the bot that you want to # keep track of (e.g. the prompt gets a rewrite or the model gets swapped). # Newest first, please. The datetime is for the deployment time (merge to main). @@ -67,10 +70,22 @@ def bot_version_for_time(date_string): def github_api_get(path, params=None): + """Fetch JSON from the GitHub REST API, retrying transient failures. Raises on non-2xx status.""" url = f"{GITHUB_API}/{path.lstrip('/')}" - response = requests.get(url, headers=GITHUB_HEADERS, params=params) - response.raise_for_status() - return response.json() + for attempt in range(3): + try: + response = requests.get(url, headers=GITHUB_HEADERS, params=params) + response.raise_for_status() + return response.json() + except requests.RequestException as e: + transient = isinstance(e, (requests.ConnectionError, requests.Timeout)) or ( + isinstance(e, requests.HTTPError) and e.response.status_code in TRANSIENT_HTTP_STATUSES + ) + if not transient or attempt == 2: + raise + wait = 2 ** attempt + print(f" Transient GitHub API error ({e}); retrying in {wait}s") + time.sleep(wait) def github_search_issues(query): @@ -161,9 +176,11 @@ def find_canonical_among(duplicate_number, candidates): if not candidates: return None + # candidate issue numbers are baked into the query body via field aliases + # (GraphQL doesn't let you parametrize alias names), so $numbers isn't needed. data = github_api_graphql( """ - query($owner: String!, $repo: String!, $numbers: [Int!]!) { + query($owner: String!, $repo: String!) { repository(owner: $owner, name: $repo) { PLACEHOLDER } @@ -174,7 +191,7 @@ def find_canonical_among(duplicate_number, candidates): f' nodes {{ ... on MarkedAsDuplicateEvent {{ duplicate {{ ... on Issue {{ number }} }} }} }} }} }}' for number in candidates )), - {"owner": REPO_OWNER, "repo": REPO_NAME, "numbers": list(candidates)}, + {"owner": REPO_OWNER, "repo": REPO_NAME}, partial_errors_ok=True, ) @@ -409,11 +426,10 @@ def classify_as_assist(issue, bot_comment): bot_comment_time=bot_comment["created_at"]) return - original = None - try: - original = find_canonical_among(issue["number"], suggested) - except (requests.RequestException, RuntimeError) as error: - print(f" Warning: failed to query candidate timelines: {error}") + # Let exceptions from find_canonical_among propagate — a query failure here is + # not the same as "no canonical match" and shouldn't be silently downgraded to + # a Needs review entry. Failing the workflow surfaces the problem immediately. + original = find_canonical_among(issue["number"], suggested) if original: status = "Auto-classified" @@ -483,6 +499,8 @@ def classify_open(): errors += 1 print(f" Done: added {added}, skipped {skipped}, errors {errors}") + if errors > 0: + sys.exit(1) if __name__ == "__main__":