vrr/zed
2
0
Fork 0
mirror of https://github.com/zed-industries/zed.git synced 2026-07-09 16:00:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 16

Duplicate Bot: Reduce noise (#58074)

Browse source 
Release Notes:

- N/A
This commit is contained in:
Lena 2026-05-29 15:28:35 +02:00 • committed by GitHub
parent a1d2ef6514
commit b7b1d1a2c7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 263 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

247
script/github-check-new-issue-for-duplicates.py
View file

@ -146,11 +146,15 @@ No action needed. A maintainer will review this shortly.
]
parts.append("**Possibly related open issues:**\n\n" + "\n".join(lines))
if related_closed_issues:
# state_reason is shown only for "duplicate" (the close type is otherwise
# already visible from GitHub's icon next to the issue number on render).
lines = [
f"- #{m['number']} (closed as {m['state_reason']}) — {m['explanation']}"
f"- #{m['number']}"
f"{' (closed as duplicate)' if m['state_reason'] == 'duplicate' else ''}"
f" — {m['explanation']}"
for m in related_closed_issues
]
parts.append("**Recently closed, possibly related:**\n\n" + "\n".join(lines))
parts.append("**Recently closed, possibly the same bug:**\n\n" + "\n".join(lines))
body = "\n\n".join(parts)
sections.append(f"""<details>
<summary>Additional recent context for triagers</summary>
@ -280,6 +284,12 @@ def detect_areas(anthropic_key, issue, area_labels):
system_prompt = """You analyze GitHub issues to identify which area labels apply.
Decide the area from the user's stated symptom and reproduction steps. Issue bodies routinely
contain pasted log output, crash dumps, stack traces, settings files, and template headers like
"Attach Zed log file" or "Relevant Zed settings" — these are evidence about the symptom and
should not push you toward labels like "logging" or "settings" unless the bug itself is about
how that subsystem works.
Respond with ONLY a comma-separated list of matching area names. No prose, no explanation,
no markdown, no preamble — just the names.
@ -500,8 +510,14 @@ def analyze_duplicates(anthropic_key, issue, magnets, search_results):
return [], [], []
log("Analyzing candidates with Claude")
log(f" Candidate pool: {len(top_magnets)} magnets, {len(open_results)} open search results, "
f"{len(closed_results)} closed search results (will pass {min(len(closed_results), 5)} closed)")
enrich_magnets(top_magnets)
closed_candidates_for_claude = closed_results[:5]
if closed_candidates_for_claude:
log(f" Closed candidates given to proposer: {[r['number'] for r in closed_candidates_for_claude]}")
candidates = [
{"number": m["number"], "title": m["title"], "body_preview": m["body_preview"],
"state": "open", "state_reason": None, "source": "known_duplicate_magnet"}
@ -509,7 +525,7 @@ def analyze_duplicates(anthropic_key, issue, magnets, search_results):
] + [
{"number": r["number"], "title": r["title"], "body_preview": r["body_preview"],
"state": r["state"], "state_reason": r["state_reason"], "source": "search_result"}
for r in open_results[:10] + closed_results[:5]
for r in open_results[:10] + closed_candidates_for_claude
]
system_prompt = """You analyze GitHub issues to (a) identify duplicates among OPEN candidates
@ -548,17 +564,63 @@ Examples of things that are NOT duplicates:
For OPEN duplicates (either bucket), false positives are MUCH worse than false negatives — they
waste the time of both the issue author and the maintainers. When in doubt, omit.
# (b) Related closed issues — CLOSED candidates only
# (b) Closed candidates that may be the same bug — CLOSED candidates only
The goal is to give triagers extra context, NOT to claim a duplicate. The bar is lower than for
duplicates: include a closed candidate if a triager would plausibly want to see it when reviewing
the new issue. Examples worth surfacing:
- A recently fixed (state_reason "completed") issue describing the same symptom — triager may ask
the reporter to retest on the latest build.
- A cluster of similar issues closed as "not_planned" — signals a known limitation or design choice.
- A previously triaged duplicate (state_reason "duplicate") in the same code area.
The goal is NOT a "related reading" list. The goal is to surface closed issues where the
new issue is plausibly the SAME bug — a duplicate that just happens to be filed against a
closed predecessor instead of an open one. Empty is preferable to weak filler — triagers
lose trust in this section quickly if it's stretched. The same false-positives-are-worse
asymmetry as for duplicates applies here.
Include at most 5 closed candidates, prioritized by relevance.
The bar: a triager reading this should be able to act — ask the reporter to retest a fix,
point at a known design decision that already declined this request, or point at the
canonical bug this is a duplicate of. "Useful context" or "shared area" is NOT a reason
to include.
Omit a candidate if ANY of these apply (in observed practice, almost everything does):
1. Self-contradiction. If you find yourself writing "while focused on X rather than Y",
"although this is about A, the new issue is about B", "this issue focuses on... rather
than...", or any acknowledgment that the candidate isn't on the same topic — STOP.
You've already made the case for omitting it.
2. Fabricated specifics. Every concrete claim about the candidate (its trigger, its scope,
its conditions) must be visible in the candidate's title or body preview. Specifics
like "when X happens", "under Y conditions", "specifically affecting Z" that aren't
supported by the candidate's actual text mean you're inventing details to fit the new
issue. Omit.
3. Weasel phrases. Paraphrases of these all indicate you don't have a real claim:
"may indicate similar...", "could provide context for...", "shows / demonstrates recent
attention to...", "indicates the team has considered...", "demonstrates a pattern
of...", "may provide useful context...". STOP and omit.
4. Retest by default. The "reporter may need to retest on the latest build" framing only
applies when the candidate's symptom is literally the same as the new issue's. It is
NOT a default justification for "this was a recent fix in roughly the same area."
5. Same area / feature, different mechanism. Examples to omit:
- "ARM compile failure" alongside "ARM runtime perf" — same area, different mechanism.
- "Worktree path bug" alongside "worktree display label confusion" — same feature,
unrelated.
6. Vague catch-all candidate. A closed issue like "Zed is slow" / "performance" / "agent
panel UX" that could be cited next to almost any new bug is filler. If you'd reuse the
same closed issue across many unrelated new issues, omit.
7. Label or single-keyword overlap. A closed issue whose only connection is a shared
area:* label or one shared keyword is not relevant.
Worth surfacing — strict examples:
- A recently fixed ("completed") issue with the SAME specific trigger as the new issue —
triager can ask the reporter to retest on the latest build.
- A cluster of "not_planned" closures about the EXACT same request — known design choice
the triager can point to.
- A previously triaged "duplicate" pointing at the same canonical issue, or sharing the
same specific mechanism.
Count: typically 0 or 1. Never more than 2 unless there's an obvious cluster of identical
"not_planned" reports. 0 is a normal outcome.
# Output format
@ -614,10 +676,164 @@ Return empty arrays where nothing relevant is found."""
likely = data.get("likely_duplicates", [])
possible = data.get("possible_duplicates", [])
closed = data.get("related_closed_issues", [])
# Claude occasionally places a closed candidate in the duplicate buckets, or vice
# versa. Enforce that each match lives in the bucket consistent with the canonical
# state of the candidate we passed in.
candidate_states = {c["number"]: c["state"] for c in candidates}
def filter_by_state(items, expected_state, label):
kept, wrong = [], []
for m in items:
(kept if candidate_states.get(m["number"]) == expected_state else wrong).append(m)
if wrong:
log(f" Dropped {len(wrong)} from {label} with wrong/unknown state: {[m['number'] for m in wrong]}")
return kept
likely = filter_by_state(likely, "open", "likely_duplicates")
possible = filter_by_state(possible, "open", "possible_duplicates")
closed = filter_by_state(closed, "closed", "related_closed_issues")
# Avoid showing the same issue in both the user-facing alert and the triage section.
likely_numbers = {m["number"] for m in likely}
overlap = [m["number"] for m in possible if m["number"] in likely_numbers]
if overlap:
log(f" Dropped {len(overlap)} from possible_duplicates already in likely_duplicates: {overlap}")
possible = [m for m in possible if m["number"] not in likely_numbers]
log(f" Found {len(likely) + len(possible) + len(closed)} potential matches")
return likely, possible, closed
CRITIQUE_SYSTEM_PROMPT = """You are evaluating ONE recently closed GitHub issue to decide whether a triager looking
at a brand-new bug report would find it useful to be told about that closed issue.
There is no slate to fill. There is no quota. You will be shown exactly one candidate.
The default verdict is OMIT. Zero is the expected outcome for most candidates.
A candidate is worth surfacing ONLY if the new issue is plausibly the SAME BUG as the
closed one — a duplicate that happens to be filed against a closed predecessor. Concretely,
the legitimate cases are exactly three:
- The candidate was closed as "completed" (a fix shipped) AND the new issue has the same
specific trigger / symptom. The triager will ask the reporter to retest.
- The candidate was closed as "not_planned" AND the new issue is the EXACT same request
(a feature decision the team already declined). The triager will point at it.
- The candidate was closed as "duplicate" AND it pointed at the same canonical bug the new
issue describes, or it shares the same specific mechanism.
"Same broad area", "similar-sounding symptom", or "recent attention to this subsystem" are
NOT reasons to include. Omit them.
Return "omit" if ANY of the following apply (in observed practice, almost everything does):
1. Self-contradiction. If your reasoning includes "while focused on X rather than Y",
"although this is about A, the new issue is about B", "this issue focuses on... rather
than...", or any acknowledgment the candidate is on a different topic — you've already
decided to omit.
2. Fabricated specifics. Every concrete claim about the candidate (its trigger, scope,
conditions) must be visible in the candidate's title or body preview. If you find
yourself describing the candidate using details that aren't in its text, you're
inventing details to fit the new issue. Omit.
3. Weasel phrases. Paraphrases of "may indicate similar...", "could provide context
for...", "shows / demonstrates recent attention to...", "indicates the team has
considered...", "demonstrates a pattern of...", "may provide useful context..." —
these mean you don't have a real claim. Omit.
4. Retest by default. The "reporter may need to retest on the latest build" framing only
applies when the closed issue's symptom is LITERALLY the same as the new issue's. "This
was a recent fix in roughly the same area" is not enough.
5. Same area / feature, different mechanism. Same area label but different bug, different
code path, different trigger. Omit.
6. Vague catch-all candidate. A closed issue like "Zed is slow" / "performance" / "agent
panel UX" that you could cite next to many unrelated new bugs. Omit.
7. Label or single-keyword overlap. Only connection is a shared area:* label or one shared
keyword. Omit.
Output only valid JSON (no markdown code blocks):
{
"verdict": "include" | "omit",
"rule_violated": null | 1 | 2 | 3 | 4 | 5 | 6 | 7,
"rationale": "one concise sentence explaining the verdict"
}
When "verdict" is "include", "rule_violated" must be null.
When "verdict" is "omit", "rule_violated" should be the most relevant rule number, or null
if the candidate is simply too unrelated for any rule to specifically apply."""
def critique_closed_candidates(anthropic_key, issue, proposed, search_results):
"""Run a strict per-candidate critique pass over the proposer's closed candidates.
For each proposed match, call Claude with only the new issue and that single candidate
(blind to the proposer's rationale) and ask for a yes/no verdict. Default is omit.
Returns the subset of `proposed` that passes critique.
"""
if not proposed:
log(" Critique: proposer surfaced 0 closed candidates; skipping")
return []
log(f" Critique: proposer surfaced {len(proposed)} closed candidate(s): "
f"{[m['number'] for m in proposed]}")
results_by_number = {r["number"]: r for r in search_results}
kept = []
for match in proposed:
number = match["number"]
candidate = results_by_number.get(number)
if candidate is None:
# Should not happen — analyze_duplicates only emits numbers from candidates it
# was given — but be defensive rather than crash the bot.
log(f" Critique: dropping #{number} — candidate context not found")
continue
state_reason = candidate.get("state_reason") or "unknown"
user_content = f"""## New Issue #{issue['number']}
**Title:** {issue['title']}
**Body:**
{issue['body'][:3000]}
## Closed Candidate #{number}
**Title:** {candidate.get('title', '')}
**State reason:** {state_reason}
**Body preview:**
{candidate.get('body_preview', '')}"""
log(f" Critique: evaluating #{number}")
try:
response = call_claude(anthropic_key, CRITIQUE_SYSTEM_PROMPT, user_content, max_tokens=300)
except requests.RequestException as e:
# If the critique call fails, prefer omitting the candidate over posting noise.
log(f" Critique: API call failed for #{number} ({e}); omitting candidate")
continue
fence = re.match(r"^\s*```(?:json)?\s*\n?(.*?)\n?```\s*$", response, re.DOTALL)
if fence:
response = fence.group(1)
try:
verdict_data = json.loads(response)
except json.JSONDecodeError as e:
log(f" Critique: failed to parse verdict for #{number} ({e}); omitting candidate")
log(f" Raw response: {response}")
continue
verdict = verdict_data.get("verdict")
rule = verdict_data.get("rule_violated")
rationale = verdict_data.get("rationale", "")
if verdict == "include":
log(f" Critique: keeping #{number} — {rationale}")
kept.append(match)
else:
rule_str = f"rule {rule}" if rule else "no specific rule"
log(f" Critique: omitting #{number} ({rule_str}) — {rationale}")
log(f" Critique: kept {len(kept)} of {len(proposed)} closed candidates")
return kept
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Identify potential duplicate issues")
parser.add_argument("issue_number", type=int, help="Issue number to analyze")
@ -658,6 +874,13 @@ if __name__ == "__main__":
anthropic_key, issue, relevant_magnets, search_results
)
# second-pass critique: prompt iteration on the proposer hit a ceiling around 30% noise.
# Re-evaluate each proposed closed candidate in isolation with a stricter prompt that
# has no slate to fill and is blind to the proposer's rationale.
related_closed_issues = critique_closed_candidates(
anthropic_key, issue, related_closed_issues, search_results
)
# resolve close reason from our search results (the source of truth) so we don't depend
# on Claude to faithfully echo it back
results_by_number = {r["number"]: r for r in search_results}

38
script/github-track-duplicate-bot-effectiveness.py
View file

@ -24,6 +24,7 @@ import functools
import os
import re
import sys
import time
from datetime import datetime, timezone
import requests
@ -47,6 +48,8 @@ BOT_START_DATE = "2026-02-18"
NEEDS_TRIAGE_LABEL = "state:needs triage"
DEFAULT_PROJECT_NUMBER = 76
VALID_CLOSED_AS_VALUES = {"duplicate", "not_planned", "completed"}
# HTTP statuses we'll retry on for GET requests
TRANSIENT_HTTP_STATUSES = {429, 500, 502, 503, 504}
# Add a new tuple when you deploy a new version of the bot that you want to
# keep track of (e.g. the prompt gets a rewrite or the model gets swapped).
# Newest first, please. The datetime is for the deployment time (merge to main).
@ -67,10 +70,22 @@ def bot_version_for_time(date_string):
def github_api_get(path, params=None):
"""Fetch JSON from the GitHub REST API, retrying transient failures. Raises on non-2xx status."""
url = f"{GITHUB_API}/{path.lstrip('/')}"
response = requests.get(url, headers=GITHUB_HEADERS, params=params)
response.raise_for_status()
return response.json()
for attempt in range(3):
try:
response = requests.get(url, headers=GITHUB_HEADERS, params=params)
response.raise_for_status()
return response.json()
except requests.RequestException as e:
transient = isinstance(e, (requests.ConnectionError, requests.Timeout)) or (
isinstance(e, requests.HTTPError) and e.response.status_code in TRANSIENT_HTTP_STATUSES
)
if not transient or attempt == 2:
raise
wait = 2 ** attempt
print(f" Transient GitHub API error ({e}); retrying in {wait}s")
time.sleep(wait)
def github_search_issues(query):
@ -161,9 +176,11 @@ def find_canonical_among(duplicate_number, candidates):
if not candidates:
return None
# candidate issue numbers are baked into the query body via field aliases
# (GraphQL doesn't let you parametrize alias names), so $numbers isn't needed.
data = github_api_graphql(
"""
query($owner: String!, $repo: String!, $numbers: [Int!]!) {
query($owner: String!, $repo: String!) {
repository(owner: $owner, name: $repo) {
PLACEHOLDER
}
@ -174,7 +191,7 @@ def find_canonical_among(duplicate_number, candidates):
f' nodes {{ ... on MarkedAsDuplicateEvent {{ duplicate {{ ... on Issue {{ number }} }} }} }} }} }}'
for number in candidates
)),
{"owner": REPO_OWNER, "repo": REPO_NAME, "numbers": list(candidates)},
{"owner": REPO_OWNER, "repo": REPO_NAME},
partial_errors_ok=True,
)
@ -409,11 +426,10 @@ def classify_as_assist(issue, bot_comment):
bot_comment_time=bot_comment["created_at"])
return
original = None
try:
original = find_canonical_among(issue["number"], suggested)
except (requests.RequestException, RuntimeError) as error:
print(f" Warning: failed to query candidate timelines: {error}")
# Let exceptions from find_canonical_among propagate — a query failure here is
# not the same as "no canonical match" and shouldn't be silently downgraded to
# a Needs review entry. Failing the workflow surfaces the problem immediately.
original = find_canonical_among(issue["number"], suggested)
if original:
status = "Auto-classified"
@ -483,6 +499,8 @@ def classify_open():
errors += 1
print(f" Done: added {added}, skipped {skipped}, errors {errors}")
if errors > 0:
sys.exit(1)
if __name__ == "__main__":