diff --git a/skyvern/cli/mcp_tools/prompts.py b/skyvern/cli/mcp_tools/prompts.py index 2dd4068d1..0d55ecab2 100644 --- a/skyvern/cli/mcp_tools/prompts.py +++ b/skyvern/cli/mcp_tools/prompts.py @@ -800,6 +800,68 @@ Use the evidence that actually matters: --- +## Step 5: Post Evidence to PR + +After generating the QA report, persist it to the pull request as a sticky comment so the +evidence survives beyond the conversation. + +### Check for an open PR + +```bash +PR_NUMBER=$(gh pr view --json number -q '.number' 2>/dev/null) +``` + +If no PR exists for the current branch: +1. Save the full report markdown to `.qa/latest-report.md` in the project root (create the directory if needed). +2. Tell the user: "No open PR found for this branch. QA report saved to `.qa/latest-report.md`. Run /qa again after creating a PR to post it." +3. Stop here — do not attempt to create a PR. + +### Post or update the sticky comment + +Use a hidden HTML marker to make the comment idempotent across multiple runs: + +```bash +# Prepare the comment body with the hidden marker +COMMENT_BODY=" +## QA Report — $(git rev-parse --short HEAD) — $(date -u +%Y-%m-%dT%H:%M:%SZ) + + +" + +# Find an existing QA comment on the PR +EXISTING_COMMENT_ID=$(gh api "repos/{owner}/{repo}/issues/${PR_NUMBER}/comments" \\ + --jq '.[] | select(.body | test("skyvern-qa-report")) | .id' \\ + 2>/dev/null | head -1) + +if [ -n "$EXISTING_COMMENT_ID" ]; then + # Update the existing comment in place + gh api "repos/{owner}/{repo}/issues/comments/${EXISTING_COMMENT_ID}" \\ + -X PATCH -f body="$COMMENT_BODY" +else + # Create a new comment + gh pr comment "$PR_NUMBER" --body "$COMMENT_BODY" +fi +``` + +### Screenshot handling + +Screenshots taken during QA (via `skyvern_screenshot()`) are saved locally for the agent's +verification. They are not uploaded to the PR comment because GitHub's API does not support +image uploads in issue comments. The text report describes what was observed. + +If the user asks to preserve screenshots, save them to `.qa/screenshots/` and tell the user +the local path. Do not include local file paths in the PR comment — they are meaningless to +other reviewers. + +### Rules + +- Always include the `` marker so repeated runs update the same comment instead of creating duplicates. +- Include the short commit hash and UTC timestamp in the comment header. +- Do not create a PR just to post a QA report — that is the user's decision. +- If `gh` is not available or not authenticated, fall back to saving the report locally and tell the user. + +--- + ## Tool Selection | What you need | Tool | Speed | diff --git a/skyvern/cli/skills/qa/SKILL.md b/skyvern/cli/skills/qa/SKILL.md index 754acae50..e2071209e 100644 --- a/skyvern/cli/skills/qa/SKILL.md +++ b/skyvern/cli/skills/qa/SKILL.md @@ -366,6 +366,66 @@ Report the evidence that actually matters: - status codes and response snippets for backend API results - command + failing assertion for unit/integration tests +## Step 6: Post Evidence to PR + +After generating the QA report, persist it to the pull request as a sticky comment so the +evidence survives beyond the conversation. + +### Check for an open PR + +```bash +PR_NUMBER=$(gh pr view --json number -q '.number' 2>/dev/null) +``` + +If no PR exists for the current branch: +1. Save the full report markdown to `.qa/latest-report.md` in the project root (create the directory if needed). +2. Tell the user: "No open PR found for this branch. QA report saved to `.qa/latest-report.md`. Run /qa again after creating a PR to post it." +3. Stop here — do not attempt to create a PR. + +### Post or update the sticky comment + +Use a hidden HTML marker to make the comment idempotent across multiple runs: + +```bash +# Prepare the comment body with the hidden marker +COMMENT_BODY=" +## QA Report — $(git rev-parse --short HEAD) — $(date -u +%Y-%m-%dT%H:%M:%SZ) + + +" + +# Find an existing QA comment on the PR +EXISTING_COMMENT_ID=$(gh api "repos/{owner}/{repo}/issues/${PR_NUMBER}/comments" \ + --jq '.[] | select(.body | test("skyvern-qa-report")) | .id' \ + 2>/dev/null | head -1) + +if [ -n "$EXISTING_COMMENT_ID" ]; then + # Update the existing comment in place + gh api "repos/{owner}/{repo}/issues/comments/${EXISTING_COMMENT_ID}" \ + -X PATCH -f body="$COMMENT_BODY" +else + # Create a new comment + gh pr comment "$PR_NUMBER" --body "$COMMENT_BODY" +fi +``` + +### Screenshot handling + +Screenshots taken during QA (via `skyvern_screenshot()`) are saved locally for the agent's +verification. They are not uploaded to the PR comment because GitHub's API does not support +image uploads in issue comments. The text report describes what was observed. + +If the user asks to preserve screenshots, save them to `.qa/screenshots/` and tell the user +the local path. Do not include local file paths in the PR comment — they are meaningless to +other reviewers. + +### Rules + +- Always include the `` marker so repeated runs update the same comment instead of creating duplicates. +- Include the short commit hash and UTC timestamp in the comment header. +- Do not create a PR just to post a QA report — that is the user's decision. +- If `gh` is not available or not authenticated, fall back to saving the report locally and tell the user. + ## Error Handling | Problem | Action | diff --git a/tests/unit/test_qa_skill_content.py b/tests/unit/test_qa_skill_content.py index 811b25722..12c13b5c3 100644 --- a/tests/unit/test_qa_skill_content.py +++ b/tests/unit/test_qa_skill_content.py @@ -1,11 +1,21 @@ from __future__ import annotations +import subprocess +import sys from pathlib import Path +import pytest + from skyvern.cli.mcp_tools.prompts import QA_TEST_CONTENT, qa_test ROOT = Path(__file__).resolve().parents[2] BUNDLED_QA_SKILL = ROOT / "skyvern" / "cli" / "skills" / "qa" / "SKILL.md" +CLAUDE_QA_SKILL = ROOT / ".claude" / "skills" / "qa" / "SKILL.md" + +_needs_cloud_repo = pytest.mark.skipif( + not CLAUDE_QA_SKILL.exists(), + reason=".claude/skills/qa/SKILL.md not present (OSS checkout)", +) def _first_nonempty_line_after_h1(text: str) -> str: @@ -21,6 +31,11 @@ def _first_nonempty_line_after_h1(text: str) -> str: return "" +@_needs_cloud_repo +def test_bundled_and_claude_qa_skill_match_exactly() -> None: + assert BUNDLED_QA_SKILL.read_text(encoding="utf-8") == CLAUDE_QA_SKILL.read_text(encoding="utf-8") + + def test_qa_skill_has_summary_line_before_note_comment() -> None: skill_text = BUNDLED_QA_SKILL.read_text(encoding="utf-8") first_line_after_h1 = _first_nonempty_line_after_h1(skill_text) @@ -78,3 +93,32 @@ def test_qa_test_prompt_includes_target_url_and_focus_area() -> None: assert "Target URL: `http://localhost:8000`" in rendered assert "Focus area: validate the workflow filters API" in rendered assert "choose the correct validation mode" in rendered + + +def test_qa_pr_evidence_markers_present() -> None: + """Assert the PR evidence posting instructions are present in all /qa surfaces.""" + skill_text = BUNDLED_QA_SKILL.read_text(encoding="utf-8") + + # Check SKILL.md + assert "" in skill_text + assert "Post Evidence to PR" in skill_text + assert ".qa/latest-report.md" in skill_text + assert "gh pr comment" in skill_text + + # Check QA_TEST_CONTENT (MCP prompt) + assert "" in QA_TEST_CONTENT + assert "Post Evidence to PR" in QA_TEST_CONTENT + assert ".qa/latest-report.md" in QA_TEST_CONTENT + assert "gh pr comment" in QA_TEST_CONTENT + + +@_needs_cloud_repo +def test_validate_skills_package_script_passes() -> None: + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "validate_skills_package.py")], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr