From 037412ec7d3b55090adc5f4c0b34af8fefe1594c Mon Sep 17 00:00:00 2001 From: brokemac79 Date: Tue, 7 Jul 2026 10:54:35 +0100 Subject: [PATCH] ci(mantis): add web UI chat proof lane (#100472) * ci(mantis): add web ui chat proof lane * ci(mantis): tighten web ui proof candidate parsing * ci: tighten Mantis Web UI proof lane --------- Co-authored-by: Peter Steinberger --- .github/workflows/mantis-scenario.yml | 13 + .../workflows/mantis-web-ui-chat-proof.yml | 410 ++++++++++++++++++ docs/concepts/mantis.md | 52 ++- scripts/mantis/build-web-ui-chat-evidence.mjs | 174 ++++++++ scripts/test-projects.test-support.mjs | 15 + .../mantis-web-ui-chat-evidence.test.ts | 103 +++++ .../mantis-web-ui-chat-proof-workflow.test.ts | 87 ++++ .../package-acceptance-workflow.test.ts | 6 + test/scripts/test-projects.test.ts | 8 + ui/src/e2e/mantis-chat-proof.e2e.test.ts | 125 ++++++ 10 files changed, 976 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/mantis-web-ui-chat-proof.yml create mode 100644 scripts/mantis/build-web-ui-chat-evidence.mjs create mode 100644 test/scripts/mantis-web-ui-chat-evidence.test.ts create mode 100644 test/scripts/mantis-web-ui-chat-proof-workflow.test.ts create mode 100644 ui/src/e2e/mantis-chat-proof.e2e.test.ts diff --git a/.github/workflows/mantis-scenario.yml b/.github/workflows/mantis-scenario.yml index 8486d658c0b..67b5315a7f7 100644 --- a/.github/workflows/mantis-scenario.yml +++ b/.github/workflows/mantis-scenario.yml @@ -14,6 +14,7 @@ on: - slack-desktop-smoke - telegram-live - telegram-desktop-proof + - web-ui-chat-proof baseline_ref: description: Optional baseline ref for before/after scenarios required: false @@ -121,6 +122,18 @@ jobs: fi gh "${args[@]}" ;; + web-ui-chat-proof) + args=( + workflow run mantis-web-ui-chat-proof.yml + --repo "$GITHUB_REPOSITORY" + --ref main + -f "candidate_ref=${CANDIDATE_REF}" + ) + if [[ -n "${PR_NUMBER:-}" ]]; then + args+=(-f "pr_number=${PR_NUMBER}") + fi + gh "${args[@]}" + ;; *) echo "Unsupported Mantis scenario: ${SCENARIO_ID}" >&2 exit 1 diff --git a/.github/workflows/mantis-web-ui-chat-proof.yml b/.github/workflows/mantis-web-ui-chat-proof.yml new file mode 100644 index 00000000000..e0c37edd90f --- /dev/null +++ b/.github/workflows/mantis-web-ui-chat-proof.yml @@ -0,0 +1,410 @@ +name: Mantis Web UI Chat Proof + +on: + issue_comment: # zizmor: ignore[dangerous-triggers] maintainer-only Mantis command; candidate refs are trusted before execution and publishing runs in a separate job + types: [created] + workflow_dispatch: + inputs: + candidate_ref: + description: Ref, tag, or SHA expected to prove Control UI web chat behavior + required: true + default: main + type: string + pr_number: + description: Optional PR number to receive the QA evidence comment + required: false + type: string + +permissions: + actions: read + contents: read + issues: write + pull-requests: write + +concurrency: + group: mantis-web-ui-chat-proof-${{ github.event.issue.number || inputs.pr_number || inputs.candidate_ref || github.run_id }}-${{ github.run_attempt }} + cancel-in-progress: false + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + NODE_VERSION: "24.x" + +jobs: + authorize_actor: + name: Authorize workflow actor + if: >- + ${{ + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'issue_comment' && + github.event.issue.pull_request && + ( + contains(github.event.comment.body, '@openclaw-mantis') || + contains(github.event.comment.body, '/openclaw-mantis') + ) + ) + }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + outputs: + authorized: ${{ steps.permission.outputs.authorized }} + steps: + - name: Require maintainer-level repository access + id: permission + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + with: + script: | + const allowed = new Set(["admin", "maintain", "write"]); + const { owner, repo } = context.repo; + const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner, + repo, + username: context.actor, + }); + const permission = data.permission; + core.info(`Actor ${context.actor} permission: ${permission}`); + if (!allowed.has(permission)) { + core.notice( + `Workflow requires write/maintain/admin access. Actor "${context.actor}" has "${permission}".`, + ); + core.setOutput("authorized", "false"); + return; + } + core.setOutput("authorized", "true"); + + resolve_request: + name: Resolve Mantis request + needs: authorize_actor + if: needs.authorize_actor.outputs.authorized == 'true' + runs-on: blacksmith-8vcpu-ubuntu-2404 + outputs: + candidate_ref: ${{ steps.resolve.outputs.candidate_ref }} + pr_number: ${{ steps.resolve.outputs.pr_number }} + request_source: ${{ steps.resolve.outputs.request_source }} + should_run: ${{ steps.resolve.outputs.should_run }} + steps: + - name: Resolve ref and target PR + id: resolve + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + with: + script: | + const eventName = context.eventName; + + function setOutput(name, value) { + core.setOutput(name, value ?? ""); + core.info(`${name}=${value ?? ""}`); + } + + if (eventName === "workflow_dispatch") { + const inputs = context.payload.inputs ?? {}; + setOutput("should_run", "true"); + setOutput("candidate_ref", inputs.candidate_ref || "main"); + setOutput("pr_number", inputs.pr_number || ""); + setOutput("request_source", "workflow_dispatch"); + return; + } + + if (eventName !== "issue_comment") { + core.setFailed(`Unsupported event: ${eventName}`); + return; + } + + const issue = context.payload.issue; + const body = context.payload.comment?.body ?? ""; + if (!issue?.pull_request) { + core.setFailed("Mantis issue_comment trigger requires a pull request comment."); + return; + } + + const normalized = body.toLowerCase(); + const mentionsMantis = + normalized.includes("@openclaw-mantis") || normalized.includes("/openclaw-mantis"); + const requestedWebUiChat = + normalized.includes("web-ui-chat") || + normalized.includes("web ui chat") || + (normalized.includes("web ui") && normalized.includes("chat")) || + (normalized.includes("control ui") && normalized.includes("chat")); + if (!mentionsMantis || !requestedWebUiChat) { + core.notice("Comment mentioned Mantis but did not request web UI chat proof."); + setOutput("should_run", "false"); + setOutput("candidate_ref", ""); + setOutput("pr_number", ""); + setOutput("request_source", "unsupported_issue_comment"); + return; + } + + const { owner, repo } = context.repo; + const { data: pr } = await github.rest.pulls.get({ + owner, + repo, + pull_number: issue.number, + }); + const candidateMatch = body.match(/\b(?:candidate|head)\s*[:=]\s*`?([^\s`]+)`?/i); + const rawCandidate = candidateMatch?.[1]; + const candidate = + rawCandidate && !["head", "pr", "pr-head"].includes(rawCandidate.toLowerCase()) + ? rawCandidate + : pr.head.sha; + + setOutput("should_run", "true"); + setOutput("candidate_ref", candidate); + setOutput("pr_number", String(issue.number)); + setOutput("request_source", "issue_comment"); + + await github.rest.reactions.createForIssueComment({ + owner, + repo, + comment_id: context.payload.comment.id, + content: "eyes", + }).catch((error) => core.warning(`Could not add eyes reaction: ${error.message}`)); + + validate_candidate: + name: Validate selected candidate + needs: resolve_request + if: ${{ needs.resolve_request.outputs.should_run == 'true' }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + outputs: + candidate_revision: ${{ steps.validate.outputs.candidate_revision }} + steps: + - name: Checkout harness ref + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Validate candidate ref is trusted + id: validate + env: + GH_TOKEN: ${{ github.token }} + CANDIDATE_REF: ${{ needs.resolve_request.outputs.candidate_ref }} + shell: bash + run: | + set -euo pipefail + + git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main + + revision="$(git rev-parse "${CANDIDATE_REF}^{commit}")" + reason="" + if git merge-base --is-ancestor "$revision" refs/remotes/origin/main; then + reason="main-ancestor" + elif git tag --points-at "$revision" | grep -Eq '^v'; then + reason="release-tag" + else + pr_head_count="$( + gh api \ + -H "Accept: application/vnd.github+json" \ + "repos/${GITHUB_REPOSITORY}/commits/${revision}/pulls" \ + --jq '[.[] | select(.state == "open" and .head.repo.full_name == "'"${GITHUB_REPOSITORY}"'" and .head.sha == "'"${revision}"'")] | length' + )" + if [[ "$pr_head_count" != "0" ]]; then + reason="open-pr-head" + fi + fi + + if [[ -z "$reason" ]]; then + echo "Candidate ref '${CANDIDATE_REF}' resolved to ${revision}, which is not trusted for this Mantis run." >&2 + exit 1 + fi + + echo "candidate_revision=${revision}" >> "$GITHUB_OUTPUT" + { + echo "candidate: \`${CANDIDATE_REF}\`" + echo "candidate SHA: \`${revision}\`" + echo "candidate trust reason: \`${reason}\`" + } >> "$GITHUB_STEP_SUMMARY" + + run_web_ui_chat: + name: Run Control UI web chat proof + needs: [resolve_request, validate_candidate] + if: ${{ needs.resolve_request.outputs.should_run == 'true' }} + permissions: + contents: read + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 60 + outputs: + artifact_name: ${{ steps.run_mantis.outputs.artifact_name }} + output_dir: ${{ steps.run_mantis.outputs.output_dir }} + proof_status: ${{ steps.run_mantis.outputs.proof_status }} + steps: + - name: Checkout harness ref + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + node-version: ${{ env.NODE_VERSION }} + install-bun: "false" + install-deps: "false" + + - name: Prepare candidate worktree + env: + CANDIDATE_SHA: ${{ needs.validate_candidate.outputs.candidate_revision }} + shell: bash + run: | + set -euo pipefail + worktree_root=".artifacts/qa-e2e/mantis/web-ui-chat-proof-worktrees" + mkdir -p "$worktree_root" + git worktree add --detach "$worktree_root/candidate" "$CANDIDATE_SHA" + pnpm --dir "$worktree_root/candidate" install --frozen-lockfile --prefer-offline + + - name: Run web UI chat proof + id: run_mantis + env: + CANDIDATE_REF: ${{ needs.resolve_request.outputs.candidate_ref }} + CANDIDATE_SHA: ${{ needs.validate_candidate.outputs.candidate_revision }} + shell: bash + run: | + set -euo pipefail + + artifact_name="mantis-web-ui-chat-proof-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + candidate_repo="$(pwd)/.artifacts/qa-e2e/mantis/web-ui-chat-proof-worktrees/candidate" + output_rel=".artifacts/qa-e2e/mantis/web-ui-chat-proof" + root="$candidate_repo/$output_rel" + mkdir -p "$root" + echo "artifact_name=${artifact_name}" >> "$GITHUB_OUTPUT" + echo "output_dir=${root}" >> "$GITHUB_OUTPUT" + + install -D -m 0644 \ + "${GITHUB_WORKSPACE}/ui/src/e2e/mantis-chat-proof.e2e.test.ts" \ + "$candidate_repo/ui/src/e2e/mantis-chat-proof.e2e.test.ts" + install -D -m 0644 \ + "${GITHUB_WORKSPACE}/ui/src/test-helpers/control-ui-e2e.ts" \ + "$candidate_repo/ui/src/test-helpers/control-ui-e2e.ts" + + cd "$candidate_repo" + node scripts/ensure-playwright-chromium.mjs + + set +e + OPENCLAW_MANTIS_WEB_UI_CHAT_OUTPUT_DIR="$root" \ + node scripts/run-vitest.mjs run \ + --config test/vitest/vitest.ui-e2e.config.ts \ + --configLoader runner \ + ui/src/e2e/mantis-chat-proof.e2e.test.ts \ + 2>&1 | tee "$root/vitest.log" + vitest_exit="${PIPESTATUS[0]}" + set -e + + proof_status="pass" + if [[ "$vitest_exit" -ne 0 ]]; then + proof_status="fail" + fi + echo "proof_status=${proof_status}" >> "$GITHUB_OUTPUT" + + node "${GITHUB_WORKSPACE}/scripts/mantis/build-web-ui-chat-evidence.mjs" \ + --output-dir "$root" \ + --candidate-ref "$CANDIDATE_REF" \ + --candidate-sha "$CANDIDATE_SHA" \ + --status "$proof_status" + + cat "$root/mantis-report.md" >> "$GITHUB_STEP_SUMMARY" + if [[ "$vitest_exit" -ne 0 ]]; then + exit "$vitest_exit" + fi + + - name: Upload Mantis web UI chat artifacts + if: ${{ always() && steps.run_mantis.outputs.output_dir != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ steps.run_mantis.outputs.artifact_name }} + path: ${{ steps.run_mantis.outputs.output_dir }} + retention-days: 14 + if-no-files-found: error + + publish_evidence: + name: Publish Mantis web UI chat evidence + needs: [resolve_request, run_web_ui_chat] + if: ${{ always() && needs.resolve_request.outputs.pr_number != '' && needs.run_web_ui_chat.outputs.artifact_name != '' }} + runs-on: ubuntu-24.04 + environment: qa-live-shared + steps: + - name: Checkout harness ref + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + + - name: Download Mantis web UI chat artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: ${{ needs.run_web_ui_chat.outputs.artifact_name }} + path: .artifacts/qa-e2e/mantis/web-ui-chat-proof + + - name: Create Mantis GitHub App token + id: mantis_app_token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + with: + app-id: ${{ secrets.MANTIS_GITHUB_APP_ID }} + private-key: ${{ secrets.MANTIS_GITHUB_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} + permission-issues: write + permission-pull-requests: write + + - name: Comment PR with inline QA evidence + env: + GH_TOKEN: ${{ steps.mantis_app_token.outputs.token }} + MANTIS_ARTIFACT_R2_ACCESS_KEY_ID: ${{ secrets.MANTIS_ARTIFACT_R2_ACCESS_KEY_ID }} + MANTIS_ARTIFACT_R2_BUCKET: openclaw-crabbox-artifacts + MANTIS_ARTIFACT_R2_ENDPOINT: ${{ vars.MANTIS_ARTIFACT_R2_ENDPOINT }} + MANTIS_ARTIFACT_R2_PUBLIC_BASE_URL: https://artifacts.openclaw.ai + MANTIS_ARTIFACT_R2_REGION: auto + MANTIS_ARTIFACT_R2_SECRET_ACCESS_KEY: ${{ secrets.MANTIS_ARTIFACT_R2_SECRET_ACCESS_KEY }} + REQUEST_SOURCE: ${{ needs.resolve_request.outputs.request_source }} + TARGET_PR: ${{ needs.resolve_request.outputs.pr_number }} + shell: bash + run: | + set -euo pipefail + root=".artifacts/qa-e2e/mantis/web-ui-chat-proof" + if [[ ! -f "$root/mantis-evidence.json" ]]; then + echo "No Mantis evidence manifest found; skipping PR evidence comment." + exit 0 + fi + node scripts/mantis/publish-pr-evidence.mjs \ + --manifest "$root/mantis-evidence.json" \ + --target-pr "$TARGET_PR" \ + --artifact-root "mantis/web-ui-chat-proof/pr-${TARGET_PR}/run-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + --marker "" \ + --run-url "https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + --request-source "$REQUEST_SOURCE" + + clear_issue_comment_reaction: + name: Clear Mantis command reaction + needs: [resolve_request, validate_candidate, run_web_ui_chat, publish_evidence] + if: ${{ always() && github.event_name == 'issue_comment' && needs.resolve_request.outputs.request_source == 'issue_comment' }} + runs-on: ubuntu-24.04 + permissions: + issues: write + steps: + - name: Remove workflow eyes reaction + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + with: + script: | + const { owner, repo } = context.repo; + const commentId = context.payload.comment?.id; + if (!commentId) { + core.info("No issue comment id found; skipping reaction cleanup."); + return; + } + + const reactions = await github.paginate(github.rest.reactions.listForIssueComment, { + owner, + repo, + comment_id: commentId, + per_page: 100, + }); + const eyes = reactions.filter( + (reaction) => reaction.content === "eyes" && reaction.user?.login === "github-actions[bot]", + ); + for (const reaction of eyes) { + await github.rest.reactions.deleteForIssueComment({ + owner, + repo, + comment_id: commentId, + reaction_id: reaction.id, + }); + core.info(`Removed eyes reaction ${reaction.id} from comment ${commentId}.`); + } + if (eyes.length === 0) { + core.info(`No workflow eyes reaction found on comment ${commentId}.`); + } diff --git a/docs/concepts/mantis.md b/docs/concepts/mantis.md index 4ddbb755c07..38c955ece98 100644 --- a/docs/concepts/mantis.md +++ b/docs/concepts/mantis.md @@ -1,18 +1,20 @@ --- -summary: "Mantis is the visual end-to-end verification system for reproducing OpenClaw bugs on live transports, capturing before and after evidence, and attaching artifacts to PRs." +summary: "Mantis captures visual end-to-end evidence for live transport comparisons and focused candidate-only browser proofs, then attaches the artifacts to PRs." title: "Mantis" read_when: - Building or running live visual QA for OpenClaw bugs - Adding before and after verification for a pull request - Adding Discord, Slack, WhatsApp, or other live transport scenarios + - Running focused Control UI browser proof for a candidate ref - Debugging QA runs that need screenshots, browser automation, or VNC access --- -Mantis reruns a bug scenario against a known-bad baseline ref and a candidate -ref on a real transport, then publishes a before/after comparison as CI -artifacts and a PR comment. Discord shipped first: real bot auth, real guild -channels, reactions, threads, and a browser witness a human can check. Slack -and Telegram lanes exist too; WhatsApp and Matrix are unimplemented. +Mantis publishes visual CI evidence and a PR comment for OpenClaw behavior. +Live transport scenarios compare a known-bad baseline with a candidate ref; +focused browser lanes may instead prove one candidate against a deterministic +mocked transport. Discord shipped first with real bot auth, guild channels, +reactions, threads, and a browser witness. Slack, Telegram, and focused Control +UI chat lanes exist too; WhatsApp and Matrix are unimplemented. ## Ownership @@ -300,14 +302,15 @@ Comments post through the Mantis GitHub App (`MANTIS_GITHUB_APP_ID` / `MANTIS_GITHUB_APP_PRIVATE_KEY`), not `github-actions[bot]`, using a hidden marker comment as the upsert key. -| Workflow | Trigger | What it does | -| --------------------------------- | ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Mantis Discord Smoke` | manual dispatch | Runs `discord-smoke` against a chosen ref. | -| `Mantis Discord Status Reactions` | PR comment or manual dispatch | Builds separate baseline/candidate worktrees, runs `discord-status-reactions-tool-only` on each, renders each lane's timeline in a Crabbox desktop browser, generates motion-trimmed GIF/MP4 previews with `crabbox media preview`, uploads artifacts, posts inline PR evidence. | -| `Mantis Scenario` | manual dispatch | Generic dispatcher: takes `scenario_id` (`discord-status-reactions-tool-only`, `discord-thread-reply-filepath-attachment`, `slack-desktop-smoke`, `telegram-live`, `telegram-desktop-proof`), `baseline_ref`, `candidate_ref`, `pr_number`, and forwards to the matching scenario workflow. | -| `Mantis Slack Desktop Smoke` | manual dispatch | Leases a Crabbox Linux desktop (defaults to `aws`, choice of `hetzner`), runs `slack-desktop-smoke --gateway-setup` against the candidate, records the desktop, generates a motion preview, uploads artifacts, posts PR evidence when a PR number is given. | -| `Mantis Telegram Live` | PR comment or manual dispatch | Runs the bot-API Telegram live QA lane (`openclaw qa telegram`), writes `mantis-evidence.json` from the QA summary, renders redacted evidence HTML through a Crabbox desktop browser, generates a motion GIF, posts PR evidence. Telegram Web login is not required for this lane. | -| `Mantis Telegram Desktop Proof` | maintainer PR label (`mantis: telegram-visible-proof`) plus PR comment, or manual dispatch | Agentic native Telegram Desktop before/after proof. Hands the PR, baseline/candidate refs, and maintainer instructions to Codex, which runs the real-user Crabbox Telegram Desktop proof lane for both refs and posts a 2-column PR evidence table. | +| Workflow | Trigger | What it does | +| --------------------------------- | ------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Mantis Discord Smoke` | manual dispatch | Runs `discord-smoke` against a chosen ref. | +| `Mantis Discord Status Reactions` | PR comment or manual dispatch | Builds separate baseline/candidate worktrees, runs `discord-status-reactions-tool-only` on each, renders each lane's timeline in a Crabbox desktop browser, generates motion-trimmed GIF/MP4 previews with `crabbox media preview`, uploads artifacts, posts inline PR evidence. | +| `Mantis Scenario` | manual dispatch | Generic dispatcher: takes `scenario_id` (`discord-status-reactions-tool-only`, `discord-thread-reply-filepath-attachment`, `slack-desktop-smoke`, `telegram-live`, `telegram-desktop-proof`, `web-ui-chat-proof`), `baseline_ref`, `candidate_ref`, `pr_number`, and forwards to the matching scenario workflow. | +| `Mantis Slack Desktop Smoke` | manual dispatch | Leases a Crabbox Linux desktop (defaults to `aws`, choice of `hetzner`), runs `slack-desktop-smoke --gateway-setup` against the candidate, records the desktop, generates a motion preview, uploads artifacts, posts PR evidence when a PR number is given. | +| `Mantis Telegram Live` | PR comment or manual dispatch | Runs the bot-API Telegram live QA lane (`openclaw qa telegram`), writes `mantis-evidence.json` from the QA summary, renders redacted evidence HTML through a Crabbox desktop browser, generates a motion GIF, posts PR evidence. Telegram Web login is not required for this lane. | +| `Mantis Telegram Desktop Proof` | maintainer PR label (`mantis: telegram-visible-proof`) plus PR comment, or manual dispatch | Agentic native Telegram Desktop before/after proof. Hands the PR, baseline/candidate refs, and maintainer instructions to Codex, which runs the real-user Crabbox Telegram Desktop proof lane for both refs and posts a 2-column PR evidence table. | +| `Mantis Web UI Chat Proof` | PR comment or manual dispatch | Runs the focused OpenClaw Control UI chat Playwright proof against the candidate, verifies the browser sends through the mocked Gateway, captures screenshot/video artifacts, and posts PR evidence. This lane is web chat proof only, not WinUI/native-app or arbitrary visual proof. | `Mantis Discord Status Reactions` and `Mantis Telegram Live` both accept `baseline_ref`/`candidate_ref` (or `baseline=`/`candidate=` in a PR comment) @@ -323,6 +326,8 @@ Comment triggers, from a PR with write/maintain/admin access: @openclaw-mantis telegram @openclaw-mantis telegram scenario=telegram-status-command @openclaw-mantis telegram scenarios=telegram-status-command,telegram-mentioned-message-reply +@openclaw-mantis web ui chat +@openclaw-mantis web-ui-chat candidate=HEAD ``` Telegram comment triggers default to the PR head SHA as candidate and @@ -331,6 +336,11 @@ Telegram comment triggers default to the PR head SHA as candidate and desktop. `Mantis Telegram Desktop Proof` only responds to a PR comment when the PR already carries the `mantis: telegram-visible-proof` label. +Web UI chat comment triggers default to the PR head SHA as candidate. They run +the Control UI mocked-Gateway chat proof and publish browser artifacts; use +normal Playwright/browser proof, maintainer screenshots, Crabbox, or local +artifacts for other web pages and native app surfaces. + ClawSweeper can also dispatch a scenario directly: ```text @@ -372,16 +382,19 @@ rotate it after the replacement secret is stored. ## Run outcomes -A scenario fails in one of two distinguishable ways, and the report separates -them so a flaky environment does not read as a product regression: +Before/after transport scenarios distinguish these outcomes so a flaky +environment does not read as a product regression: - **Bug reproduced**: baseline failed the way the scenario expects. - **Harness failure**: environment setup, credentials, transport API, browser, or provider failed before the oracle was meaningful. +Candidate-only browser proof reports whether the candidate passed the mocked +Gateway and visible UI assertions; it does not claim baseline reproduction. + ## Adding a scenario -Scenarios are TypeScript-defined per transport (see +Live transport scenarios are TypeScript-defined per transport (see `MANTIS_SCENARIO_CONFIGS` in `extensions/qa-lab/src/mantis/run.runtime.ts` for the Discord before/after shape), not a standalone declarative file format. Each scenario needs: id and title, transport, required credentials, baseline @@ -389,6 +402,11 @@ ref policy, candidate ref policy, OpenClaw config patch, setup/stimulus steps, expected baseline and candidate oracle, visual capture targets, timeout budget, and cleanup steps. +Focused candidate-only browser proof can use a dedicated deterministic E2E test +and workflow. Keep its scope explicit, validate the candidate ref before +execution, isolate secret-backed publishing, and emit the same evidence +manifest contract. + Prefer small, typed oracles over vision checks: Discord reaction state or message references, Slack thread `ts`/reaction API state, email message ids and headers. Use browser screenshots when UI is the only reliable observable, diff --git a/scripts/mantis/build-web-ui-chat-evidence.mjs b/scripts/mantis/build-web-ui-chat-evidence.mjs new file mode 100644 index 00000000000..6dd531ac3da --- /dev/null +++ b/scripts/mantis/build-web-ui-chat-evidence.mjs @@ -0,0 +1,174 @@ +#!/usr/bin/env node +// Builds a Mantis evidence manifest from Control UI web chat proof artifacts. +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +function parseArgs(argv) { + const args = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith("--")) { + throw new Error(`Unexpected argument: ${key}`); + } + const name = key.slice(2).replaceAll("-", "_"); + const value = argv[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error(`Missing value for ${key}`); + } + args[name] = value; + index += 1; + } + return args; +} + +function normalizeStatus(value) { + const normalized = String(value ?? "") + .trim() + .toLowerCase(); + if (normalized === "pass") { + return "pass"; + } + if (normalized === "fail") { + return "fail"; + } + throw new Error(`Unsupported web UI chat proof status: ${value}`); +} + +function artifactEntry({ inline = false, kind, label, path: artifactPath, required, targetPath }) { + return { + kind, + lane: "candidate", + label, + path: artifactPath, + targetPath, + required, + ...(inline ? { alt: label, inline: true, width: 900 } : {}), + }; +} + +export function buildWebUiChatEvidenceManifest({ candidateRef, candidateSha, status }) { + const passed = status === "pass"; + return { + schemaVersion: 1, + id: "web-ui-chat-proof", + title: "Mantis Web UI Chat Proof", + summary: + "Mantis ran the OpenClaw Control UI chat proof against the candidate ref, sent a message through the mocked Gateway, rendered the final reply in the browser, and captured browser artifacts for review.", + scenario: "web-ui-chat-proof", + comparison: { + candidate: { + ...(candidateSha ? { sha: candidateSha } : {}), + ...(candidateRef ? { ref: candidateRef } : {}), + expected: "Control UI chat sends through the Gateway and renders the final reply", + status, + fixed: passed, + }, + pass: passed, + }, + artifacts: [ + artifactEntry({ + inline: true, + kind: "desktopScreenshot", + label: "Control UI web chat proof", + path: "web-ui-chat.png", + required: passed, + targetPath: "web-ui-chat.png", + }), + artifactEntry({ + kind: "fullVideo", + label: "Control UI web chat recording", + path: "web-ui-chat.webm", + required: false, + targetPath: "web-ui-chat.webm", + }), + artifactEntry({ + kind: "metadata", + label: "Control UI web chat proof metadata", + path: "web-ui-chat-proof.json", + required: passed, + targetPath: "web-ui-chat-proof.json", + }), + artifactEntry({ + kind: "metadata", + label: "Control UI web chat Vitest log", + path: "vitest.log", + required: false, + targetPath: "vitest.log", + }), + { + kind: "report", + lane: "run", + label: "Mantis web UI chat report", + path: "mantis-report.md", + targetPath: "mantis-report.md", + }, + ], + }; +} + +function renderReport({ candidateRef, candidateSha, outputDir, status }) { + const artifactStatus = (artifactPath) => + existsSync(path.join(outputDir, artifactPath)) ? "present" : "missing"; + return [ + "# Mantis Web UI Chat Proof", + "", + `Status: ${status}`, + `Candidate ref: ${candidateRef || "unspecified"}`, + `Candidate SHA: ${candidateSha || "unspecified"}`, + "", + "## Scenario", + "", + "OpenClaw Control UI chat was loaded in a browser with the mocked Gateway harness. The proof sends a chat message through the GUI, verifies the `chat.send` request, emits a final Gateway reply, and waits for the reply to render in the web chat thread.", + "", + "## Artifacts", + "", + `- Screenshot: \`web-ui-chat.png\` (${artifactStatus("web-ui-chat.png")})`, + `- Recording: \`web-ui-chat.webm\` (${artifactStatus("web-ui-chat.webm")})`, + `- Proof metadata: \`web-ui-chat-proof.json\` (${artifactStatus("web-ui-chat-proof.json")})`, + `- Vitest log: \`vitest.log\` (${artifactStatus("vitest.log")})`, + "", + ].join("\n"); +} + +export function writeWebUiChatEvidence(rawArgs = process.argv.slice(2)) { + const args = parseArgs(rawArgs); + if (!args.output_dir) { + throw new Error("Missing --output-dir."); + } + if (!args.status) { + throw new Error("Missing --status."); + } + const outputDir = path.resolve(args.output_dir); + mkdirSync(outputDir, { recursive: true }); + const status = normalizeStatus(args.status); + const manifest = buildWebUiChatEvidenceManifest({ + candidateRef: args.candidate_ref, + candidateSha: args.candidate_sha, + status, + }); + const reportPath = path.join(outputDir, "mantis-report.md"); + writeFileSync( + reportPath, + renderReport({ + candidateRef: args.candidate_ref, + candidateSha: args.candidate_sha, + outputDir, + status, + }), + "utf8", + ); + const manifestPath = path.join(outputDir, "mantis-evidence.json"); + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); + return { manifest, manifestPath, reportPath }; +} + +const executedPath = process.argv[1] ? path.resolve(process.argv[1]) : ""; +if (executedPath === fileURLToPath(import.meta.url)) { + try { + writeWebUiChatEvidence(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index bb50400167f..51ef67307dc 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -500,6 +500,13 @@ const GITHUB_WORKFLOW_OWNER_TEST_TARGETS = new Map([ "test/scripts/package-acceptance-workflow.test.ts", ], ], + [ + ".github/workflows/mantis-web-ui-chat-proof.yml", + [ + "test/scripts/mantis-web-ui-chat-proof-workflow.test.ts", + "test/scripts/package-acceptance-workflow.test.ts", + ], + ], [ ".github/workflows/mantis-discord-smoke.yml", ["test/scripts/package-acceptance-workflow.test.ts"], @@ -1331,6 +1338,10 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ "scripts/mantis/build-telegram-desktop-proof-evidence.mjs", ["test/scripts/mantis-build-telegram-desktop-proof-evidence.test.ts"], ], + [ + "scripts/mantis/build-web-ui-chat-evidence.mjs", + ["test/scripts/mantis-web-ui-chat-evidence.test.ts"], + ], ["scripts/mantis/publish-pr-evidence.mjs", ["test/scripts/mantis-publish-pr-evidence.test.ts"]], ["scripts/qa-e2e.ts", ["test/scripts/qa-e2e.test.ts"]], ["scripts/qa-lab-up.ts", ["test/scripts/qa-lab-up.test.ts"]], @@ -2014,6 +2025,10 @@ const TOOLING_TEST_TARGETS = new Map([ "test/scripts/mantis-build-telegram-desktop-proof-evidence.test.ts", ["test/scripts/mantis-build-telegram-desktop-proof-evidence.test.ts"], ], + [ + "test/scripts/mantis-web-ui-chat-evidence.test.ts", + ["test/scripts/mantis-web-ui-chat-evidence.test.ts"], + ], [ "test/scripts/plugin-prerelease-test-plan.test.ts", ["test/scripts/plugin-prerelease-test-plan.test.ts"], diff --git a/test/scripts/mantis-web-ui-chat-evidence.test.ts b/test/scripts/mantis-web-ui-chat-evidence.test.ts new file mode 100644 index 00000000000..20bf6f3a6f2 --- /dev/null +++ b/test/scripts/mantis-web-ui-chat-evidence.test.ts @@ -0,0 +1,103 @@ +// Mantis web UI chat evidence tests cover manifest generation. +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + buildWebUiChatEvidenceManifest, + writeWebUiChatEvidence, +} from "../../scripts/mantis/build-web-ui-chat-evidence.mjs"; + +function withTempDir(fn: (dir: string) => T): T { + const dir = mkdtempSync(path.join(tmpdir(), "openclaw-mantis-web-ui-chat-")); + try { + return fn(dir); + } finally { + rmSync(dir, { force: true, recursive: true }); + } +} + +describe("build-web-ui-chat-evidence", () => { + it("marks a passing Control UI chat proof as publishable visible Mantis evidence", () => { + const manifest = buildWebUiChatEvidenceManifest({ + candidateRef: "HEAD", + candidateSha: "1234567890abcdef1234567890abcdef12345678", + status: "pass", + }); + + expect(manifest).toMatchObject({ + id: "web-ui-chat-proof", + scenario: "web-ui-chat-proof", + comparison: { + candidate: { + fixed: true, + ref: "HEAD", + sha: "1234567890abcdef1234567890abcdef12345678", + status: "pass", + }, + pass: true, + }, + }); + expect(manifest.artifacts).toContainEqual( + expect.objectContaining({ + inline: true, + kind: "desktopScreenshot", + path: "web-ui-chat.png", + required: true, + }), + ); + }); + + it("writes a failing manifest without requiring a missing screenshot", () => { + withTempDir((dir) => { + const result = writeWebUiChatEvidence([ + "--output-dir", + dir, + "--candidate-sha", + "abcdef", + "--status", + "fail", + ]); + + expect(existsSync(result.manifestPath)).toBe(true); + expect(existsSync(result.reportPath)).toBe(true); + const manifest = JSON.parse(readFileSync(result.manifestPath, "utf8")); + expect(manifest.comparison.pass).toBe(false); + expect( + manifest.artifacts.find( + (artifact: { path: string }) => artifact.path === "web-ui-chat.png", + ), + ).toMatchObject({ + required: false, + }); + expect(readFileSync(result.reportPath, "utf8")).toContain("Status: fail"); + }); + }); + + it("uses the explicit pass status and includes report output", () => { + withTempDir((dir) => { + writeFileSync(path.join(dir, "web-ui-chat.png"), "png"); + writeFileSync( + path.join(dir, "web-ui-chat-proof.json"), + `${JSON.stringify({ status: "pass" })}\n`, + ); + + const result = writeWebUiChatEvidence([ + "--output-dir", + dir, + "--candidate-ref", + "main", + "--status", + "pass", + ]); + + expect(result.manifest.comparison).toMatchObject({ + candidate: { fixed: true, ref: "main", status: "pass" }, + pass: true, + }); + expect(readFileSync(result.reportPath, "utf8")).toContain( + "Screenshot: `web-ui-chat.png` (present)", + ); + }); + }); +}); diff --git a/test/scripts/mantis-web-ui-chat-proof-workflow.test.ts b/test/scripts/mantis-web-ui-chat-proof-workflow.test.ts new file mode 100644 index 00000000000..8f4f65a0c8b --- /dev/null +++ b/test/scripts/mantis-web-ui-chat-proof-workflow.test.ts @@ -0,0 +1,87 @@ +// Mantis Web UI Chat Proof Workflow tests cover mantis web ui chat proof workflow behavior. +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +const WORKFLOW = ".github/workflows/mantis-web-ui-chat-proof.yml"; + +type WorkflowStep = { + name?: string; + uses?: string; + with?: Record; +}; + +type WorkflowJob = { + permissions?: Record; + steps?: WorkflowStep[]; +}; + +type Workflow = { + jobs?: Record; +}; + +function resolveRequestScript(): string { + const workflow = parse(readFileSync(WORKFLOW, "utf8")) as Workflow; + const steps = workflow.jobs?.resolve_request?.steps ?? []; + const step = steps.find((candidate) => candidate.name === "Resolve ref and target PR"); + if (!step?.with?.script) { + throw new Error("Missing Resolve ref and target PR script"); + } + return step.with.script; +} + +function workflowJob(name: string): WorkflowJob { + const workflow = parse(readFileSync(WORKFLOW, "utf8")) as Workflow; + const job = workflow.jobs?.[name]; + if (!job) { + throw new Error(`Missing ${name} job`); + } + return job; +} + +function candidateOverridePattern(): RegExp { + const script = resolveRequestScript(); + const match = script.match(/const candidateMatch = body\.match\((\/.*\/i)\);/); + if (!match) { + throw new Error("Missing candidate override regex"); + } + return Function(`"use strict"; return ${match[1]};`)() as RegExp; +} + +describe("Mantis Web UI chat proof workflow", () => { + it("keeps candidate execution read-only and installs dependencies only in the candidate", () => { + const job = workflowJob("run_web_ui_chat"); + const setup = job.steps?.find((step) => step.name === "Setup Node environment"); + + expect(job.permissions).toEqual({ contents: "read" }); + expect(setup?.with).toMatchObject({ + "install-bun": "false", + "install-deps": "false", + }); + }); + + it("only treats explicit candidate assignments as PR head overrides", () => { + const pattern = candidateOverridePattern(); + + expect( + "verify this PR head produces a redacted Control UI chat transcript artifact".match( + pattern, + )?.[1], + ).toBeUndefined(); + expect( + "@openclaw-mantis web ui chat proof: verify candidate=e63393c publishes evidence".match( + pattern, + )?.[1], + ).toBe("e63393c"); + expect( + "@openclaw-mantis web ui chat proof: verify head: e63393c publishes evidence".match( + pattern, + )?.[1], + ).toBe("e63393c"); + expect( + "@openclaw-mantis web ui chat proof: verify candidate=`e63393c` publishes evidence".match( + pattern, + )?.[1], + ).toBe("e63393c"); + }); +}); diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index b63af61ac18..3fecc1b7c80 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -18,6 +18,7 @@ const MANTIS_SLACK_DESKTOP_SMOKE_WORKFLOW = ".github/workflows/mantis-slack-desk const MANTIS_TELEGRAM_DESKTOP_PROOF_WORKFLOW = ".github/workflows/mantis-telegram-desktop-proof.yml"; const MANTIS_TELEGRAM_LIVE_WORKFLOW = ".github/workflows/mantis-telegram-live.yml"; +const MANTIS_WEB_UI_CHAT_PROOF_WORKFLOW = ".github/workflows/mantis-web-ui-chat-proof.yml"; const PACKAGE_JSON = "package.json"; const SETUP_PNPM_STORE_CACHE_ACTION = ".github/actions/setup-pnpm-store-cache/action.yml"; const DOCKER_E2E_PLAN_ACTION = ".github/actions/docker-e2e-plan/action.yml"; @@ -1570,6 +1571,11 @@ describe("package artifact reuse", () => { jobName: "run_telegram_live", stepName: "Upload Mantis Telegram artifacts", }, + { + workflowPath: MANTIS_WEB_UI_CHAT_PROOF_WORKFLOW, + jobName: "run_web_ui_chat", + stepName: "Upload Mantis web UI chat artifacts", + }, { workflowPath: NPM_TELEGRAM_WORKFLOW, jobName: "run_package_telegram_e2e", diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index c71e814f192..60004d7c5a0 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -1304,6 +1304,14 @@ describe("scripts/test-projects changed-target routing", () => { "test/scripts/ci-workflow-guards.test.ts", ], ], + [ + ".github/workflows/mantis-web-ui-chat-proof.yml", + [ + "test/scripts/mantis-web-ui-chat-proof-workflow.test.ts", + "test/scripts/package-acceptance-workflow.test.ts", + "test/scripts/ci-workflow-guards.test.ts", + ], + ], ]); for (const [workflowPath, targets] of workflowTargets) { diff --git a/ui/src/e2e/mantis-chat-proof.e2e.test.ts b/ui/src/e2e/mantis-chat-proof.e2e.test.ts new file mode 100644 index 00000000000..157aaef6623 --- /dev/null +++ b/ui/src/e2e/mantis-chat-proof.e2e.test.ts @@ -0,0 +1,125 @@ +// Control UI Mantis proof covers the focused web chat browser path. +import { copyFile, mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { chromium, type Browser, type BrowserContext } from "playwright"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + canRunPlaywrightChromium, + installMockGateway, + resolvePlaywrightChromiumExecutablePath, + startControlUiE2eServer, + type ControlUiE2eServer, +} from "../test-helpers/control-ui-e2e.ts"; + +const chromiumExecutablePath = resolvePlaywrightChromiumExecutablePath(chromium.executablePath()); +const chromiumAvailable = canRunPlaywrightChromium(chromiumExecutablePath); +const allowMissingChromium = process.env.OPENCLAW_UI_E2E_ALLOW_MISSING_CHROMIUM === "1"; +const describeMantisWebUiChat = + chromiumAvailable || !allowMissingChromium ? describe : describe.skip; +const artifactDir = path.resolve( + process.env.OPENCLAW_MANTIS_WEB_UI_CHAT_OUTPUT_DIR ?? + path.join(process.cwd(), ".artifacts", "qa-e2e", "mantis", "web-ui-chat-proof"), +); + +let server: ControlUiE2eServer; +const contextBrowsers = new WeakMap(); + +async function newBrowserContext(options: Parameters[0]) { + const browser = await chromium.launch({ executablePath: chromiumExecutablePath }); + try { + const context = await browser.newContext(options); + contextBrowsers.set(context, browser); + return context; + } catch (error) { + await browser.close().catch(() => {}); + throw error; + } +} + +async function closeBrowserContext(context: BrowserContext): Promise { + const browser = contextBrowsers.get(context); + contextBrowsers.delete(context); + await context.close().catch(() => {}); + await browser?.close().catch(() => {}); +} + +describeMantisWebUiChat("Mantis Control UI web chat proof", () => { + beforeAll(async () => { + if (!chromiumAvailable) { + throw new Error( + `Playwright Chromium is not installed or cannot start at ${chromiumExecutablePath}. Run \`pnpm --dir ui exec playwright install --with-deps chromium\`, set PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH to a compatible browser, or set OPENCLAW_UI_E2E_ALLOW_MISSING_CHROMIUM=1 only when intentionally skipping this lane.`, + ); + } + await mkdir(artifactDir, { recursive: true }); + server = await startControlUiE2eServer(); + }); + + afterAll(async () => { + await server?.close(); + }); + + it("sends a chat message and captures visible browser proof", async () => { + const rawVideoDir = path.join(artifactDir, "raw-video"); + await mkdir(rawVideoDir, { recursive: true }); + const context = await newBrowserContext({ + locale: "en-US", + recordVideo: { dir: rawVideoDir, size: { height: 900, width: 1280 } }, + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, + }); + const page = await context.newPage(); + const gateway = await installMockGateway(page, { + historyMessages: [ + { + content: [{ text: "Mantis web UI proof is ready.", type: "text" }], + role: "assistant", + timestamp: Date.now(), + }, + ], + }); + const prompt = "capture a Mantis web UI chat proof"; + const reply = "Mantis web UI chat proof rendered."; + const startedAt = new Date().toISOString(); + + try { + await page.goto(`${server.baseUrl}chat`); + await page.getByText("Mantis web UI proof is ready.").waitFor({ timeout: 10_000 }); + await page.locator(".agent-chat__composer-combobox textarea").fill(prompt); + await page.getByRole("button", { name: "Send message" }).click(); + + const sendRequest = await gateway.waitForRequest("chat.send"); + expect(sendRequest.params).toMatchObject({ + deliver: false, + message: prompt, + sessionKey: "main", + }); + const params = sendRequest.params as { idempotencyKey?: string }; + expect(params.idempotencyKey).toEqual(expect.any(String)); + + await gateway.emitChatFinal({ runId: params.idempotencyKey ?? "", text: reply }); + await page.getByText(reply).waitFor({ timeout: 10_000 }); + await page.screenshot({ fullPage: true, path: path.join(artifactDir, "web-ui-chat.png") }); + await writeFile( + path.join(artifactDir, "web-ui-chat-proof.json"), + `${JSON.stringify( + { + finishedAt: new Date().toISOString(), + prompt, + reply, + startedAt, + status: "pass", + }, + null, + 2, + )}\n`, + ); + } finally { + const video = page.video(); + await closeBrowserContext(context); + const videoPath = await video?.path().catch(() => undefined); + if (videoPath) { + await copyFile(videoPath, path.join(artifactDir, "web-ui-chat.webm")); + } + } + }); +});