fix(ci): harden tmux triage reporting (#5548)

* fix(ci): harden tmux triage reporting

* fix(ci): address tmux triage review feedback

* fix(ci): tighten tmux triage review followups

* fix(ci): address tmux triage review followups

* fix(ci): simplify tmux triage hardening

* fix(ci): sanitize tmux triage verdict output

* fix(ci): harden tmux result rendering

* ci: strip step summary from triage build

* fix(ci): report missing tmux artifacts

* fix(ci): clarify tmux prepare diagnostics

.github/workflows/qwen-triage.yml

@@ -272,11 +272,11 @@ jobs:
       verdict: '${{ steps.run.outputs.verdict || steps.prepare.outputs.verdict || steps.pr.outputs.verdict }}'
       failure_phase: '${{ steps.prepare.outputs.failure_phase }}'
     steps:
-      - name: 'Install tmux runner tools'
+      - name: 'Install PR resolver tools'
         run: |-
           set -euo pipefail
           apt-get update
-          apt-get install -y --no-install-recommends ca-certificates curl git gnupg jq tmux util-linux
+          apt-get install -y --no-install-recommends ca-certificates curl git gnupg jq
 
           install -d -m 755 /etc/apt/keyrings
           curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \
@@ -287,10 +287,7 @@ jobs:
           apt-get update
           apt-get install -y --no-install-recommends gh
 
-          npm install -g '@qwen-code/qwen-code@latest'
-          qwen --version
           gh --version
-          tmux -V
 
       - name: 'Resolve PR and check state'
         id: 'pr'
@@ -348,6 +345,17 @@ jobs:
             echo "decision=na" >> "$GITHUB_OUTPUT"
           fi
 
+      # Install before checkout so PR-controlled .npmrc cannot affect npm.
+      - name: 'Install tmux runner tools'
+        if: "steps.pr.outputs.decision == 'run'"
+        run: |-
+          set -euo pipefail
+          apt-get install -y --no-install-recommends tmux util-linux
+
+          npm install -g --registry=https://registry.npmjs.org '@qwen-code/qwen-code@latest'
+          qwen --version
+          tmux -V
+
       - name: 'Clean stale review worktrees'
         if: "steps.pr.outputs.decision == 'run'"
         run: |-
@@ -381,13 +389,15 @@ jobs:
           set +e
           {
             printf '%s\n' '$ npm ci --prefer-offline --no-audit --progress=false'
-            runuser -u node -- npm ci --prefer-offline --no-audit --progress=false
+            runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \
+              npm ci --prefer-offline --no-audit --progress=false
             install_status=$?
             if [ "$install_status" -ne 0 ]; then
               printf '\n%s\n' "npm ci failed with exit code ${install_status}."
             else
               printf '\n%s\n' '$ npm run build'
-              runuser -u node -- npm run build
+              runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \
+                npm run build
               build_status=$?
               if [ "$build_status" -ne 0 ]; then
                 printf '\n%s\n' "npm run build failed with exit code ${build_status}."
@@ -663,6 +673,9 @@ jobs:
             "QWEN_CI_REAL_GH=${QWEN_CI_REAL_GH:-}"
             "QWEN_CI_REAL_GIT=${QWEN_CI_REAL_GIT:-}"
           )
+          if [ -n "${OPENAI_MODEL:-}" ]; then
+            QWEN_ENV+=("OPENAI_MODEL=$OPENAI_MODEL")
+          fi
 
           set +e
           timeout --kill-after=10s 20m runuser -u node -- env -i "${QWEN_ENV[@]}" "${QWEN_CMD[@]}" \
@@ -771,17 +784,27 @@ jobs:
           # as markdown — which a
           # backtick fence (breakable by a long enough ``` run) cannot guarantee.
           # Order matters: escape & first so the < / > entities aren't re-escaped.
+          html_escape() {
+            sed -e 's/&/\&amp;/g' -e 's/</\&lt;/g' -e 's/>/\&gt;/g'
+          }
+
           emit_block() {
-            local summary="$1" file="$2" max="$3" content
+            local summary="$1" file="$2" max="$3" content truncated='' summary_html
             [ -n "$file" ] && [ -f "$file" ] || return 0
-            content="$(head -c "$max" "$file")"
+            summary_html="$(printf '%s' "$summary" | html_escape)"
             if [ "$(wc -c < "$file")" -gt "$max" ]; then
-              content="${content}"$'\n\n...truncated -- full log in the run artifacts.'
+              truncated=$'\n\n...truncated -- full log in the run artifacts.'
             fi
-            content="${content//&/&amp;}"
-            content="${content//</&lt;}"
-            content="${content//>/&gt;}"
-            printf '<details>\n<summary>%s</summary>\n\n<pre><code>\n' "$summary"
+            if ! content="$(
+              set -o pipefail
+              head -c "$max" "$file" | tr -d '\000' | html_escape
+            )"; then
+              echo "::warning::emit_block failed while rendering $summary; see run artifacts." >&2
+              content='<em>Log could not be rendered; see run artifacts.</em>'
+            elif [ -n "$truncated" ]; then
+              content="${content}${truncated}"
+            fi
+            printf '<details>\n<summary>%s</summary>\n\n<pre><code>\n' "$summary_html"
             printf '%s\n' "$content"
             printf '</code></pre>\n\n</details>\n\n'
           }
@@ -807,22 +830,68 @@ jobs:
             case "$PREPARE_FAILURE_PHASE" in
               install) PREPARE_COMMAND='npm ci' ;;
               build) PREPARE_COMMAND='npm run build' ;;
-              *) PREPARE_COMMAND="$PREPARE_FAILURE_PHASE" ;;
+              *)
+                PREPARE_COMMAND='install/build'
+                UNKNOWN_PREPARE_PHASE="$(
+                  printf '%s' "$PREPARE_FAILURE_PHASE" | tr -d '\000' | tr '\r\n' '  ' | head -c 200 | html_escape
+                )"
+                echo "::warning::Unrecognized prepare failure phase: ${UNKNOWN_PREPARE_PHASE}"
+                ;;
             esac
+            if [ -z "$PREPARE_LOG" ]; then
+              PREPARE_LOG_NOTE='No prepare.log was found in tmux-results, so the install/build log section is omitted.'
+              echo "::warning::${PREPARE_LOG_NOTE}"
+            fi
             {
               printf '%s\n\n' '<!-- qwen-triage:tmux -->'
               printf '**tmux real-user testing: fail** - [workflow run](%s)\n\n' "$RUN_URL"
               printf 'The PR app could not be launched because `%s` failed before the tmux session started. This is treated as a PR failure verdict rather than an infrastructure failure.\n\n' "$PREPARE_COMMAND"
+              if [ -n "${PREPARE_LOG_NOTE:-}" ]; then
+                printf '%s\n\n' "$PREPARE_LOG_NOTE"
+              fi
               emit_block 'Install/build log' "$PREPARE_LOG" 20000
               printf '%s\n' '— _Qwen Code · tmux real-user testing_'
             } > "$BODY_FILE"
           else
             REPORT="$(find tmux-results -name 'report.md' 2>/dev/null | head -1 || true)"
             TRANSCRIPT="$(find tmux-results -name 'tmux-readable-full.log' 2>/dev/null | head -1 || true)"
+            if [ -z "$REPORT" ] && [ -z "$TRANSCRIPT" ]; then
+              MISSING_ARTIFACTS_NOTE='No report.md or tmux-readable-full.log was found in tmux-results, so detailed report sections are omitted.'
+              echo "::warning::${MISSING_ARTIFACTS_NOTE}"
+            fi
+            case "${VERDICT:-}" in
+              infra-error)
+                VERDICT_LABEL='infra-error (crash/OOM)'
+                DESCRIPTION='The tmux test did not complete because the qwen process failed or was killed. This is not a pass/fail result for the affected flow; check runner resources and PR code for crashes or memory leaks.'
+                ;;
+              timeout)
+                VERDICT_LABEL='timeout'
+                DESCRIPTION='The tmux test did not complete before the time limit. This is not a pass/fail result for the affected flow; see the workflow run and artifacts for details.'
+                ;;
+              pass)
+                VERDICT_LABEL='pass'
+                DESCRIPTION='Launched the changed app in a real tmux session and exercised the affected flow.'
+                ;;
+              fail)
+                VERDICT_LABEL='fail'
+                DESCRIPTION='Launched the changed app in a real tmux session and exercised the affected flow.'
+                ;;
+              *)
+                VERDICT_LABEL='unknown'
+                UNKNOWN_VERDICT="$(
+                  printf '%s' "${VERDICT:-}" | tr -d '\000' | tr '\r\n' '  ' | head -c 200 | html_escape
+                )"
+                echo "::warning::Unrecognized tmux verdict: ${UNKNOWN_VERDICT}"
+                DESCRIPTION="The tmux test produced an unrecognized verdict (<code>${UNKNOWN_VERDICT}</code>), so this is not a pass/fail result for the affected flow. See the workflow run and artifacts for details."
+                ;;
+            esac
             {
               printf '%s\n\n' '<!-- qwen-triage:tmux -->'
-              printf '**tmux real-user testing: %s** - [workflow run](%s)\n\n' "$VERDICT" "$RUN_URL"
-              printf 'Launched the changed app in a real tmux session and exercised the affected flow.\n\n'
+              printf '**tmux real-user testing: %s** - [workflow run](%s)\n\n' "$VERDICT_LABEL" "$RUN_URL"
+              printf '%s\n\n' "$DESCRIPTION"
+              if [ -n "${MISSING_ARTIFACTS_NOTE:-}" ]; then
+                printf '%s\n\n' "$MISSING_ARTIFACTS_NOTE"
+              fi
               emit_block 'E2E test report' "$REPORT" 20000
               emit_block 'Full tmux transcript' "$TRANSCRIPT" 30000
               printf '%s\n' '— _Qwen Code · tmux real-user testing_'

scripts/tests/qwen-triage-workflow.test.js

@@ -0,0 +1,142 @@
+/**
+ * @license
+ * Copyright 2025 Qwen Team
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { readFileSync } from 'node:fs';
+import { describe, expect, it } from 'vitest';
+
+const workflow = readFileSync('.github/workflows/qwen-triage.yml', 'utf8');
+
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function step(name) {
+  const escaped = escapeRegExp(name);
+  const match = workflow.match(
+    new RegExp(
+      `\\n\\s+- name:\\s*(['"])${escaped}\\1[\\s\\S]*?(?=\\n\\s+- name:\\s*['"]|\\n\\s{2}[a-zA-Z0-9_-]+:|$)`,
+    ),
+  );
+  return match?.[0] ?? '';
+}
+
+describe('qwen-triage tmux workflow', () => {
+  it('escapes embedded tmux artifacts without bash pattern replacement ampersands', () => {
+    const postStep = step('Post tmux result comment');
+
+    expect(postStep).not.toContain('content="${content//&/&}"');
+    expect(postStep).not.toContain('content="${content//</&lt;}"');
+    expect(postStep).not.toContain('content="${content//>/&gt;}"');
+    expect(postStep).toContain("sed -e 's/&/\\&amp;/g'");
+    expect(postStep).toContain("-e 's/</\\&lt;/g'");
+    expect(postStep).toContain("-e 's/>/\\&gt;/g'");
+    expect(postStep).toContain('html_escape()');
+    expect(postStep).toContain("tr -d '\\000'");
+    expect(postStep).toContain('Log could not be rendered');
+    expect(postStep).toContain('if ! content="$(');
+    expect(postStep).toContain('set -o pipefail');
+    expect(postStep).toContain('::warning::emit_block failed');
+    expect(postStep).toContain(
+      'summary_html="$(printf \'%s\' "$summary" | html_escape)"',
+    );
+    expect(postStep).toContain(
+      '\'<details>\\n<summary>%s</summary>\\n\\n<pre><code>\\n\' "$summary_html"',
+    );
+  });
+
+  it('passes the selected OpenAI model into the app under tmux test', () => {
+    const runStep = step('Run tmux real-user testing');
+
+    expect(runStep).toContain('if [ -n "${OPENAI_MODEL:-}" ]; then');
+    expect(runStep).toContain('"OPENAI_MODEL=$OPENAI_MODEL"');
+  });
+
+  it('reports timeout and infra-error without claiming the flow was exercised', () => {
+    const postStep = step('Post tmux result comment');
+
+    expect(postStep).toContain('case "${VERDICT:-}" in');
+    expect(postStep).toContain("VERDICT_LABEL='infra-error (crash/OOM)'");
+    expect(postStep).toContain("VERDICT_LABEL='timeout'");
+    expect(postStep).toContain("VERDICT_LABEL='pass'");
+    expect(postStep).toContain("VERDICT_LABEL='fail'");
+    expect(postStep).toContain("VERDICT_LABEL='unknown'");
+    expect(postStep).toContain('The tmux test did not complete');
+    expect(postStep).toContain(
+      'The tmux test did not complete before the time limit',
+    );
+    expect(postStep).toContain('not a pass/fail result');
+    expect(postStep).toContain('crashes or memory leaks');
+    expect(postStep).toContain(
+      'Launched the changed app in a real tmux session and exercised the affected flow.',
+    );
+    expect(postStep).toContain('produced an unrecognized verdict');
+    expect(postStep).toContain('UNKNOWN_VERDICT="$(');
+    expect(postStep).toContain('::warning::Unrecognized tmux verdict');
+    expect(postStep).toContain("tr '\\r\\n' '  '");
+    expect(postStep).toContain('<code>${UNKNOWN_VERDICT}</code>');
+    expect(postStep).toContain('"$VERDICT_LABEL" "$RUN_URL"');
+    expect(postStep).toContain('printf \'%s\\n\\n\' "$DESCRIPTION"');
+    expect(postStep).toContain('MISSING_ARTIFACTS_NOTE=');
+    expect(postStep).toContain(
+      'No report.md or tmux-readable-full.log was found in tmux-results',
+    );
+  });
+
+  it('removes GitHub command files from PR-controlled lifecycle scripts', () => {
+    const prepareStep = step('Install and build PR app');
+    const strippedEnv =
+      'env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY';
+
+    expect(prepareStep).toContain('-u GITHUB_OUTPUT');
+    expect(prepareStep).toContain('-u GITHUB_STATE');
+    expect(prepareStep).toContain('-u GITHUB_ENV');
+    expect(prepareStep).toContain('-u GITHUB_PATH');
+    expect(prepareStep).toContain('-u GITHUB_STEP_SUMMARY');
+    expect(prepareStep).toMatch(
+      new RegExp(`${escapeRegExp(strippedEnv)} \\\\\\s+npm ci`),
+    );
+    expect(prepareStep).toMatch(
+      new RegExp(`${escapeRegExp(strippedEnv)} \\\\\\s+npm run build`),
+    );
+  });
+
+  it('does not echo unrecognized prepare failure phases into comments', () => {
+    const postStep = step('Post tmux result comment');
+
+    expect(postStep).toContain("PREPARE_COMMAND='install/build'");
+    expect(postStep).toContain('::warning::Unrecognized prepare failure phase');
+    expect(postStep).toContain('PREPARE_LOG_NOTE=');
+    expect(postStep).toContain(
+      'No prepare.log was found in tmux-results, so the install/build log section is omitted.',
+    );
+    expect(postStep).not.toContain('PREPARE_COMMAND="$PREPARE_FAILURE_PHASE"');
+  });
+
+  it('installs the heavy tmux test harness only for runnable PRs', () => {
+    const installStep = step('Install tmux runner tools');
+    const resolverStep = step('Install PR resolver tools');
+
+    expect(resolverStep).toContain('apt-get install');
+    expect(installStep).toContain('if: "steps.pr.outputs.decision == \'run\'"');
+    expect(installStep).toContain(
+      'apt-get install -y --no-install-recommends tmux util-linux',
+    );
+    expect(installStep).toContain(
+      "npm install -g --registry=https://registry.npmjs.org '@qwen-code/qwen-code@latest'",
+    );
+    expect(installStep).toContain('qwen --version');
+    expect(installStep).toContain('tmux -V');
+    expect(resolverStep).not.toContain('tmux');
+    expect(resolverStep).not.toContain('npm install');
+    expect(resolverStep).not.toContain('qwen --version');
+    expect(
+      workflow.indexOf("- name: 'Resolve PR and check state'"),
+    ).toBeLessThan(workflow.indexOf("- name: 'Install tmux runner tools'"));
+    expect(
+      workflow.indexOf("- name: 'Install tmux runner tools'"),
+    ).toBeLessThan(workflow.indexOf("- name: 'Checkout PR merge ref'"));
+  });
+});