feat(e2e): AI-powered log review catches non-fatal issues (#2875)

* feat(e2e): add AI-powered log review after provisioning

Feeds provision stderr/stdout logs to an LLM after each agent deploys.
Catches non-fatal issues that binary pass/fail checks miss: silent 404s,
failed component installs, connection instability, swallowed warnings.

This would have caught the keep-alive 404 and the sprite idle shutdown
that the existing E2E tests missed because installSpriteKeepAlive() is
non-fatal and the binary checks only verify final state.

- Uses gemini-flash-lite-2.0 via OpenRouter (cheap, fast)
- Advisory only — never fails the test, reports findings as warnings
- Truncates logs to last 200 lines to stay within token limits
- Skips gracefully if OPENROUTER_API_KEY is missing or API fails

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* feat(e2e): add AI log review and --fast mode testing

AI log review:
- After each agent provisions, feeds stderr/stdout to gemini-flash-lite
  to catch non-fatal issues binary checks miss (404s, failed installs,
  connection drops, swallowed warnings)
- Advisory only — never fails the test, surfaces findings as warnings
- Would have caught the keep-alive 404 and sprite idle shutdown

--fast mode E2E:
- Add --fast flag to e2e.sh, passed through to spawn CLI during provision
- Update QA e2e-tester protocol to run both normal and --fast passes
- --fast enables images + tarballs + parallel boot

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: L <6723574+louisgv@users.noreply.github.com>

.claude/skills/setup-agent-team/qa-quality-prompt.md

@@ -129,9 +129,12 @@ cd REPO_ROOT_PLACEHOLDER && git worktree remove WORKTREE_BASE_PLACEHOLDER/TASK_N
    ```bash
    cd REPO_ROOT_PLACEHOLDER
    chmod +x sh/e2e/e2e.sh
+   # Normal mode — standard provisioning
    ./sh/e2e/e2e.sh --cloud all --parallel 6 --skip-input-test
+   # Fast mode — tests --fast flag (images + tarballs + parallel boot)
+   ./sh/e2e/e2e.sh --cloud sprite --fast --parallel 4 --skip-input-test
    ```
-2. Capture the full output. Note which clouds ran, which agents passed, which failed, and which clouds were skipped (no credentials).
+2. Capture the full output from BOTH runs. Note which clouds ran, which agents passed, which failed, and which clouds were skipped (no credentials).
 3. If all configured clouds pass (or only skipped clouds): report results and you're done. No PR needed.
 4. If any agent fails on a configured cloud, investigate the root cause. Failure categories:
 

sh/e2e/e2e.sh

@@ -32,6 +32,7 @@ source "${SCRIPT_DIR}/lib/verify.sh"
 source "${SCRIPT_DIR}/lib/teardown.sh"
 source "${SCRIPT_DIR}/lib/soak.sh"
 source "${SCRIPT_DIR}/lib/interactive.sh"
+source "${SCRIPT_DIR}/lib/ai-review.sh"
 
 # ---------------------------------------------------------------------------
 # All supported clouds (excluding local — no infra to provision)
@@ -49,6 +50,7 @@ SKIP_INPUT_TEST="${SKIP_INPUT_TEST:-0}"
 SEQUENTIAL_MODE=0
 SOAK_MODE=0
 INTERACTIVE_MODE=0
+FAST_MODE=0
 
 while [ $# -gt 0 ]; do
   case "$1" in
@@ -114,6 +116,10 @@ while [ $# -gt 0 ]; do
       INTERACTIVE_MODE=1
       shift
       ;;
+    --fast)
+      FAST_MODE=1
+      shift
+      ;;
     --help|-h)
       printf "Usage: %s --cloud CLOUD [--cloud CLOUD2 ...] [agents...] [options]\n\n" "$0"
       printf "Clouds: %s\n" "${ALL_CLOUDS}"
@@ -125,6 +131,7 @@ while [ $# -gt 0 ]; do
       printf "  --sequential        Force sequential agent execution\n"
       printf "  --skip-cleanup      Skip stale e2e-* instance cleanup\n"
       printf "  --skip-input-test   Skip live input tests\n"
+      printf "  --fast              Provision with --fast flag (images + tarballs + parallel)\n"
       printf "  --soak              Run Telegram soak test (OpenClaw on Sprite)\n"
       printf "  --interactive       AI-driven interactive test (requires ANTHROPIC_API_KEY)\n"
       printf "  --help              Show this help\n"
@@ -228,6 +235,8 @@ run_single_agent() {
     else
       # Standard headless mode
       if provision_agent "${agent}" "${app_name}" "${LOG_DIR}"; then
+        # AI review of provision logs — advisory only, runs regardless of verify result
+        ai_review_logs "${agent}" "${app_name}" "${LOG_DIR}" || true
         if verify_agent "${agent}" "${app_name}"; then
           if run_input_test "${agent}" "${app_name}"; then
             _inner_status="pass"
@@ -639,6 +648,12 @@ fi
 if [ "${SKIP_INPUT_TEST}" -eq 1 ]; then
   log_info "Input tests: SKIPPED"
 fi
+if [ "${FAST_MODE}" -eq 1 ]; then
+  log_info "Fast mode: ENABLED (--fast passed to spawn)"
+fi
+
+# Export FAST_MODE so provision.sh can read it
+export E2E_FAST_MODE="${FAST_MODE}"
 
 # Create temp log directory
 LOG_DIR=$(mktemp -d "${TMPDIR:-/tmp}/spawn-e2e.XXXXXX")

sh/e2e/lib/ai-review.sh

@@ -0,0 +1,157 @@
+#!/bin/bash
+# e2e/lib/ai-review.sh — AI-powered log analysis for E2E test output
+#
+# After provision + verify pass, feeds stderr/stdout logs to an LLM to catch
+# non-fatal issues that binary pass/fail checks miss: silent 404s, degraded
+# installs, swallowed warnings, connection instability, etc.
+#
+# Requires: OPENROUTER_API_KEY (reuses the same key used for E2E provisioning)
+# Skips gracefully if the key is missing or the API call fails.
+set -eo pipefail
+
+# ---------------------------------------------------------------------------
+# ai_review_logs AGENT APP_NAME LOG_DIR
+#
+# Analyzes provision logs for an agent and reports findings as warnings.
+# Returns 0 always (advisory only — never fails the test).
+# ---------------------------------------------------------------------------
+ai_review_logs() {
+  local agent="$1"
+  local app_name="$2"
+  local log_dir="$3"
+
+  local api_key="${OPENROUTER_API_KEY:-}"
+  if [ -z "${api_key}" ]; then
+    return 0
+  fi
+
+  local stdout_file="${log_dir}/${app_name}.stdout"
+  local stderr_file="${log_dir}/${app_name}.stderr"
+
+  # Collect log content (truncate to last 200 lines each to stay within token limits)
+  local log_content=""
+  if [ -f "${stderr_file}" ] && [ -s "${stderr_file}" ]; then
+    log_content="=== STDERR (last 200 lines) ===
+$(tail -200 "${stderr_file}" 2>/dev/null || true)
+"
+  fi
+  if [ -f "${stdout_file}" ] && [ -s "${stdout_file}" ]; then
+    log_content="${log_content}=== STDOUT (last 200 lines) ===
+$(tail -200 "${stdout_file}" 2>/dev/null || true)
+"
+  fi
+
+  # Skip if no log content
+  if [ -z "${log_content}" ]; then
+    return 0
+  fi
+
+  log_step "AI reviewing ${agent} logs..."
+
+  # Build the prompt
+  local system_prompt='You are a QA engineer reviewing deployment logs from an automated E2E test of "spawn" — a tool that provisions cloud VMs and installs AI coding agents.
+
+Your job: find issues that passed the binary tests but indicate degraded or broken behavior. Focus on:
+- HTTP errors (404, 500, timeouts) even if the step was marked non-fatal
+- Failed installations of components (keep-alive scripts, browser, plugins)
+- Connection drops, retries, or timeouts during provisioning
+- Warnings that indicate missing functionality
+- Security warnings (exposed credentials, insecure connections)
+- Package deprecation warnings that could break future builds
+
+Do NOT flag:
+- Normal npm deprecation warnings for transient dependencies (these are upstream)
+- Successful retries (only flag if all retries failed)
+- Expected "non-interactive" or "headless" mode messages
+- Informational step progress messages
+
+Output format: If you find issues, output one line per issue:
+ISSUE: <severity:low|medium|high> <brief description>
+
+If no issues found, output exactly: NO_ISSUES
+
+Be concise. Max 5 issues.'
+
+  # Use a temp file for the request body to avoid shell quoting issues
+  local req_file
+  req_file=$(mktemp /tmp/e2e-ai-review-XXXXXX.json)
+
+  # Build JSON safely via bun to avoid shell injection
+  local ts_file
+  ts_file=$(mktemp /tmp/e2e-ai-build-XXXXXX.ts)
+  cat > "${ts_file}" << 'TS_EOF'
+const system = process.env._AI_SYSTEM ?? "";
+const logs = process.env._AI_LOGS ?? "";
+const agent = process.env._AI_AGENT ?? "";
+const outFile = process.env._AI_OUT ?? "";
+
+const body = {
+  model: "google/gemini-flash-lite-2.0",
+  max_tokens: 512,
+  messages: [
+    { role: "system", content: system },
+    { role: "user", content: `Agent: ${agent}\n\nDeployment logs:\n\n${logs}` },
+  ],
+};
+
+await Bun.write(outFile, JSON.stringify(body));
+TS_EOF
+
+  _AI_SYSTEM="${system_prompt}" \
+  _AI_LOGS="${log_content}" \
+  _AI_AGENT="${agent}" \
+  _AI_OUT="${req_file}" \
+    bun run "${ts_file}" 2>/dev/null
+
+  rm -f "${ts_file}" 2>/dev/null || true
+
+  # Call OpenRouter API
+  local response
+  response=$(curl -sf --max-time 30 \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Bearer ${api_key}" \
+    -d @"${req_file}" \
+    "https://openrouter.ai/api/v1/chat/completions" 2>/dev/null) || {
+    rm -f "${req_file}" 2>/dev/null || true
+    log_warn "AI review skipped (API call failed)"
+    return 0
+  }
+
+  rm -f "${req_file}" 2>/dev/null || true
+
+  # Extract the response content
+  local ai_output
+  ai_output=$(printf '%s' "${response}" | bun -e "
+    const data = JSON.parse(await Bun.stdin.text());
+    const content = data?.choices?.[0]?.message?.content ?? '';
+    process.stdout.write(content);
+  " 2>/dev/null) || {
+    log_warn "AI review skipped (failed to parse response)"
+    return 0
+  }
+
+  # Parse and report findings
+  if printf '%s' "${ai_output}" | grep -q "NO_ISSUES"; then
+    log_ok "AI review: no issues found"
+    return 0
+  fi
+
+  # Report each issue as a warning
+  local issue_count=0
+  while IFS= read -r line; do
+    case "${line}" in
+      ISSUE:*)
+        issue_count=$((issue_count + 1))
+        log_warn "AI review: ${line#ISSUE: }"
+        ;;
+    esac
+  done <<< "${ai_output}"
+
+  if [ "${issue_count}" -eq 0 ]; then
+    log_ok "AI review: no issues found"
+  else
+    log_warn "AI review: ${issue_count} issue(s) found for ${agent}"
+  fi
+
+  return 0
+}

sh/e2e/lib/provision.sh

@@ -112,7 +112,12 @@ provision_agent() {
 $(cloud_headless_env "${app_name}" "${agent}")
 CLOUD_ENV
 
-    bun run "${cli_entry}" "${agent}" "${ACTIVE_CLOUD}" --headless --output json \
+    # Build CLI args — add --fast when E2E_FAST_MODE is enabled
+    _cli_args="${agent} ${ACTIVE_CLOUD} --headless --output json"
+    if [ "${E2E_FAST_MODE:-0}" = "1" ]; then
+      _cli_args="${_cli_args} --fast"
+    fi
+    bun run "${cli_entry}" ${_cli_args} \
       > "${stdout_file}" 2> "${stderr_file}"
     printf '%s' "$?" > "${exit_file}"
   ) &