diff --git a/.claude/skills/setup-agent-team/qa-quality-prompt.md b/.claude/skills/setup-agent-team/qa-quality-prompt.md index 28bd2a16..fc489e1a 100644 --- a/.claude/skills/setup-agent-team/qa-quality-prompt.md +++ b/.claude/skills/setup-agent-team/qa-quality-prompt.md @@ -129,9 +129,12 @@ cd REPO_ROOT_PLACEHOLDER && git worktree remove WORKTREE_BASE_PLACEHOLDER/TASK_N ```bash cd REPO_ROOT_PLACEHOLDER chmod +x sh/e2e/e2e.sh + # Normal mode — standard provisioning ./sh/e2e/e2e.sh --cloud all --parallel 6 --skip-input-test + # Fast mode — tests --fast flag (images + tarballs + parallel boot) + ./sh/e2e/e2e.sh --cloud sprite --fast --parallel 4 --skip-input-test ``` -2. Capture the full output. Note which clouds ran, which agents passed, which failed, and which clouds were skipped (no credentials). +2. Capture the full output from BOTH runs. Note which clouds ran, which agents passed, which failed, and which clouds were skipped (no credentials). 3. If all configured clouds pass (or only skipped clouds): report results and you're done. No PR needed. 4. If any agent fails on a configured cloud, investigate the root cause. Failure categories: diff --git a/sh/e2e/e2e.sh b/sh/e2e/e2e.sh index a5eb0c8e..8dd76962 100755 --- a/sh/e2e/e2e.sh +++ b/sh/e2e/e2e.sh @@ -32,6 +32,7 @@ source "${SCRIPT_DIR}/lib/verify.sh" source "${SCRIPT_DIR}/lib/teardown.sh" source "${SCRIPT_DIR}/lib/soak.sh" source "${SCRIPT_DIR}/lib/interactive.sh" +source "${SCRIPT_DIR}/lib/ai-review.sh" # --------------------------------------------------------------------------- # All supported clouds (excluding local — no infra to provision) @@ -49,6 +50,7 @@ SKIP_INPUT_TEST="${SKIP_INPUT_TEST:-0}" SEQUENTIAL_MODE=0 SOAK_MODE=0 INTERACTIVE_MODE=0 +FAST_MODE=0 while [ $# -gt 0 ]; do case "$1" in @@ -114,6 +116,10 @@ while [ $# -gt 0 ]; do INTERACTIVE_MODE=1 shift ;; + --fast) + FAST_MODE=1 + shift + ;; --help|-h) printf "Usage: %s --cloud CLOUD [--cloud CLOUD2 ...] [agents...] [options]\n\n" "$0" printf "Clouds: %s\n" "${ALL_CLOUDS}" @@ -125,6 +131,7 @@ while [ $# -gt 0 ]; do printf " --sequential Force sequential agent execution\n" printf " --skip-cleanup Skip stale e2e-* instance cleanup\n" printf " --skip-input-test Skip live input tests\n" + printf " --fast Provision with --fast flag (images + tarballs + parallel)\n" printf " --soak Run Telegram soak test (OpenClaw on Sprite)\n" printf " --interactive AI-driven interactive test (requires ANTHROPIC_API_KEY)\n" printf " --help Show this help\n" @@ -228,6 +235,8 @@ run_single_agent() { else # Standard headless mode if provision_agent "${agent}" "${app_name}" "${LOG_DIR}"; then + # AI review of provision logs — advisory only, runs regardless of verify result + ai_review_logs "${agent}" "${app_name}" "${LOG_DIR}" || true if verify_agent "${agent}" "${app_name}"; then if run_input_test "${agent}" "${app_name}"; then _inner_status="pass" @@ -639,6 +648,12 @@ fi if [ "${SKIP_INPUT_TEST}" -eq 1 ]; then log_info "Input tests: SKIPPED" fi +if [ "${FAST_MODE}" -eq 1 ]; then + log_info "Fast mode: ENABLED (--fast passed to spawn)" +fi + +# Export FAST_MODE so provision.sh can read it +export E2E_FAST_MODE="${FAST_MODE}" # Create temp log directory LOG_DIR=$(mktemp -d "${TMPDIR:-/tmp}/spawn-e2e.XXXXXX") diff --git a/sh/e2e/lib/ai-review.sh b/sh/e2e/lib/ai-review.sh new file mode 100644 index 00000000..da7af138 --- /dev/null +++ b/sh/e2e/lib/ai-review.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# e2e/lib/ai-review.sh — AI-powered log analysis for E2E test output +# +# After provision + verify pass, feeds stderr/stdout logs to an LLM to catch +# non-fatal issues that binary pass/fail checks miss: silent 404s, degraded +# installs, swallowed warnings, connection instability, etc. +# +# Requires: OPENROUTER_API_KEY (reuses the same key used for E2E provisioning) +# Skips gracefully if the key is missing or the API call fails. +set -eo pipefail + +# --------------------------------------------------------------------------- +# ai_review_logs AGENT APP_NAME LOG_DIR +# +# Analyzes provision logs for an agent and reports findings as warnings. +# Returns 0 always (advisory only — never fails the test). +# --------------------------------------------------------------------------- +ai_review_logs() { + local agent="$1" + local app_name="$2" + local log_dir="$3" + + local api_key="${OPENROUTER_API_KEY:-}" + if [ -z "${api_key}" ]; then + return 0 + fi + + local stdout_file="${log_dir}/${app_name}.stdout" + local stderr_file="${log_dir}/${app_name}.stderr" + + # Collect log content (truncate to last 200 lines each to stay within token limits) + local log_content="" + if [ -f "${stderr_file}" ] && [ -s "${stderr_file}" ]; then + log_content="=== STDERR (last 200 lines) === +$(tail -200 "${stderr_file}" 2>/dev/null || true) +" + fi + if [ -f "${stdout_file}" ] && [ -s "${stdout_file}" ]; then + log_content="${log_content}=== STDOUT (last 200 lines) === +$(tail -200 "${stdout_file}" 2>/dev/null || true) +" + fi + + # Skip if no log content + if [ -z "${log_content}" ]; then + return 0 + fi + + log_step "AI reviewing ${agent} logs..." + + # Build the prompt + local system_prompt='You are a QA engineer reviewing deployment logs from an automated E2E test of "spawn" — a tool that provisions cloud VMs and installs AI coding agents. + +Your job: find issues that passed the binary tests but indicate degraded or broken behavior. Focus on: +- HTTP errors (404, 500, timeouts) even if the step was marked non-fatal +- Failed installations of components (keep-alive scripts, browser, plugins) +- Connection drops, retries, or timeouts during provisioning +- Warnings that indicate missing functionality +- Security warnings (exposed credentials, insecure connections) +- Package deprecation warnings that could break future builds + +Do NOT flag: +- Normal npm deprecation warnings for transient dependencies (these are upstream) +- Successful retries (only flag if all retries failed) +- Expected "non-interactive" or "headless" mode messages +- Informational step progress messages + +Output format: If you find issues, output one line per issue: +ISSUE: + +If no issues found, output exactly: NO_ISSUES + +Be concise. Max 5 issues.' + + # Use a temp file for the request body to avoid shell quoting issues + local req_file + req_file=$(mktemp /tmp/e2e-ai-review-XXXXXX.json) + + # Build JSON safely via bun to avoid shell injection + local ts_file + ts_file=$(mktemp /tmp/e2e-ai-build-XXXXXX.ts) + cat > "${ts_file}" << 'TS_EOF' +const system = process.env._AI_SYSTEM ?? ""; +const logs = process.env._AI_LOGS ?? ""; +const agent = process.env._AI_AGENT ?? ""; +const outFile = process.env._AI_OUT ?? ""; + +const body = { + model: "google/gemini-flash-lite-2.0", + max_tokens: 512, + messages: [ + { role: "system", content: system }, + { role: "user", content: `Agent: ${agent}\n\nDeployment logs:\n\n${logs}` }, + ], +}; + +await Bun.write(outFile, JSON.stringify(body)); +TS_EOF + + _AI_SYSTEM="${system_prompt}" \ + _AI_LOGS="${log_content}" \ + _AI_AGENT="${agent}" \ + _AI_OUT="${req_file}" \ + bun run "${ts_file}" 2>/dev/null + + rm -f "${ts_file}" 2>/dev/null || true + + # Call OpenRouter API + local response + response=$(curl -sf --max-time 30 \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${api_key}" \ + -d @"${req_file}" \ + "https://openrouter.ai/api/v1/chat/completions" 2>/dev/null) || { + rm -f "${req_file}" 2>/dev/null || true + log_warn "AI review skipped (API call failed)" + return 0 + } + + rm -f "${req_file}" 2>/dev/null || true + + # Extract the response content + local ai_output + ai_output=$(printf '%s' "${response}" | bun -e " + const data = JSON.parse(await Bun.stdin.text()); + const content = data?.choices?.[0]?.message?.content ?? ''; + process.stdout.write(content); + " 2>/dev/null) || { + log_warn "AI review skipped (failed to parse response)" + return 0 + } + + # Parse and report findings + if printf '%s' "${ai_output}" | grep -q "NO_ISSUES"; then + log_ok "AI review: no issues found" + return 0 + fi + + # Report each issue as a warning + local issue_count=0 + while IFS= read -r line; do + case "${line}" in + ISSUE:*) + issue_count=$((issue_count + 1)) + log_warn "AI review: ${line#ISSUE: }" + ;; + esac + done <<< "${ai_output}" + + if [ "${issue_count}" -eq 0 ]; then + log_ok "AI review: no issues found" + else + log_warn "AI review: ${issue_count} issue(s) found for ${agent}" + fi + + return 0 +} diff --git a/sh/e2e/lib/provision.sh b/sh/e2e/lib/provision.sh index 8519122c..b90580ed 100644 --- a/sh/e2e/lib/provision.sh +++ b/sh/e2e/lib/provision.sh @@ -112,7 +112,12 @@ provision_agent() { $(cloud_headless_env "${app_name}" "${agent}") CLOUD_ENV - bun run "${cli_entry}" "${agent}" "${ACTIVE_CLOUD}" --headless --output json \ + # Build CLI args — add --fast when E2E_FAST_MODE is enabled + _cli_args="${agent} ${ACTIVE_CLOUD} --headless --output json" + if [ "${E2E_FAST_MODE:-0}" = "1" ]; then + _cli_args="${_cli_args} --fast" + fi + bun run "${cli_entry}" ${_cli_args} \ > "${stdout_file}" 2> "${stderr_file}" printf '%s' "$?" > "${exit_file}" ) &