From 48163ea2ee6307e593ee5e7aaee771291aec162f Mon Sep 17 00:00:00 2001 From: Ahmed Abushagur Date: Sun, 22 Mar 2026 12:15:09 -0700 Subject: [PATCH] feat(e2e): AI-powered log review catches non-fatal issues (#2875) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(e2e): add AI-powered log review after provisioning Feeds provision stderr/stdout logs to an LLM after each agent deploys. Catches non-fatal issues that binary pass/fail checks miss: silent 404s, failed component installs, connection instability, swallowed warnings. This would have caught the keep-alive 404 and the sprite idle shutdown that the existing E2E tests missed because installSpriteKeepAlive() is non-fatal and the binary checks only verify final state. - Uses gemini-flash-lite-2.0 via OpenRouter (cheap, fast) - Advisory only — never fails the test, reports findings as warnings - Truncates logs to last 200 lines to stay within token limits - Skips gracefully if OPENROUTER_API_KEY is missing or API fails Co-Authored-By: Claude Opus 4.6 (1M context) * feat(e2e): add AI log review and --fast mode testing AI log review: - After each agent provisions, feeds stderr/stdout to gemini-flash-lite to catch non-fatal issues binary checks miss (404s, failed installs, connection drops, swallowed warnings) - Advisory only — never fails the test, surfaces findings as warnings - Would have caught the keep-alive 404 and sprite idle shutdown --fast mode E2E: - Add --fast flag to e2e.sh, passed through to spawn CLI during provision - Update QA e2e-tester protocol to run both normal and --fast passes - --fast enables images + tarballs + parallel boot Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: L <6723574+louisgv@users.noreply.github.com> --- .../setup-agent-team/qa-quality-prompt.md | 5 +- sh/e2e/e2e.sh | 15 ++ sh/e2e/lib/ai-review.sh | 157 ++++++++++++++++++ sh/e2e/lib/provision.sh | 7 +- 4 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 sh/e2e/lib/ai-review.sh diff --git a/.claude/skills/setup-agent-team/qa-quality-prompt.md b/.claude/skills/setup-agent-team/qa-quality-prompt.md index 28bd2a16..fc489e1a 100644 --- a/.claude/skills/setup-agent-team/qa-quality-prompt.md +++ b/.claude/skills/setup-agent-team/qa-quality-prompt.md @@ -129,9 +129,12 @@ cd REPO_ROOT_PLACEHOLDER && git worktree remove WORKTREE_BASE_PLACEHOLDER/TASK_N ```bash cd REPO_ROOT_PLACEHOLDER chmod +x sh/e2e/e2e.sh + # Normal mode — standard provisioning ./sh/e2e/e2e.sh --cloud all --parallel 6 --skip-input-test + # Fast mode — tests --fast flag (images + tarballs + parallel boot) + ./sh/e2e/e2e.sh --cloud sprite --fast --parallel 4 --skip-input-test ``` -2. Capture the full output. Note which clouds ran, which agents passed, which failed, and which clouds were skipped (no credentials). +2. Capture the full output from BOTH runs. Note which clouds ran, which agents passed, which failed, and which clouds were skipped (no credentials). 3. If all configured clouds pass (or only skipped clouds): report results and you're done. No PR needed. 4. If any agent fails on a configured cloud, investigate the root cause. Failure categories: diff --git a/sh/e2e/e2e.sh b/sh/e2e/e2e.sh index a5eb0c8e..8dd76962 100755 --- a/sh/e2e/e2e.sh +++ b/sh/e2e/e2e.sh @@ -32,6 +32,7 @@ source "${SCRIPT_DIR}/lib/verify.sh" source "${SCRIPT_DIR}/lib/teardown.sh" source "${SCRIPT_DIR}/lib/soak.sh" source "${SCRIPT_DIR}/lib/interactive.sh" +source "${SCRIPT_DIR}/lib/ai-review.sh" # --------------------------------------------------------------------------- # All supported clouds (excluding local — no infra to provision) @@ -49,6 +50,7 @@ SKIP_INPUT_TEST="${SKIP_INPUT_TEST:-0}" SEQUENTIAL_MODE=0 SOAK_MODE=0 INTERACTIVE_MODE=0 +FAST_MODE=0 while [ $# -gt 0 ]; do case "$1" in @@ -114,6 +116,10 @@ while [ $# -gt 0 ]; do INTERACTIVE_MODE=1 shift ;; + --fast) + FAST_MODE=1 + shift + ;; --help|-h) printf "Usage: %s --cloud CLOUD [--cloud CLOUD2 ...] [agents...] [options]\n\n" "$0" printf "Clouds: %s\n" "${ALL_CLOUDS}" @@ -125,6 +131,7 @@ while [ $# -gt 0 ]; do printf " --sequential Force sequential agent execution\n" printf " --skip-cleanup Skip stale e2e-* instance cleanup\n" printf " --skip-input-test Skip live input tests\n" + printf " --fast Provision with --fast flag (images + tarballs + parallel)\n" printf " --soak Run Telegram soak test (OpenClaw on Sprite)\n" printf " --interactive AI-driven interactive test (requires ANTHROPIC_API_KEY)\n" printf " --help Show this help\n" @@ -228,6 +235,8 @@ run_single_agent() { else # Standard headless mode if provision_agent "${agent}" "${app_name}" "${LOG_DIR}"; then + # AI review of provision logs — advisory only, runs regardless of verify result + ai_review_logs "${agent}" "${app_name}" "${LOG_DIR}" || true if verify_agent "${agent}" "${app_name}"; then if run_input_test "${agent}" "${app_name}"; then _inner_status="pass" @@ -639,6 +648,12 @@ fi if [ "${SKIP_INPUT_TEST}" -eq 1 ]; then log_info "Input tests: SKIPPED" fi +if [ "${FAST_MODE}" -eq 1 ]; then + log_info "Fast mode: ENABLED (--fast passed to spawn)" +fi + +# Export FAST_MODE so provision.sh can read it +export E2E_FAST_MODE="${FAST_MODE}" # Create temp log directory LOG_DIR=$(mktemp -d "${TMPDIR:-/tmp}/spawn-e2e.XXXXXX") diff --git a/sh/e2e/lib/ai-review.sh b/sh/e2e/lib/ai-review.sh new file mode 100644 index 00000000..da7af138 --- /dev/null +++ b/sh/e2e/lib/ai-review.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# e2e/lib/ai-review.sh — AI-powered log analysis for E2E test output +# +# After provision + verify pass, feeds stderr/stdout logs to an LLM to catch +# non-fatal issues that binary pass/fail checks miss: silent 404s, degraded +# installs, swallowed warnings, connection instability, etc. +# +# Requires: OPENROUTER_API_KEY (reuses the same key used for E2E provisioning) +# Skips gracefully if the key is missing or the API call fails. +set -eo pipefail + +# --------------------------------------------------------------------------- +# ai_review_logs AGENT APP_NAME LOG_DIR +# +# Analyzes provision logs for an agent and reports findings as warnings. +# Returns 0 always (advisory only — never fails the test). +# --------------------------------------------------------------------------- +ai_review_logs() { + local agent="$1" + local app_name="$2" + local log_dir="$3" + + local api_key="${OPENROUTER_API_KEY:-}" + if [ -z "${api_key}" ]; then + return 0 + fi + + local stdout_file="${log_dir}/${app_name}.stdout" + local stderr_file="${log_dir}/${app_name}.stderr" + + # Collect log content (truncate to last 200 lines each to stay within token limits) + local log_content="" + if [ -f "${stderr_file}" ] && [ -s "${stderr_file}" ]; then + log_content="=== STDERR (last 200 lines) === +$(tail -200 "${stderr_file}" 2>/dev/null || true) +" + fi + if [ -f "${stdout_file}" ] && [ -s "${stdout_file}" ]; then + log_content="${log_content}=== STDOUT (last 200 lines) === +$(tail -200 "${stdout_file}" 2>/dev/null || true) +" + fi + + # Skip if no log content + if [ -z "${log_content}" ]; then + return 0 + fi + + log_step "AI reviewing ${agent} logs..." + + # Build the prompt + local system_prompt='You are a QA engineer reviewing deployment logs from an automated E2E test of "spawn" — a tool that provisions cloud VMs and installs AI coding agents. + +Your job: find issues that passed the binary tests but indicate degraded or broken behavior. Focus on: +- HTTP errors (404, 500, timeouts) even if the step was marked non-fatal +- Failed installations of components (keep-alive scripts, browser, plugins) +- Connection drops, retries, or timeouts during provisioning +- Warnings that indicate missing functionality +- Security warnings (exposed credentials, insecure connections) +- Package deprecation warnings that could break future builds + +Do NOT flag: +- Normal npm deprecation warnings for transient dependencies (these are upstream) +- Successful retries (only flag if all retries failed) +- Expected "non-interactive" or "headless" mode messages +- Informational step progress messages + +Output format: If you find issues, output one line per issue: +ISSUE: + +If no issues found, output exactly: NO_ISSUES + +Be concise. Max 5 issues.' + + # Use a temp file for the request body to avoid shell quoting issues + local req_file + req_file=$(mktemp /tmp/e2e-ai-review-XXXXXX.json) + + # Build JSON safely via bun to avoid shell injection + local ts_file + ts_file=$(mktemp /tmp/e2e-ai-build-XXXXXX.ts) + cat > "${ts_file}" << 'TS_EOF' +const system = process.env._AI_SYSTEM ?? ""; +const logs = process.env._AI_LOGS ?? ""; +const agent = process.env._AI_AGENT ?? ""; +const outFile = process.env._AI_OUT ?? ""; + +const body = { + model: "google/gemini-flash-lite-2.0", + max_tokens: 512, + messages: [ + { role: "system", content: system }, + { role: "user", content: `Agent: ${agent}\n\nDeployment logs:\n\n${logs}` }, + ], +}; + +await Bun.write(outFile, JSON.stringify(body)); +TS_EOF + + _AI_SYSTEM="${system_prompt}" \ + _AI_LOGS="${log_content}" \ + _AI_AGENT="${agent}" \ + _AI_OUT="${req_file}" \ + bun run "${ts_file}" 2>/dev/null + + rm -f "${ts_file}" 2>/dev/null || true + + # Call OpenRouter API + local response + response=$(curl -sf --max-time 30 \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${api_key}" \ + -d @"${req_file}" \ + "https://openrouter.ai/api/v1/chat/completions" 2>/dev/null) || { + rm -f "${req_file}" 2>/dev/null || true + log_warn "AI review skipped (API call failed)" + return 0 + } + + rm -f "${req_file}" 2>/dev/null || true + + # Extract the response content + local ai_output + ai_output=$(printf '%s' "${response}" | bun -e " + const data = JSON.parse(await Bun.stdin.text()); + const content = data?.choices?.[0]?.message?.content ?? ''; + process.stdout.write(content); + " 2>/dev/null) || { + log_warn "AI review skipped (failed to parse response)" + return 0 + } + + # Parse and report findings + if printf '%s' "${ai_output}" | grep -q "NO_ISSUES"; then + log_ok "AI review: no issues found" + return 0 + fi + + # Report each issue as a warning + local issue_count=0 + while IFS= read -r line; do + case "${line}" in + ISSUE:*) + issue_count=$((issue_count + 1)) + log_warn "AI review: ${line#ISSUE: }" + ;; + esac + done <<< "${ai_output}" + + if [ "${issue_count}" -eq 0 ]; then + log_ok "AI review: no issues found" + else + log_warn "AI review: ${issue_count} issue(s) found for ${agent}" + fi + + return 0 +} diff --git a/sh/e2e/lib/provision.sh b/sh/e2e/lib/provision.sh index 8519122c..b90580ed 100644 --- a/sh/e2e/lib/provision.sh +++ b/sh/e2e/lib/provision.sh @@ -112,7 +112,12 @@ provision_agent() { $(cloud_headless_env "${app_name}" "${agent}") CLOUD_ENV - bun run "${cli_entry}" "${agent}" "${ACTIVE_CLOUD}" --headless --output json \ + # Build CLI args — add --fast when E2E_FAST_MODE is enabled + _cli_args="${agent} ${ACTIVE_CLOUD} --headless --output json" + if [ "${E2E_FAST_MODE:-0}" = "1" ]; then + _cli_args="${_cli_args} --fast" + fi + bun run "${cli_entry}" ${_cli_args} \ > "${stdout_file}" 2> "${stderr_file}" printf '%s' "$?" > "${exit_file}" ) &