mirror of
https://github.com/OpenRouterTeam/spawn.git
synced 2026-05-18 23:51:40 +00:00
feat: add Telegram soak test for OpenClaw (--soak mode) (#2492)
Add a soak test that provisions OpenClaw on Sprite, waits 1 hour for stabilization, injects a Telegram bot token, and runs integration tests against the Telegram Bot API (getMe, sendMessage, getWebhookInfo). - New: sh/e2e/lib/soak.sh — soak test library with all Telegram-specific logic - Modified: sh/e2e/e2e.sh — add --soak flag to arg parser - Modified: qa.sh — add soak run mode (bypasses Claude, runs e2e.sh directly) - Modified: trigger-server.ts — add "soak" to VALID_REASONS - Modified: qa.yml — add soak to workflow_dispatch options Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: A <258483684+la14-1@users.noreply.github.com>
This commit is contained in:
Ahmed Abushagur
GitHub
parent
c0cedc3887
commit
330c10fcd2
5 changed files with 367 additions and 3 deletions
|
|
@ -23,7 +23,12 @@ if [[ -n "${SPAWN_ISSUE}" ]] && [[ ! "${SPAWN_ISSUE}" =~ ^[0-9]+$ ]]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${SPAWN_REASON}" == "e2e" ]]; then
|
||||
if [[ "${SPAWN_REASON}" == "soak" ]]; then
|
||||
RUN_MODE="soak"
|
||||
WORKTREE_BASE="/tmp/spawn-worktrees/qa-soak"
|
||||
TEAM_NAME="spawn-qa-soak"
|
||||
CYCLE_TIMEOUT=5400 # 90 min for soak test (60 min wait + buffer)
|
||||
elif [[ "${SPAWN_REASON}" == "e2e" ]]; then
|
||||
RUN_MODE="e2e"
|
||||
WORKTREE_BASE="/tmp/spawn-worktrees/qa-e2e"
|
||||
TEAM_NAME="spawn-qa-e2e"
|
||||
|
|
@ -184,7 +189,7 @@ done
|
|||
log "Pre-cycle cleanup done."
|
||||
|
||||
# --- Load cloud credentials (quality + fixtures + e2e modes) ---
|
||||
if [[ "${RUN_MODE}" == "fixtures" ]] || [[ "${RUN_MODE}" == "quality" ]] || [[ "${RUN_MODE}" == "e2e" ]]; then
|
||||
if [[ "${RUN_MODE}" == "fixtures" ]] || [[ "${RUN_MODE}" == "quality" ]] || [[ "${RUN_MODE}" == "e2e" ]] || [[ "${RUN_MODE}" == "soak" ]]; then
|
||||
if [[ -f "${REPO_ROOT}/sh/shared/key-request.sh" ]]; then
|
||||
source "${REPO_ROOT}/sh/shared/key-request.sh"
|
||||
load_cloud_keys_from_config
|
||||
|
|
@ -371,8 +376,21 @@ ISSUE_FOOTER
|
|||
rm -f "${issue_body_file}" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# --- Soak mode: run e2e.sh --soak directly (no Claude needed) ---
|
||||
if [[ "${RUN_MODE}" == "soak" ]]; then
|
||||
log "Running soak test directly (no Claude needed)..."
|
||||
cd "${REPO_ROOT}"
|
||||
bash sh/e2e/e2e.sh --soak 2>&1 | tee -a "${LOG_FILE}"
|
||||
CLAUDE_EXIT=$?
|
||||
|
||||
if [[ "${CLAUDE_EXIT}" -eq 0 ]]; then
|
||||
log "Soak test completed successfully"
|
||||
else
|
||||
log "Soak test failed (exit_code=${CLAUDE_EXIT})"
|
||||
fi
|
||||
|
||||
# --- Quality mode: retry up to 3 times, then file issue ---
|
||||
if [[ "${RUN_MODE}" == "quality" ]]; then
|
||||
elif [[ "${RUN_MODE}" == "quality" ]]; then
|
||||
MAX_ATTEMPTS=3
|
||||
ATTEMPT=0
|
||||
CLAUDE_EXIT=1
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ const VALID_REASONS = new Set([
|
|||
"hygiene",
|
||||
"fixtures",
|
||||
"e2e",
|
||||
"soak",
|
||||
]);
|
||||
|
||||
/** Check if a process is still alive via kill(0) */
|
||||
|
|
|
|||
1
.github/workflows/qa.yml
vendored
1
.github/workflows/qa.yml
vendored
|
|
@ -13,6 +13,7 @@ on:
|
|||
- schedule
|
||||
- e2e
|
||||
- fixtures
|
||||
- soak
|
||||
jobs:
|
||||
trigger:
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ source "${SCRIPT_DIR}/lib/common.sh"
|
|||
source "${SCRIPT_DIR}/lib/provision.sh"
|
||||
source "${SCRIPT_DIR}/lib/verify.sh"
|
||||
source "${SCRIPT_DIR}/lib/teardown.sh"
|
||||
source "${SCRIPT_DIR}/lib/soak.sh"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# All supported clouds (excluding local — no infra to provision)
|
||||
|
|
@ -45,6 +46,7 @@ PARALLEL_COUNT=99
|
|||
SKIP_CLEANUP=0
|
||||
SKIP_INPUT_TEST="${SKIP_INPUT_TEST:-0}"
|
||||
SEQUENTIAL_MODE=0
|
||||
SOAK_MODE=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
|
|
@ -98,6 +100,10 @@ while [ $# -gt 0 ]; do
|
|||
SKIP_INPUT_TEST=1
|
||||
shift
|
||||
;;
|
||||
--soak)
|
||||
SOAK_MODE=1
|
||||
shift
|
||||
;;
|
||||
--help|-h)
|
||||
printf "Usage: %s --cloud CLOUD [--cloud CLOUD2 ...] [agents...] [options]\n\n" "$0"
|
||||
printf "Clouds: %s\n" "${ALL_CLOUDS}"
|
||||
|
|
@ -109,6 +115,7 @@ while [ $# -gt 0 ]; do
|
|||
printf " --sequential Force sequential agent execution\n"
|
||||
printf " --skip-cleanup Skip stale e2e-* instance cleanup\n"
|
||||
printf " --skip-input-test Skip live input tests\n"
|
||||
printf " --soak Run Telegram soak test (OpenClaw on Sprite)\n"
|
||||
printf " --help Show this help\n"
|
||||
exit 0
|
||||
;;
|
||||
|
|
@ -139,6 +146,14 @@ while [ $# -gt 0 ]; do
|
|||
esac
|
||||
done
|
||||
|
||||
# Soak mode: run Telegram soak test and exit (no --cloud required)
|
||||
if [ "${SOAK_MODE}" -eq 1 ]; then
|
||||
LOG_DIR=$(mktemp -d "${TMPDIR:-/tmp}/spawn-e2e.XXXXXX")
|
||||
export LOG_DIR
|
||||
run_soak_test "${LOG_DIR}"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# Require at least one cloud
|
||||
if [ -z "${CLOUDS}" ]; then
|
||||
printf "Error: --cloud is required. Use --cloud aws, --cloud all, etc.\n" >&2
|
||||
|
|
|
|||
329
sh/e2e/lib/soak.sh
Normal file
329
sh/e2e/lib/soak.sh
Normal file
|
|
@ -0,0 +1,329 @@
|
|||
#!/bin/bash
|
||||
# e2e/lib/soak.sh — Telegram soak test for OpenClaw
|
||||
#
|
||||
# Provisions OpenClaw on Sprite, waits for stabilization, injects a Telegram
|
||||
# bot token, and runs integration tests against the Telegram Bot API.
|
||||
#
|
||||
# Required env vars:
|
||||
# TELEGRAM_BOT_TOKEN — Bot token from @BotFather
|
||||
# TELEGRAM_TEST_CHAT_ID — Chat ID to send test messages to
|
||||
#
|
||||
# Optional env vars:
|
||||
# SOAK_WAIT_SECONDS — Override the default 1-hour soak wait (default: 3600)
|
||||
set -eo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
SOAK_WAIT_SECONDS="${SOAK_WAIT_SECONDS:-3600}"
|
||||
SOAK_HEARTBEAT_INTERVAL=300 # 5 minutes
|
||||
SOAK_GATEWAY_PORT=18789
|
||||
TELEGRAM_API_BASE="https://api.telegram.org"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# soak_validate_telegram_env
|
||||
#
|
||||
# Checks that TELEGRAM_BOT_TOKEN and TELEGRAM_TEST_CHAT_ID are set.
|
||||
# ---------------------------------------------------------------------------
|
||||
soak_validate_telegram_env() {
|
||||
local missing=0
|
||||
|
||||
if [ -z "${TELEGRAM_BOT_TOKEN:-}" ]; then
|
||||
log_err "TELEGRAM_BOT_TOKEN is not set"
|
||||
missing=1
|
||||
fi
|
||||
|
||||
if [ -z "${TELEGRAM_TEST_CHAT_ID:-}" ]; then
|
||||
log_err "TELEGRAM_TEST_CHAT_ID is not set"
|
||||
missing=1
|
||||
fi
|
||||
|
||||
if [ "${missing}" -eq 1 ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_ok "Telegram env validated (token + chat ID present)"
|
||||
return 0
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# soak_wait APP_NAME
|
||||
#
|
||||
# Sleeps for SOAK_WAIT_SECONDS with a heartbeat every 5 minutes.
|
||||
# Each heartbeat checks gateway port 18789 is still listening.
|
||||
# ---------------------------------------------------------------------------
|
||||
soak_wait() {
|
||||
local app="$1"
|
||||
local elapsed=0
|
||||
local port_check='ss -tln 2>/dev/null | grep -q ":18789 " || (echo >/dev/tcp/127.0.0.1/18789) 2>/dev/null || nc -z 127.0.0.1 18789 2>/dev/null'
|
||||
|
||||
log_header "Soak wait: ${SOAK_WAIT_SECONDS}s (heartbeat every ${SOAK_HEARTBEAT_INTERVAL}s)"
|
||||
|
||||
while [ "${elapsed}" -lt "${SOAK_WAIT_SECONDS}" ]; do
|
||||
local remaining=$((SOAK_WAIT_SECONDS - elapsed))
|
||||
local sleep_time="${SOAK_HEARTBEAT_INTERVAL}"
|
||||
if [ "${remaining}" -lt "${sleep_time}" ]; then
|
||||
sleep_time="${remaining}"
|
||||
fi
|
||||
|
||||
sleep "${sleep_time}"
|
||||
elapsed=$((elapsed + sleep_time))
|
||||
|
||||
# Heartbeat: check gateway is alive
|
||||
if cloud_exec "${app}" "${port_check}" >/dev/null 2>&1; then
|
||||
log_info "Heartbeat ${elapsed}/${SOAK_WAIT_SECONDS}s — gateway alive on :${SOAK_GATEWAY_PORT}"
|
||||
else
|
||||
log_warn "Heartbeat ${elapsed}/${SOAK_WAIT_SECONDS}s — gateway NOT responding on :${SOAK_GATEWAY_PORT}"
|
||||
fi
|
||||
done
|
||||
|
||||
log_ok "Soak wait complete (${SOAK_WAIT_SECONDS}s)"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# soak_inject_telegram_config APP_NAME
|
||||
#
|
||||
# Injects TELEGRAM_BOT_TOKEN into ~/.openclaw/openclaw.json on the remote VM,
|
||||
# then restarts the gateway to pick up the new config.
|
||||
# ---------------------------------------------------------------------------
|
||||
soak_inject_telegram_config() {
|
||||
local app="$1"
|
||||
|
||||
log_header "Injecting Telegram config"
|
||||
|
||||
# Base64-encode the token to avoid shell metacharacter issues
|
||||
local encoded_token
|
||||
encoded_token=$(printf '%s' "${TELEGRAM_BOT_TOKEN}" | base64 -w 0 2>/dev/null || printf '%s' "${TELEGRAM_BOT_TOKEN}" | base64 | tr -d '\n')
|
||||
|
||||
log_step "Patching ~/.openclaw/openclaw.json with Telegram bot token..."
|
||||
|
||||
# Use bun -e on the remote to JSON-patch the config file
|
||||
cloud_exec "${app}" "source ~/.spawnrc 2>/dev/null; \
|
||||
export PATH=\$HOME/.npm-global/bin:\$HOME/.bun/bin:\$HOME/.local/bin:\$PATH; \
|
||||
_TOKEN=\$(printf '%s' '${encoded_token}' | base64 -d); \
|
||||
bun -e ' \
|
||||
const fs = require(\"fs\"); \
|
||||
const configPath = process.env.HOME + \"/.openclaw/openclaw.json\"; \
|
||||
let config = {}; \
|
||||
try { config = JSON.parse(fs.readFileSync(configPath, \"utf-8\")); } catch {} \
|
||||
if (!config.channels) config.channels = {}; \
|
||||
if (!config.channels.telegram) config.channels.telegram = {}; \
|
||||
config.channels.telegram.botToken = process.env._TOKEN; \
|
||||
fs.mkdirSync(require(\"path\").dirname(configPath), { recursive: true }); \
|
||||
fs.writeFileSync(configPath, JSON.stringify(config, null, 2)); \
|
||||
console.log(\"Telegram config injected\"); \
|
||||
'" >/dev/null 2>&1
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
log_err "Failed to inject Telegram config"
|
||||
return 1
|
||||
fi
|
||||
log_ok "Telegram bot token injected into openclaw.json"
|
||||
|
||||
# Restart gateway to pick up new config
|
||||
_openclaw_restart_gateway "${app}"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# soak_test_telegram_getme APP_NAME
|
||||
#
|
||||
# Calls Telegram getMe API from the remote VM to verify the bot token is valid.
|
||||
# ---------------------------------------------------------------------------
|
||||
soak_test_telegram_getme() {
|
||||
local app="$1"
|
||||
|
||||
log_step "Testing Telegram getMe API..."
|
||||
|
||||
local encoded_token
|
||||
encoded_token=$(printf '%s' "${TELEGRAM_BOT_TOKEN}" | base64 -w 0 2>/dev/null || printf '%s' "${TELEGRAM_BOT_TOKEN}" | base64 | tr -d '\n')
|
||||
|
||||
local output
|
||||
output=$(cloud_exec "${app}" "_TOKEN=\$(printf '%s' '${encoded_token}' | base64 -d); \
|
||||
curl -sS \"https://api.telegram.org/bot\${_TOKEN}/getMe\"" 2>&1) || true
|
||||
|
||||
if printf '%s' "${output}" | grep -q '"ok":true'; then
|
||||
log_ok "Telegram getMe — bot token is valid"
|
||||
return 0
|
||||
else
|
||||
log_err "Telegram getMe — unexpected response"
|
||||
log_err "Response: ${output}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# soak_test_telegram_send APP_NAME
|
||||
#
|
||||
# Sends a timestamped test message to TELEGRAM_TEST_CHAT_ID.
|
||||
# ---------------------------------------------------------------------------
|
||||
soak_test_telegram_send() {
|
||||
local app="$1"
|
||||
|
||||
log_step "Testing Telegram sendMessage API..."
|
||||
|
||||
local encoded_token
|
||||
encoded_token=$(printf '%s' "${TELEGRAM_BOT_TOKEN}" | base64 -w 0 2>/dev/null || printf '%s' "${TELEGRAM_BOT_TOKEN}" | base64 | tr -d '\n')
|
||||
|
||||
local marker
|
||||
marker="SPAWN_SOAK_TEST_$(date +%s)"
|
||||
|
||||
local output
|
||||
output=$(cloud_exec "${app}" "_TOKEN=\$(printf '%s' '${encoded_token}' | base64 -d); \
|
||||
curl -sS \"https://api.telegram.org/bot\${_TOKEN}/sendMessage\" \
|
||||
-d chat_id='${TELEGRAM_TEST_CHAT_ID}' \
|
||||
-d text='${marker}'" 2>&1) || true
|
||||
|
||||
if printf '%s' "${output}" | grep -q '"ok":true'; then
|
||||
log_ok "Telegram sendMessage — message sent (marker: ${marker})"
|
||||
return 0
|
||||
else
|
||||
log_err "Telegram sendMessage — failed to send message"
|
||||
log_err "Response: ${output}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# soak_test_telegram_webhook APP_NAME
|
||||
#
|
||||
# Calls getWebhookInfo to verify gateway registered a webhook (or is polling).
|
||||
# ---------------------------------------------------------------------------
|
||||
soak_test_telegram_webhook() {
|
||||
local app="$1"
|
||||
|
||||
log_step "Testing Telegram getWebhookInfo API..."
|
||||
|
||||
local encoded_token
|
||||
encoded_token=$(printf '%s' "${TELEGRAM_BOT_TOKEN}" | base64 -w 0 2>/dev/null || printf '%s' "${TELEGRAM_BOT_TOKEN}" | base64 | tr -d '\n')
|
||||
|
||||
local output
|
||||
output=$(cloud_exec "${app}" "_TOKEN=\$(printf '%s' '${encoded_token}' | base64 -d); \
|
||||
curl -sS \"https://api.telegram.org/bot\${_TOKEN}/getWebhookInfo\"" 2>&1) || true
|
||||
|
||||
if printf '%s' "${output}" | grep -q '"ok":true'; then
|
||||
log_ok "Telegram getWebhookInfo — responded OK"
|
||||
# Log webhook URL if set (informational — polling mode has empty url)
|
||||
local webhook_url
|
||||
webhook_url=$(printf '%s' "${output}" | grep -o '"url":"[^"]*"' | head -1) || true
|
||||
if [ -n "${webhook_url}" ]; then
|
||||
log_info "Webhook info: ${webhook_url}"
|
||||
else
|
||||
log_info "No webhook URL set — bot is likely in polling mode"
|
||||
fi
|
||||
return 0
|
||||
else
|
||||
log_err "Telegram getWebhookInfo — unexpected response"
|
||||
log_err "Response: ${output}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# soak_run_telegram_tests APP_NAME
|
||||
#
|
||||
# Runs all 3 Telegram tests and returns the failure count.
|
||||
# ---------------------------------------------------------------------------
|
||||
soak_run_telegram_tests() {
|
||||
local app="$1"
|
||||
local failures=0
|
||||
|
||||
log_header "Telegram Integration Tests"
|
||||
|
||||
soak_test_telegram_getme "${app}" || failures=$((failures + 1))
|
||||
soak_test_telegram_send "${app}" || failures=$((failures + 1))
|
||||
soak_test_telegram_webhook "${app}" || failures=$((failures + 1))
|
||||
|
||||
if [ "${failures}" -eq 0 ]; then
|
||||
log_ok "All 3 Telegram tests passed"
|
||||
else
|
||||
log_err "${failures}/3 Telegram test(s) failed"
|
||||
fi
|
||||
|
||||
return "${failures}"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# run_soak_test [LOG_DIR]
|
||||
#
|
||||
# Orchestrator: validate env → load sprite driver → provision openclaw →
|
||||
# verify → soak wait → inject telegram config → run tests → teardown.
|
||||
# ---------------------------------------------------------------------------
|
||||
run_soak_test() {
|
||||
local log_dir="${1:-${LOG_DIR:-}}"
|
||||
if [ -z "${log_dir}" ]; then
|
||||
log_dir=$(mktemp -d "${TMPDIR:-/tmp}/spawn-soak.XXXXXX")
|
||||
fi
|
||||
|
||||
log_header "Spawn Soak Test: OpenClaw + Telegram"
|
||||
log_info "Soak wait: ${SOAK_WAIT_SECONDS}s"
|
||||
|
||||
# Validate Telegram secrets
|
||||
if ! soak_validate_telegram_env; then
|
||||
log_err "Soak test aborted — missing Telegram env vars"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Load sprite cloud driver
|
||||
load_cloud_driver "sprite"
|
||||
|
||||
# Validate cloud environment
|
||||
if ! require_env; then
|
||||
log_err "Soak test aborted — cloud env validation failed"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Provision OpenClaw
|
||||
local app_name
|
||||
app_name=$(make_app_name "openclaw")
|
||||
track_app "${app_name}"
|
||||
|
||||
local soak_start
|
||||
soak_start=$(date +%s)
|
||||
|
||||
if ! provision_agent "openclaw" "${app_name}" "${log_dir}"; then
|
||||
log_err "Soak test aborted — provisioning failed"
|
||||
teardown_agent "${app_name}" || log_warn "Teardown failed for ${app_name}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Standard verification
|
||||
if ! verify_agent "openclaw" "${app_name}"; then
|
||||
log_err "Soak test aborted — verification failed"
|
||||
teardown_agent "${app_name}" || log_warn "Teardown failed for ${app_name}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Soak wait
|
||||
soak_wait "${app_name}"
|
||||
|
||||
# Inject Telegram config
|
||||
if ! soak_inject_telegram_config "${app_name}"; then
|
||||
log_err "Soak test aborted — Telegram config injection failed"
|
||||
teardown_agent "${app_name}" || log_warn "Teardown failed for ${app_name}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Run Telegram tests
|
||||
local test_failures=0
|
||||
soak_run_telegram_tests "${app_name}" || test_failures=$?
|
||||
|
||||
# Teardown
|
||||
teardown_agent "${app_name}" || log_warn "Teardown failed for ${app_name}"
|
||||
|
||||
# Summary
|
||||
local soak_end
|
||||
soak_end=$(date +%s)
|
||||
local soak_duration=$((soak_end - soak_start))
|
||||
local duration_str
|
||||
duration_str=$(format_duration "${soak_duration}")
|
||||
|
||||
printf "\n"
|
||||
log_header "Soak Test Summary"
|
||||
if [ "${test_failures}" -eq 0 ]; then
|
||||
log_ok "All Telegram tests passed (${duration_str})"
|
||||
else
|
||||
log_err "${test_failures} Telegram test(s) failed (${duration_str})"
|
||||
fi
|
||||
|
||||
return "${test_failures}"
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue