test: add mock test coverage for all 15 Fly.io agent scripts (#1390)

Fly.io had zero test coverage — every bug fixed this session (stale tokens, FlyV1 auth, name-taken failures, SSH hangs, PATH issues) went undetected. This adds the full mock test infrastructure: - test/fixtures/fly/ — env vars, API assertions, fixture JSONs for app creation, machine creation, and token validation endpoints - test/mock-curl-script.sh — URL stripping for api.machines.dev, body validation for machine creation, synthetic status responses, app creation POST handler, state tracking - test/mock.sh — mock fly/flyctl CLI binary (ssh console, auth token), URL stripping, required field validation, base64 mock - test/record.sh — Fly.io REST endpoints now recordable, live create+delete cycle, error detection, auth var mapping All 15 agent scripts (aider, claude, openclaw, etc.) are automatically discovered and tested: 75 passed, 0 failed. Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-05-09 19:49:58 +00:00 · 2026-02-17 02:52:26 -08:00 · 2026-02-17 02:52:26 -08:00 · a9d0ee9863
commit a9d0ee9863
parent c3dff4be7b
8 changed files with 173 additions and 6 deletions
--- a/test/fixtures/fly/_api_assertions.sh
+++ b/test/fixtures/fly/_api_assertions.sh
@ -0,0 +1,2 @@
+assert_api_called "POST" "/apps" "creates Fly.io app"
+assert_api_called "POST" "/machines" "creates Fly.io machine"
--- a/test/fixtures/fly/_env.sh
+++ b/test/fixtures/fly/_env.sh
@ -0,0 +1,7 @@
+export FLY_API_TOKEN="test-token-fly"
+export FLY_APP_NAME="test-app"
+export FLY_REGION="iad"
+export FLY_VM_SIZE="shared-cpu-1x"
+export FLY_VM_MEMORY="1024"
+export FLY_ORG="personal"
+export MODEL_ID="openrouter/auto"
--- a/test/fixtures/fly/_metadata.json
+++ b/test/fixtures/fly/_metadata.json
@ -0,0 +1,9 @@
+{
+  "cloud": "fly",
+  "recorded_at": "2026-02-17T00:00:00Z",
+  "fixtures": {
+    "apps": {"endpoint": "/apps?org_slug=personal", "type": "synthetic", "recorded_at": "2026-02-17T00:00:00Z"},
+    "create_app": {"endpoint": "POST /apps", "type": "synthetic", "recorded_at": "2026-02-17T00:00:00Z"},
+    "create_server": {"endpoint": "POST /apps/{name}/machines", "type": "synthetic", "recorded_at": "2026-02-17T00:00:00Z"}
+  }
+}
--- a/test/fixtures/fly/apps.json
+++ b/test/fixtures/fly/apps.json
@ -0,0 +1,13 @@
+{
+  "apps": [
+    {
+      "id": "test-app-id",
+      "name": "test-app",
+      "organization": {
+        "slug": "personal"
+      },
+      "status": "deployed"
+    }
+  ],
+  "total_apps": 1
+}
--- a/test/fixtures/fly/create_server.json
+++ b/test/fixtures/fly/create_server.json
@ -0,0 +1,16 @@
+{
+  "config": {
+    "guest": {
+      "cpu_kind": "shared",
+      "cpus": 1,
+      "memory_mb": 1024
+    },
+    "image": "ubuntu:24.04"
+  },
+  "id": "d890e84b0d3089",
+  "instance_id": "01JTEST",
+  "name": "test-app",
+  "private_ip": "fdaa:0:0:0:a7b:0:0:2",
+  "region": "iad",
+  "state": "created"
+}
--- a/test/mock-curl-script.sh
+++ b/test/mock-curl-script.sh
@ -53,7 +53,7 @@ _maybe_inject_error() {
        create_failure)
            if [ "$METHOD" = "POST" ]; then
                case "$URL" in
-                    *servers*|*droplets*|*instances*)
+                    *servers*|*droplets*|*instances*|*machines*)
                        printf '{"error":"Unprocessable entity"}'
                        if [ "$HAS_WRITE_OUT" = "true" ]; then printf '\n422'; fi
                        exit 1 ;;
@ -87,6 +87,7 @@ _strip_api_base() {
        https://api.hetzner.cloud/v1*)     ENDPOINT="${URL#https://api.hetzner.cloud/v1}" ;;
        https://api.digitalocean.com/v2*)   ENDPOINT="${URL#https://api.digitalocean.com/v2}" ;;
        *eu.api.ovh.com*)                   ENDPOINT=$(echo "$URL" | sed 's|https://eu.api.ovh.com/1.0||') ;;
+        https://api.machines.dev/v1*)       ENDPOINT="${URL#https://api.machines.dev/v1}" ;;
    esac
    EP_CLEAN=$(echo "$ENDPOINT" | sed 's|?.*||')
 }
@ -110,6 +111,7 @@ _validate_body() {
        hetzner)     case "$EP_CLEAN" in /servers)          _check_fields "name server_type image location" ;; esac ;;
        digitalocean) case "$EP_CLEAN" in /droplets)        _check_fields "name region size image" ;; esac ;;
        ovh)         case "$EP_CLEAN" in */create)          _check_fields "name" ;; esac ;;
+        fly)         case "$EP_CLEAN" in */machines)        _check_fields "name region config" ;; esac ;;
    esac
 }

@ -123,6 +125,7 @@ _synthetic_active_response() {
    case "$MOCK_CLOUD" in
        digitalocean) printf '{"droplet":{"id":12345678,"name":"test-srv","status":"active","networks":{"v4":[{"ip_address":"10.0.0.1","type":"public"}]}}}' ;;
        hetzner)      printf '{"server":{"id":99999,"name":"test-srv","status":"running","public_net":{"ipv4":{"ip":"10.0.0.1"}}}}' ;;
+        fly)          printf '{"id":"d890e84b0d3089","name":"test-app","state":"started","region":"iad","private_ip":"fdaa:0:0:0:a7b:0:0:2"}' ;;
        *)            printf '{}' ;;
    esac
 }
@ -155,6 +158,9 @@ _respond_post() {
        /ssh_keys|/ssh-keys|/account/keys|/profile/sshkeys|/sshkeys|*/sshkey)
            printf '{"ssh_key":{"id":99999,"name":"test-key","fingerprint":"af:0d:c5:57:a8:fd:b2:82:5e:d4:c1:65:f0:0c:8a:9d"}}'
            ;;
+        /apps)
+            printf '{"id":"test-app","name":"test-app","status":"deployed","organization":{"slug":"personal"}}'
+            ;;
        *)
            if _try_fixture "create_server"; then
                :
@ -177,7 +183,7 @@ _track_state() {
    case "$METHOD" in
        POST)
            case "$EP_CLEAN" in
-                /servers|/droplets|/instances|/instance-operations/launch)
+                /servers|/droplets|/instances|/instance-operations/launch|*/machines)
                    echo "CREATED:${MOCK_CLOUD}:${TS}" >> "${MOCK_STATE_FILE}" ;;
            esac ;;
        DELETE)
--- a/test/mock.sh
+++ b/test/mock.sh
@ -244,14 +244,39 @@ setup_mock_agents() {
    # Agent binaries
    _create_logging_mock claude aider goose codex interpreter gemini amazonq cline gptme opencode plandex kilocode openclaw nanoclaw q

-    # Tools used during agent install
-    _create_logging_mock pip pip3 npm npx bun node openssl shred cargo go git
+    # Tools used during agent install and file upload
+    _create_logging_mock pip pip3 npm npx bun node openssl shred cargo go git base64

    # Silent mocks (no logging needed)
    _create_silent_mock clear sleep

    # Mock 'ssh-keygen' — returns MD5 fingerprint matching fixture data
    _create_ssh_keygen_mock
+
+    # Mock fly/flyctl CLI — handles ssh console, auth token, version
+    _create_fly_mock
+}
+
+_create_fly_mock() {
+    cat > "${TEST_DIR}/fly" << 'MOCK'
+#!/bin/bash
+echo "fly $*" >> "${MOCK_LOG}"
+case "$1" in
+    auth)
+        case "${2:-}" in
+            token) echo "test-token-fly" ;;
+        esac ;;
+    ssh)
+        # fly ssh console -a APP -C "bash -c CMD" --quiet
+        # Succeed silently — the command "ran"
+        ;;
+    version)
+        echo "fly v0.3.50" ;;
+esac
+exit 0
+MOCK
+    chmod +x "${TEST_DIR}/fly"
+    cp "${TEST_DIR}/fly" "${TEST_DIR}/flyctl"
 }

 setup_fake_home() {
@ -295,6 +320,8 @@ _strip_api_base() {
            endpoint="${url#https://api.digitalocean.com/v2}" ;;
        *eu.api.ovh.com*)
            endpoint=$(echo "$url" | sed 's|https://eu.api.ovh.com/1.0||') ;;
+        https://api.machines.dev/v1*)
+            endpoint="${url#https://api.machines.dev/v1}" ;;
    esac

    echo "$endpoint" | sed 's|?.*||'
@ -309,6 +336,7 @@ _get_required_fields() {
        hetzner:/servers) echo "name server_type image location" ;;
        digitalocean:/droplets) echo "name region size image" ;;
        ovh:*/create) echo "name" ;;
+        fly:*/machines) echo "name region config" ;;
    esac
 }

--- a/test/record.sh
+++ b/test/record.sh
@ -31,7 +31,7 @@ ERRORS=0
 PROMPT_FOR_CREDS=true

 # All clouds with REST APIs that we can record from
-ALL_RECORDABLE_CLOUDS="hetzner digitalocean ovh"
+ALL_RECORDABLE_CLOUDS="hetzner digitalocean ovh fly"

 # --- Endpoint registry ---
 # Declare endpoints as string literal for each cloud
@ -58,6 +58,9 @@ images:/cloud/project/\${OVH_PROJECT_ID:-MISSING}/image
 ssh_keys:/cloud/project/\${OVH_PROJECT_ID:-MISSING}/sshkey
 "

+_ENDPOINTS_fly="
+apps:/apps?org_slug=personal
+"

 get_endpoints() {
    local cloud="$1"
@ -165,6 +168,7 @@ get_auth_env_var() {
        hetzner)       printf "HCLOUD_TOKEN" ;;
        digitalocean)  printf "DO_API_TOKEN" ;;
        ovh)           printf "OVH_APPLICATION_KEY" ;;
+        fly)           printf "FLY_API_TOKEN" ;;
    esac
 }

@ -319,6 +323,7 @@ call_api() {
        hetzner)       hetzner_api GET "$endpoint" ;;
        digitalocean)  do_api GET "$endpoint" ;;
        ovh)           ovh_api_call GET "$endpoint" ;;
+        fly)           curl -fsSL -H "Authorization: ${FLY_API_TOKEN}" "https://api.machines.dev/v1${endpoint}" ;;
    esac
 }

@ -343,6 +348,7 @@ error_checks = {
    'hetzner': lambda d: d.get('error') and isinstance(d.get('error'), dict),
    'digitalocean': lambda d: 'id' in d and isinstance(d.get('id'), str) and 'message' in d,
    'ovh': lambda d: 'message' in d and len(d) <= 3 and not any(k in d for k in success_keys),
+    'fly': lambda d: 'error' in d and isinstance(d.get('error'), str),
 }

 if cloud in error_checks:
@ -372,6 +378,7 @@ _record_live_cycle() {
    case "$cloud" in
        hetzner)       _live_hetzner "$fixture_dir" ;;
        digitalocean)  _live_digitalocean "$fixture_dir" ;;
+        fly)           _live_fly "$fixture_dir" ;;
        *)  return 0 ;;  # No live cycle for this cloud yet
    esac
 }
@ -583,6 +590,85 @@ _live_digitalocean() {
        '{"status":"deleted","http_code":204}'
 }

+_live_fly_body() {
+    local fixture_dir="$1"
+    local name="spawn-record-$(date +%s)"
+    printf '%b\n' "  ${CYAN}live${NC} Creating test app+machine '${name}' (shared-cpu-1x, iad)..." >&2
+
+    python3 -c "
+import json, sys
+print(json.dumps({
+    'name': sys.argv[1], 'region': 'iad',
+    'config': {
+        'image': 'ubuntu:24.04', 'auto_destroy': True,
+        'guest': {'cpu_kind': 'shared', 'cpus': 1, 'memory_mb': 256}
+    }
+}))
+" "$name"
+}
+
+_live_fly() {
+    local fixture_dir="$1"
+    local name="spawn-record-$(date +%s)"
+    local fly_api_base="https://api.machines.dev/v1"
+    local auth_header="Authorization: ${FLY_API_TOKEN}"
+
+    # Detect FlyV1 tokens (dashboard/deploy tokens use FlyV1 scheme, not Bearer)
+    if [[ "$FLY_API_TOKEN" == FlyV1\ * ]]; then
+        auth_header="Authorization: ${FLY_API_TOKEN}"
+    else
+        auth_header="Authorization: Bearer ${FLY_API_TOKEN}"
+    fi
+
+    # Create app
+    printf '%b\n' "  ${CYAN}live${NC} Creating Fly.io app '${name}'..."
+    local app_resp
+    app_resp=$(curl -fsSL -X POST "${fly_api_base}/apps" \
+        -H "${auth_header}" \
+        -H "Content-Type: application/json" \
+        -d "{\"app_name\":\"${name}\",\"org_slug\":\"personal\"}") || true
+
+    if [[ -n "$app_resp" ]]; then
+        _save_live_fixture "$fixture_dir" "create_app" "POST /apps" "$app_resp" || {
+            printf '%b\n' "  ${RED}fail${NC} App creation failed — skipping machine"
+            return 0
+        }
+    fi
+
+    # Create machine
+    local body
+    body=$(_live_fly_body "$fixture_dir")
+    local machine_resp
+    machine_resp=$(curl -fsSL -X POST "${fly_api_base}/apps/${name}/machines" \
+        -H "${auth_header}" \
+        -H "Content-Type: application/json" \
+        -d "$body") || true
+
+    _save_live_fixture "$fixture_dir" "create_server" "POST /apps/{name}/machines" "$machine_resp" || {
+        # Cleanup app even if machine failed
+        curl -fsSL -X DELETE "${fly_api_base}/apps/${name}" -H "${auth_header}" >/dev/null 2>&1 || true
+        return 0
+    }
+
+    local machine_id
+    machine_id=$(echo "$machine_resp" | python3 -c "import json,sys; print(json.loads(sys.stdin.read())['id'])" 2>/dev/null) || true
+
+    # Cleanup: stop + delete machine, delete app
+    printf '%b\n' "  ${CYAN}live${NC} Cleaning up..."
+    if [[ -n "$machine_id" ]]; then
+        curl -fsSL -X POST "${fly_api_base}/apps/${name}/machines/${machine_id}/stop" \
+            -H "${auth_header}" >/dev/null 2>&1 || true
+        sleep 3
+        local del_resp
+        del_resp=$(curl -fsSL -X DELETE "${fly_api_base}/apps/${name}/machines/${machine_id}?force=true" \
+            -H "${auth_header}" 2>/dev/null) || true
+        if [[ -n "$del_resp" ]]; then
+            _save_live_fixture "$fixture_dir" "delete_server" "DELETE /apps/{name}/machines/{id}" "$del_resp" || true
+        fi
+    fi
+    curl -fsSL -X DELETE "${fly_api_base}/apps/${name}" -H "${auth_header}" >/dev/null 2>&1 || true
+    printf '%b\n' "  ${CYAN}live${NC} Cleanup complete"
+}

 # --- Record one cloud ---
 # Check credentials and prompt if needed; returns 1 to skip this cloud
@ -781,7 +867,7 @@ list_clouds() {
    total_count=$(echo "$ALL_RECORDABLE_CLOUDS" | wc -w | tr -d ' ')
    printf '%b\n' "  ${ready_count}/${total_count} clouds have credentials set"
    printf '\n'
-    printf "  CLI-based clouds (not recordable): sprite, gcp, fly, daytona, aws, oracle, local\n"
+    printf "  CLI-based clouds (not recordable): sprite, gcp, daytona, aws, oracle, local\n"
 }

 # --- Main ---