fix(e2e): robust DigitalOcean teardown with retry and deletion confirmation (#2046)

The teardown was doing a single DELETE without --max-time, so connection
timeouts caused HTTP 000 and the droplet was never deleted. When running
6 agents in batches of 3, batch 1's stale droplet caused batch 2 to fail
with "will exceed your droplet limit."

Fix:
- Add --max-time 30 to prevent curl hangs
- Retry DELETE up to 3 times on failure
- Poll the API after DELETE to confirm the droplet is actually gone (up to 60s)
- Remove -f flag from curl so %{http_code} is always captured

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ahmed Abushagur 2026-02-28 19:09:15 -08:00 committed by GitHub
parent d8dbf952c2
commit cd758589c3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -189,6 +189,9 @@ _digitalocean_exec_long() {
# _digitalocean_teardown APP
#
# Deletes the droplet by its ID (read from the .meta file) and untracks it.
# Retries the DELETE up to 3 times on failure, then polls the API to confirm
# the droplet is actually gone (up to 60s). This prevents batch 2 from
# launching while batch 1 droplets still occupy the account's droplet limit.
# ---------------------------------------------------------------------------
_digitalocean_teardown() {
local app="$1"
@ -211,17 +214,55 @@ _digitalocean_teardown() {
return 0
fi
local http_code
http_code=$(curl -sf -o /dev/null -w '%{http_code}' \
-X DELETE \
-H "Authorization: Bearer ${DO_API_TOKEN}" \
-H "Content-Type: application/json" \
"${_DO_API}/droplets/${droplet_id}" 2>/dev/null || true)
# Retry DELETE up to 3 times with --max-time to prevent hangs
local attempt=0
local delete_accepted=0
while [ "${attempt}" -lt 3 ]; do
attempt=$((attempt + 1))
if [ "${http_code}" = "204" ] || [ "${http_code}" = "404" ]; then
log_ok "Droplet ${app} (ID: ${droplet_id}) torn down"
else
log_warn "Droplet deletion returned HTTP ${http_code} for ${app} (ID: ${droplet_id})"
local http_code
http_code=$(curl -s -o /dev/null -w '%{http_code}' \
--max-time 30 \
-X DELETE \
-H "Authorization: Bearer ${DO_API_TOKEN}" \
-H "Content-Type: application/json" \
"${_DO_API}/droplets/${droplet_id}" 2>/dev/null || printf '000')
if [ "${http_code}" = "204" ] || [ "${http_code}" = "404" ]; then
delete_accepted=1
break
fi
if [ "${attempt}" -lt 3 ]; then
log_warn "Droplet DELETE attempt ${attempt}/3 returned HTTP ${http_code} — retrying in 5s..."
sleep 5
else
log_warn "Droplet DELETE failed after 3 attempts (last HTTP ${http_code}) for ${app} (ID: ${droplet_id})"
fi
done
# Poll to confirm the droplet is actually gone (up to 60s).
# The API may accept the DELETE (204) but the droplet lingers briefly.
if [ "${delete_accepted}" -eq 1 ]; then
local poll_waited=0
while [ "${poll_waited}" -lt 60 ]; do
local check_code
check_code=$(curl -s -o /dev/null -w '%{http_code}' \
--max-time 10 \
-H "Authorization: Bearer ${DO_API_TOKEN}" \
"${_DO_API}/droplets/${droplet_id}" 2>/dev/null || printf '000')
if [ "${check_code}" = "404" ]; then
log_ok "Droplet ${app} (ID: ${droplet_id}) confirmed destroyed"
untrack_app "${app}"
return 0
fi
sleep 5
poll_waited=$((poll_waited + 5))
done
log_warn "Droplet ${app} (ID: ${droplet_id}) not yet gone after 60s — may still be deleting"
fi
untrack_app "${app}"