From cd758589c3343f0c100fec590624e515d403ed60 Mon Sep 17 00:00:00 2001
From: Ahmed Abushagur <ahmed@abushagur.com>
Date: Sat, 28 Feb 2026 19:09:15 -0800
Subject: [PATCH] fix(e2e): robust DigitalOcean teardown with retry and
 deletion confirmation (#2046)

The teardown was doing a single DELETE without --max-time, so connection
timeouts caused HTTP 000 and the droplet was never deleted. When running
6 agents in batches of 3, batch 1's stale droplet caused batch 2 to fail
with "will exceed your droplet limit."

Fix:
- Add --max-time 30 to prevent curl hangs
- Retry DELETE up to 3 times on failure
- Poll the API after DELETE to confirm the droplet is actually gone (up to 60s)
- Remove -f flag from curl so %{http_code} is always captured

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
---
 sh/e2e/lib/clouds/digitalocean.sh | 61 ++++++++++++++++++++++++++-----
 1 file changed, 51 insertions(+), 10 deletions(-)

diff --git a/sh/e2e/lib/clouds/digitalocean.sh b/sh/e2e/lib/clouds/digitalocean.sh
index 61a92d8f..38f03d64 100644
--- a/sh/e2e/lib/clouds/digitalocean.sh
+++ b/sh/e2e/lib/clouds/digitalocean.sh
@@ -189,6 +189,9 @@ _digitalocean_exec_long() {
 # _digitalocean_teardown APP
 #
 # Deletes the droplet by its ID (read from the .meta file) and untracks it.
+# Retries the DELETE up to 3 times on failure, then polls the API to confirm
+# the droplet is actually gone (up to 60s). This prevents batch 2 from
+# launching while batch 1 droplets still occupy the account's droplet limit.
 # ---------------------------------------------------------------------------
 _digitalocean_teardown() {
   local app="$1"
@@ -211,17 +214,55 @@ _digitalocean_teardown() {
     return 0
   fi
 
-  local http_code
-  http_code=$(curl -sf -o /dev/null -w '%{http_code}' \
-    -X DELETE \
-    -H "Authorization: Bearer ${DO_API_TOKEN}" \
-    -H "Content-Type: application/json" \
-    "${_DO_API}/droplets/${droplet_id}" 2>/dev/null || true)
+  # Retry DELETE up to 3 times with --max-time to prevent hangs
+  local attempt=0
+  local delete_accepted=0
+  while [ "${attempt}" -lt 3 ]; do
+    attempt=$((attempt + 1))
 
-  if [ "${http_code}" = "204" ] || [ "${http_code}" = "404" ]; then
-    log_ok "Droplet ${app} (ID: ${droplet_id}) torn down"
-  else
-    log_warn "Droplet deletion returned HTTP ${http_code} for ${app} (ID: ${droplet_id})"
+    local http_code
+    http_code=$(curl -s -o /dev/null -w '%{http_code}' \
+      --max-time 30 \
+      -X DELETE \
+      -H "Authorization: Bearer ${DO_API_TOKEN}" \
+      -H "Content-Type: application/json" \
+      "${_DO_API}/droplets/${droplet_id}" 2>/dev/null || printf '000')
+
+    if [ "${http_code}" = "204" ] || [ "${http_code}" = "404" ]; then
+      delete_accepted=1
+      break
+    fi
+
+    if [ "${attempt}" -lt 3 ]; then
+      log_warn "Droplet DELETE attempt ${attempt}/3 returned HTTP ${http_code} — retrying in 5s..."
+      sleep 5
+    else
+      log_warn "Droplet DELETE failed after 3 attempts (last HTTP ${http_code}) for ${app} (ID: ${droplet_id})"
+    fi
+  done
+
+  # Poll to confirm the droplet is actually gone (up to 60s).
+  # The API may accept the DELETE (204) but the droplet lingers briefly.
+  if [ "${delete_accepted}" -eq 1 ]; then
+    local poll_waited=0
+    while [ "${poll_waited}" -lt 60 ]; do
+      local check_code
+      check_code=$(curl -s -o /dev/null -w '%{http_code}' \
+        --max-time 10 \
+        -H "Authorization: Bearer ${DO_API_TOKEN}" \
+        "${_DO_API}/droplets/${droplet_id}" 2>/dev/null || printf '000')
+
+      if [ "${check_code}" = "404" ]; then
+        log_ok "Droplet ${app} (ID: ${droplet_id}) confirmed destroyed"
+        untrack_app "${app}"
+        return 0
+      fi
+
+      sleep 5
+      poll_waited=$((poll_waited + 5))
+    done
+
+    log_warn "Droplet ${app} (ID: ${droplet_id}) not yet gone after 60s — may still be deleting"
   fi
 
   untrack_app "${app}"