mirror of
https://github.com/OpenRouterTeam/spawn.git
synced 2026-04-28 03:49:31 +00:00
fix(digitalocean): add OAuth recovery in doApi for mid-session 401 errors (#2723)
When a DigitalOcean token expires mid-session (after ensureDoToken succeeds), API calls like ensureSshKey, createServer, listServers, destroyServer would crash with "Fatal: DigitalOcean API error 401" because doApi had no recovery path for 401 responses. Now doApi detects 401, attempts OAuth browser flow recovery via tryDoOAuth(), and retries the request with the new token. A re-entrancy guard prevents infinite loops (doApi → tryDoOAuth → doApi → ...). If OAuth recovery fails, the original 401 error is thrown as before. Co-authored-by: Claude <claude@anthropic.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
00863b0172
commit
1733903a1f
4 changed files with 169 additions and 6 deletions
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@openrouter/spawn",
|
||||
"version": "0.20.10",
|
||||
"version": "0.20.11",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"spawn": "cli.js"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
|
||||
import { _testHelpers } from "../digitalocean/digitalocean";
|
||||
|
||||
const { testDoToken, state } = _testHelpers;
|
||||
const { testDoToken, doApi, state } = _testHelpers;
|
||||
|
||||
describe("testDoToken", () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
|
@ -14,6 +14,7 @@ describe("testDoToken", () => {
|
|||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
state.token = savedToken;
|
||||
_testHelpers.recovering401 = false;
|
||||
});
|
||||
|
||||
it("returns false when token is empty", async () => {
|
||||
|
|
@ -39,6 +40,8 @@ describe("testDoToken", () => {
|
|||
|
||||
it("returns false (not throws) when API returns 401", async () => {
|
||||
state.token = "expired-token";
|
||||
// Set recovering401 to skip OAuth recovery during testDoToken validation
|
||||
_testHelpers.recovering401 = true;
|
||||
globalThis.fetch = mock(() =>
|
||||
Promise.resolve(
|
||||
new Response("Unauthorized", {
|
||||
|
|
@ -46,8 +49,6 @@ describe("testDoToken", () => {
|
|||
}),
|
||||
),
|
||||
);
|
||||
// Before the fix, this would throw: "DigitalOcean API error 401..."
|
||||
// After the fix, asyncTryCatch catches the error and unwrapOr returns false
|
||||
expect(await testDoToken()).toBe(false);
|
||||
});
|
||||
|
||||
|
|
@ -69,3 +70,112 @@ describe("testDoToken", () => {
|
|||
expect(await testDoToken()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("doApi 401 OAuth recovery", () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let savedToken: string;
|
||||
|
||||
beforeEach(() => {
|
||||
savedToken = state.token;
|
||||
_testHelpers.recovering401 = false;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
state.token = savedToken;
|
||||
_testHelpers.recovering401 = false;
|
||||
});
|
||||
|
||||
it("attempts OAuth recovery on 401 before throwing", async () => {
|
||||
state.token = "expired-token";
|
||||
let callCount = 0;
|
||||
globalThis.fetch = mock((url: string | URL | Request) => {
|
||||
callCount++;
|
||||
const urlStr = String(url);
|
||||
// First call: the actual API call returning 401
|
||||
if (callCount === 1) {
|
||||
return Promise.resolve(
|
||||
new Response("Unauthorized", {
|
||||
status: 401,
|
||||
}),
|
||||
);
|
||||
}
|
||||
// Second call: OAuth connectivity check — fail it so tryDoOAuth returns null quickly
|
||||
// (avoids starting a real Bun.serve OAuth server)
|
||||
if (urlStr.includes("cloud.digitalocean.com")) {
|
||||
return Promise.reject(new Error("network unavailable"));
|
||||
}
|
||||
return Promise.resolve(
|
||||
new Response("Unauthorized", {
|
||||
status: 401,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
// OAuth recovery fails (connectivity check fails), so doApi throws the 401
|
||||
await expect(doApi("GET", "/account", undefined, 1)).rejects.toThrow("DigitalOcean API error 401");
|
||||
// Verify recovery was attempted: 1 API call + 1 connectivity check = 2
|
||||
expect(callCount).toBe(2);
|
||||
});
|
||||
|
||||
it("succeeds after OAuth recovery provides a new token", async () => {
|
||||
state.token = "expired-token";
|
||||
let callCount = 0;
|
||||
globalThis.fetch = mock((url: string | URL | Request) => {
|
||||
callCount++;
|
||||
const urlStr = String(url);
|
||||
// First call: the actual API call returning 401
|
||||
if (callCount === 1) {
|
||||
return Promise.resolve(
|
||||
new Response("Unauthorized", {
|
||||
status: 401,
|
||||
}),
|
||||
);
|
||||
}
|
||||
// OAuth connectivity check — fail so tryDoOAuth returns null
|
||||
if (urlStr.includes("cloud.digitalocean.com")) {
|
||||
return Promise.reject(new Error("network unavailable"));
|
||||
}
|
||||
return Promise.resolve(new Response("ok"));
|
||||
});
|
||||
|
||||
// tryDoOAuth returns null, so this should throw
|
||||
await expect(doApi("GET", "/account", undefined, 1)).rejects.toThrow("DigitalOcean API error 401");
|
||||
});
|
||||
|
||||
it("skips OAuth recovery when re-entrancy guard is set", async () => {
|
||||
state.token = "expired-token";
|
||||
_testHelpers.recovering401 = true;
|
||||
let callCount = 0;
|
||||
globalThis.fetch = mock(() => {
|
||||
callCount++;
|
||||
return Promise.resolve(
|
||||
new Response("Unauthorized", {
|
||||
status: 401,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
// Should throw immediately — only 1 fetch (the API call), no OAuth attempt
|
||||
await expect(doApi("GET", "/account", undefined, 1)).rejects.toThrow("DigitalOcean API error 401");
|
||||
expect(callCount).toBe(1);
|
||||
});
|
||||
|
||||
it("resets re-entrancy guard after failed recovery", async () => {
|
||||
state.token = "expired-token";
|
||||
globalThis.fetch = mock((url: string | URL | Request) => {
|
||||
const urlStr = String(url);
|
||||
if (urlStr.includes("cloud.digitalocean.com")) {
|
||||
return Promise.reject(new Error("network error"));
|
||||
}
|
||||
return Promise.resolve(
|
||||
new Response("Unauthorized", {
|
||||
status: 401,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
await expect(doApi("GET", "/account", undefined, 1)).rejects.toThrow("DigitalOcean API error 401");
|
||||
expect(_testHelpers.recovering401).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -5,22 +5,29 @@
|
|||
* invalid IDs, name filtering, and network failures.
|
||||
*/
|
||||
|
||||
import { afterAll, afterEach, describe, expect, it, mock } from "bun:test";
|
||||
import { afterAll, afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
|
||||
|
||||
// ── Import under test ─────────────────────────────────────────────────────
|
||||
// digitalocean.ts only imports a CSS constant from oauth, so no mock needed.
|
||||
|
||||
const { findSpawnSnapshot } = await import("../digitalocean/digitalocean");
|
||||
const { findSpawnSnapshot, _testHelpers } = await import("../digitalocean/digitalocean");
|
||||
|
||||
describe("findSpawnSnapshot", () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
// Prevent doApi from triggering real OAuth recovery on 401 during tests
|
||||
_testHelpers.recovering401 = true;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
_testHelpers.recovering401 = false;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
_testHelpers.recovering401 = false;
|
||||
});
|
||||
|
||||
it("returns the latest snapshot ID sorted by created_at", async () => {
|
||||
|
|
|
|||
|
|
@ -134,6 +134,9 @@ export function getConnectionInfo(): {
|
|||
|
||||
// ─── API Client ──────────────────────────────────────────────────────────────
|
||||
|
||||
/** Guard to prevent re-entrant OAuth recovery (doApi → tryDoOAuth → doApi → …). */
|
||||
let _recovering401 = false;
|
||||
|
||||
async function doApi(method: string, endpoint: string, body?: string, maxRetries = 3): Promise<string> {
|
||||
const url = `${DO_API_BASE}${endpoint}`;
|
||||
|
||||
|
|
@ -157,6 +160,42 @@ async function doApi(method: string, endpoint: string, body?: string, maxRetries
|
|||
});
|
||||
const text = await resp.text();
|
||||
|
||||
// 401: token expired/revoked — try OAuth recovery once before giving up
|
||||
if (resp.status === 401 && !_recovering401) {
|
||||
logWarn("DigitalOcean token expired or revoked, attempting OAuth recovery...");
|
||||
_recovering401 = true;
|
||||
const recoveryResult = await asyncTryCatch(async () => {
|
||||
const newToken = await tryDoOAuth();
|
||||
if (newToken) {
|
||||
_state.token = newToken;
|
||||
await saveTokenToConfig(newToken);
|
||||
logInfo("OAuth recovery succeeded, retrying request...");
|
||||
// Retry the same request with the new token
|
||||
const retryResp = await fetch(url, {
|
||||
...opts,
|
||||
headers: {
|
||||
...headers,
|
||||
Authorization: `Bearer ${newToken}`,
|
||||
},
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
});
|
||||
const retryText = await retryResp.text();
|
||||
if (!retryResp.ok) {
|
||||
throw new Error(
|
||||
`DigitalOcean API error ${retryResp.status} for ${method} ${endpoint}: ${retryText.slice(0, 200)}`,
|
||||
);
|
||||
}
|
||||
return retryText;
|
||||
}
|
||||
return null;
|
||||
});
|
||||
_recovering401 = false;
|
||||
if (recoveryResult.ok && recoveryResult.data !== null) {
|
||||
return recoveryResult.data;
|
||||
}
|
||||
throw new Error(`DigitalOcean API error 401 for ${method} ${endpoint}: ${text.slice(0, 200)}`);
|
||||
}
|
||||
|
||||
if ((resp.status === 429 || resp.status >= 500) && attempt < maxRetries) {
|
||||
logWarn(`API ${resp.status} (attempt ${attempt}/${maxRetries}), retrying in ${interval}s...`);
|
||||
await sleep(interval * 1000);
|
||||
|
|
@ -1516,7 +1555,14 @@ export async function destroyServer(dropletId?: string): Promise<void> {
|
|||
/** @internal Exposed for testing only. */
|
||||
export const _testHelpers = {
|
||||
testDoToken,
|
||||
doApi,
|
||||
get state() {
|
||||
return _state;
|
||||
},
|
||||
get recovering401() {
|
||||
return _recovering401;
|
||||
},
|
||||
set recovering401(v: boolean) {
|
||||
_recovering401 = v;
|
||||
},
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue