From e5259fa8bb4cf9990c19437c576ce84a9fc60d00 Mon Sep 17 00:00:00 2001 From: Alix-007 Date: Thu, 9 Jul 2026 20:46:33 +0800 Subject: [PATCH] fix(google): add timeout to Vertex ADC token refresh (#102050) * fix(google): add timeout to Vertex ADC token refresh * test(google): consolidate ADC timeout proof * fix(google): bound all ADC token refreshes * test(google): observe ADC timeout before advancing time * fix(google): keep dependency timeout claims exact * fix(google): bound library-managed ADC refreshes * fix(google): recover after ADC refresh timeout --------- Co-authored-by: Peter Steinberger --- extensions/google/transport-stream.test.ts | 79 ++++++++++++++++++++++ extensions/google/vertex-adc.ts | 55 +++++++++++++-- 2 files changed, 127 insertions(+), 7 deletions(-) diff --git a/extensions/google/transport-stream.test.ts b/extensions/google/transport-stream.test.ts index fd9de5a0e73..67e04b1ef8b 100644 --- a/extensions/google/transport-stream.test.ts +++ b/extensions/google/transport-stream.test.ts @@ -1133,11 +1133,40 @@ describe("google transport stream", () => { expect(googleAuthMock).toHaveBeenCalledWith({ scopes: ["https://www.googleapis.com/auth/cloud-platform"], + clientOptions: { transporterOptions: { timeout: 30_000 } }, }); expect(googleAuthGetAccessTokenMock).toHaveBeenCalledTimes(1); expect(tokenFetchMock).not.toHaveBeenCalled(); }); + it("bounds google-auth-library ADC token resolution at the Vertex owner", async () => { + const tempDir = await mkdtemp( + path.join(os.tmpdir(), "openclaw-google-vertex-authlib-timeout-"), + ); + vi.stubEnv("GOOGLE_APPLICATION_CREDENTIALS", ""); + vi.stubEnv("HOME", path.join(tempDir, "home")); + vi.stubEnv("APPDATA", ""); + vi.useFakeTimers(); + googleAuthGetAccessTokenMock + .mockReturnValueOnce(new Promise(() => {})) + .mockResolvedValueOnce("ya29.recovered-token"); + + const pendingRefresh = resolveGoogleVertexAuthorizedUserHeaders(vi.fn()); + const refreshError = pendingRefresh.catch((error: unknown) => error); + await vi.waitFor(() => expect(googleAuthGetAccessTokenMock).toHaveBeenCalledOnce()); + await vi.advanceTimersByTimeAsync(30_000); + + await expect(refreshError).resolves.toMatchObject({ + name: "TimeoutError", + message: "request timed out", + }); + await expect(resolveGoogleVertexAuthorizedUserHeaders(vi.fn())).resolves.toEqual({ + Authorization: "Bearer ya29.recovered-token", + }); + expect(googleAuthMock).toHaveBeenCalledTimes(2); + expect(googleAuthGetAccessTokenMock).toHaveBeenCalledTimes(2); + }); + it("does not cache google-auth ADC tokens when fallback expiry would exceed Date range", async () => { const tempDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-google-vertex-authlib-expiry-")); vi.useFakeTimers(); @@ -1314,6 +1343,56 @@ describe("google transport stream", () => { expect(result.content).toEqual([{ type: "text", text: "ok" }]); }); + it("times out an authorized_user ADC token refresh", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-google-vertex-adc-timeout-")); + const credentialsPath = path.join(tempDir, "application_default_credentials.json"); + await writeFile( + credentialsPath, + JSON.stringify({ + type: "authorized_user", + client_id: "client-id", + client_secret: "client-secret", + refresh_token: "timeout-refresh-token", + }), + "utf8", + ); + vi.stubEnv("GOOGLE_APPLICATION_CREDENTIALS", credentialsPath); + vi.useFakeTimers(); + + let observedSignal: AbortSignal | undefined; + const tokenFetchMock = vi.fn((_input: string | URL | Request, init?: RequestInit) => { + const signal = init?.signal; + if (!signal) { + throw new Error("expected token refresh deadline signal"); + } + observedSignal = signal; + const body = new ReadableStream({ + start(controller) { + signal.addEventListener("abort", () => controller.error(signal.reason), { once: true }); + }, + }); + return Promise.resolve(new Response(body, { status: 200 })); + }); + + const pendingRefresh = resolveGoogleVertexAuthorizedUserHeaders(tokenFetchMock); + // Attach the rejection handler before advancing fake time so the expected + // timeout cannot surface as an unhandled rejection between timer ticks. + const refreshError = pendingRefresh.catch((error: unknown) => error); + await vi.waitFor(() => expect(tokenFetchMock).toHaveBeenCalledOnce()); + const signal = observedSignal; + if (!signal) { + throw new Error("expected token refresh deadline signal"); + } + expect(signal.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(30_000); + + expect(signal.aborted).toBe(true); + await expect(refreshError).resolves.toMatchObject({ + name: "TimeoutError", + message: "request timed out", + }); + }); + it("refreshes authorized_user ADC from a compressed token response", async () => { const tempDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-google-vertex-adc-gzip-")); const credentialsPath = path.join(tempDir, "application_default_credentials.json"); diff --git a/extensions/google/vertex-adc.ts b/extensions/google/vertex-adc.ts index 2300fa5de53..a2e4c03d9da 100644 --- a/extensions/google/vertex-adc.ts +++ b/extensions/google/vertex-adc.ts @@ -4,6 +4,7 @@ import { readFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { gunzipSync } from "node:zlib"; +import { buildTimeoutAbortSignal } from "openclaw/plugin-sdk/extension-shared"; import { asDateTimestampMs, resolveExpiresAtMsFromDurationMs, @@ -11,6 +12,7 @@ import { } from "openclaw/plugin-sdk/number-runtime"; import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime"; import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { withTimeout } from "openclaw/plugin-sdk/text-utility-runtime"; type GoogleAuthorizedUserCredentials = { type: "authorized_user"; @@ -41,6 +43,7 @@ type GoogleOauthTokenResponsePayload = { const GCP_VERTEX_CREDENTIALS_MARKER = "gcp-vertex-credentials"; const GOOGLE_OAUTH_TOKEN_URL = "https://oauth2.googleapis.com/token"; const GOOGLE_VERTEX_OAUTH_SCOPE = "https://www.googleapis.com/auth/cloud-platform"; +const GOOGLE_VERTEX_ADC_TOKEN_REFRESH_TIMEOUT_MS = 30_000; // Hold tokens slightly less long than reported expiry (Google's recommendation // is a 60s buffer) so we don't ship a request that's already revoked when it // leaves the gateway. @@ -243,12 +246,26 @@ async function refreshGoogleVertexAuthorizedUserAccessToken(params: { refresh_token: refreshToken, grant_type: "refresh_token", }); - const response = await (params.fetchImpl ?? fetch)(GOOGLE_OAUTH_TOKEN_URL, { - method: "POST", - headers: { "Content-Type": "application/x-www-form-urlencoded" }, - body, + const { signal, cleanup } = buildTimeoutAbortSignal({ + timeoutMs: GOOGLE_VERTEX_ADC_TOKEN_REFRESH_TIMEOUT_MS, + operation: "google-vertex-adc-token-refresh", + url: GOOGLE_OAUTH_TOKEN_URL, }); - const payload = await readGoogleOauthTokenResponsePayload(response); + let response: Response; + let payload: GoogleOauthTokenResponsePayload | undefined; + try { + response = await (params.fetchImpl ?? fetch)(GOOGLE_OAUTH_TOKEN_URL, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body, + signal, + }); + // Keep the request deadline active through body consumption. Fetch resolves + // at headers, so cleanup here would leave a stalled token body unbounded. + payload = await readGoogleOauthTokenResponsePayload(response); + } finally { + cleanup(); + } if (!response.ok) { const description = normalizeOptionalString(payload?.error_description); const code = normalizeOptionalString(payload?.error); @@ -345,18 +362,42 @@ async function resolveGoogleVertexAccessTokenViaGoogleAuth(): Promise { // It also caches tokens internally and refreshes before expiry. return new GoogleAuth({ scopes: [GOOGLE_VERTEX_OAUTH_SCOPE], + // Best-effort cancellation for clients that use the shared transporter. + // WIF STS and GCE metadata need the owner-level deadline below. + clientOptions: { + transporterOptions: { timeout: GOOGLE_VERTEX_ADC_TOKEN_REFRESH_TIMEOUT_MS }, + }, }); }), }; } - const auth = await cachedGoogleAuthClient.promise; + const authClient = cachedGoogleAuthClient; + const auth = await authClient.promise; const cached = cachedGoogleVertexAdcToken; if (cached && isGoogleVertexTokenFresh(cached.expiresAtMs)) { return cached.token; } - const token = await auth.getAccessToken(); + // Some google-auth-library ADC implementations bypass the configured Gaxios + // transporter, so this owner-level deadline also bounds STS and metadata paths. + let token: string | null | undefined; + try { + token = await withTimeout(auth.getAccessToken(), GOOGLE_VERTEX_ADC_TOKEN_REFRESH_TIMEOUT_MS, { + createError: () => new DOMException("request timed out", "TimeoutError"), + }); + } catch (error) { + // The dependency coalesces in-flight refreshes. Drop only this timed-out + // client so a recovered identity endpoint gets a fresh attempt next time. + if ( + error instanceof DOMException && + error.name === "TimeoutError" && + cachedGoogleAuthClient === authClient + ) { + cachedGoogleAuthClient = undefined; + } + throw error; + } const normalized = normalizeOptionalString(token); if (!normalized) { throw new Error(