fix(sse): honor Google RPC ErrorInfo quota-reset metadata as an upstream retry hint (#16529)

Google RPC ErrorInfo quota-reset metadata is honored as an upstream retry hint, and compound durations such as 2h7m23s parse as durations.

Boarded on release/v3.8.52 at e01340c854. Focused tests: 317 passed. File-size kept only the inherited executeTargetAttempt.ts overage (1288 > 1284). This pull request does not touch that file.

⚠️ base-red inherited: #15306

changelog.d/fixes/16529-google-rpc-errorinfo-reset-hint.md

@@ -0,0 +1 @@
+- **fix(sse):** honor Google RPC `ErrorInfo` `quotaResetDelay`/`quotaResetTimeStamp` as an upstream retry hint and accept compound durations (`151h58m18s`) in retry-delay parsing ([#16529](https://github.com/diegosouzapw/OmniRoute/pull/16529)) — thanks @fasilu

open-sse/services/retryAfterJson.ts

@@ -47,11 +47,29 @@ function futureResetAtMs(value: unknown, maxMs: number): number | null {
 // adversarial upstream value cannot masquerade as a multi-day reset (#7940).
 export const MAX_SHORT_RETRY_HINT_MS = 24 * 60 * 60 * 1000; // 24h
 
+// Compound Go-style duration ("151h58m18.015s", "2h7m", "45m3s"). Anchored, every
+// quantifier bounded and each group ends in a distinct unit letter, so the pattern
+// cannot backtrack catastrophically on untrusted input (ReDoS rule). At least two units
+// are required — single-unit strings keep their dedicated branches above.
+const COMPOUND_DURATION_RE =
+  /^(?:(\d{1,9})h(\d{1,9})m(?:(\d{1,9}(?:\.\d{1,12})?)s)?|(\d{1,9})h(\d{1,9}(?:\.\d{1,12})?)s|(\d{1,9})m(\d{1,9}(?:\.\d{1,12})?)s)$/i;
+
+function compoundDurationMs(str: string): number | null {
+  const match = COMPOUND_DURATION_RE.exec(str);
+  if (!match) return null;
+  const hours = Number(match[1] ?? match[4] ?? 0);
+  const minutes = Number(match[2] ?? match[6] ?? 0);
+  const seconds = Number.parseFloat(match[3] ?? match[5] ?? match[7] ?? "0");
+  return Math.round(hours * 3600 * 1000 + minutes * 60 * 1000 + seconds * 1000);
+}
+
 /**
- * Parse delay strings like "33s", "26.660853464s", "2m", "1h", "1500ms", or a bare
- * number of seconds. Shared by `parseRetryAfterFromBody` (rateLimitManager wiring)
- * and `parseRetryHintFromJsonBody` (model-lockout wiring) so both honor the same
- * upstream `RetryInfo.retryDelay` grammar (#7940).
+ * Parse delay strings like "33s", "26.660853464s", "2m", "1h", "1500ms", compound
+ * Go-style durations ("151h58m18.015s", "2h7m", "45m3s"), or a bare number of seconds.
+ * Shared by `parseRetryAfterFromBody` (rateLimitManager wiring) and
+ * `parseRetryHintFromJsonBody` (model-lockout wiring) so both honor the same upstream
+ * `RetryInfo.retryDelay` / `ErrorInfo.metadata.quotaResetDelay` grammar (#7940).
+ * `src/shared/utils/classify429.ts::parseDelaySeconds` mirrors this grammar.
  */
 export function parseDelayString(value: unknown): number | null {
   if (!value) return null;
@@ -64,6 +82,8 @@ export function parseDelayString(value: unknown): number | null {
   if (minMatch) return Math.round(Number.parseFloat(minMatch[1]) * 60 * 1000);
   const hrMatch = /^(\d+(?:\.\d+)?)\s*h$/i.exec(str);
   if (hrMatch) return Math.round(Number.parseFloat(hrMatch[1]) * 3600 * 1000);
+  const compoundMs = compoundDurationMs(str);
+  if (compoundMs !== null) return compoundMs;
   // Bare number → seconds
   const num = Number.parseFloat(str);
   return Number.isFinite(num) ? Math.round(num * 1000) : null;
@@ -82,9 +102,33 @@ function retryInfoDetailsMs(details: unknown): number | null {
   return null;
 }
 
+// Google RPC quota-exhaustion 429s (Antigravity / Cloud Code) can carry the long-lived
+// reset only in `error.details[].{"@type": ".../google.rpc.ErrorInfo", "metadata":
+// {"quotaResetTimeStamp": "<ISO>", "quotaResetDelay": "151h58m18.015s"}}`. Unlike
+// RetryInfo this is a quota reset, not a short throttle, so it is capped by the
+// caller's maxMs instead of MAX_SHORT_RETRY_HINT_MS. The absolute timestamp wins over
+// the relative delay when both are present.
+function errorInfoQuotaResetMs(details: unknown, maxMs: number): number | null {
+  for (const detail of Array.isArray(details) ? details : []) {
+    const detailRecord = objectRecord(detail);
+    const type = String(detailRecord["@type"] ?? "");
+    if (!type.includes("ErrorInfo")) continue;
+    const metadata = objectRecord(detailRecord.metadata);
+    const timestampMs = futureTimestampMs(metadata.quotaResetTimeStamp, maxMs);
+    if (timestampMs !== null) return timestampMs;
+    const delayMs = parseDelayString(metadata.quotaResetDelay);
+    if (delayMs !== null && delayMs > 0) return Math.min(delayMs, maxMs);
+  }
+  return null;
+}
+
 /**
  * Parse Retry-After hints from a 429 JSON response body. Providers use both
  * top-level and nested `error` fields for ISO timestamps and millisecond values.
+ *
+ * Precedence: google.rpc.RetryInfo.retryDelay > google.rpc.ErrorInfo quota-reset
+ * metadata > resets_at/resetsAt > retryAfter (ISO) > retry_after_ms/retryAfterMs.
+ * Response headers and prose hints are handled by the callers (headers first, prose last).
  */
 export function parseDetailedRetryHintFromJsonBody(
   body: string,
@@ -101,11 +145,15 @@ export function parseDetailedRetryHintFromJsonBody(
   if (!Object.keys(root).length) return null;
   const errorObj = objectRecord(root.error);
 
-  const retryInfoMs = retryInfoDetailsMs(errorObj.details ?? root.details);
+  const details = errorObj.details ?? root.details;
+  const retryInfoMs = retryInfoDetailsMs(details);
   if (retryInfoMs !== null) {
     return { retryAfterMs: retryInfoMs, provenance: "google_rpc_retry_info" };
   }
 
+  const errorInfoMs = errorInfoQuotaResetMs(details, maxMs);
+  if (errorInfoMs !== null) return { retryAfterMs: errorInfoMs, provenance: "body" };
+
   const resetAtHint = futureResetAtMs(
     errorObj.resets_at ?? root.resets_at ?? errorObj.resetsAt ?? root.resetsAt,
     maxMs

src/shared/utils/classify429.ts

@@ -228,9 +228,14 @@ const TERMINAL_QUOTA_PATTERNS: ReadonlyArray<RegExp> = [
   /insufficient balance/i,
 ];
 
+// Mirror of retryAfterJson's COMPOUND_DURATION_RE ("151h58m18.015s", "2h7m",
+// "45m3s") — anchored and bounded (ReDoS rule); see parseDelaySeconds below.
+const COMPOUND_DURATION_RE =
+  /^(?:(\d{1,9})h(\d{1,9})m(?:(\d{1,9}(?:\.\d{1,12})?)s)?|(\d{1,9})h(\d{1,9}(?:\.\d{1,12})?)s|(\d{1,9})m(\d{1,9}(?:\.\d{1,12})?)s)$/i;
+
 /**
  * Parse an upstream delay string ("38s", "26.66s", "1500ms", "2m", "1h",
- * or a bare number of seconds) into seconds.
+ * compound Go-style "2h7m23s", or a bare number of seconds) into seconds.
  *
  * Deliberately mirrors `parseDelayString` in
  * open-sse/services/retryAfterJson.ts (#7940) rather than importing it:
@@ -249,6 +254,13 @@ function parseDelaySeconds(value: unknown): number | null {
   if (min) return Number.parseFloat(min[1]) * 60;
   const hr = /^(\d+(?:\.\d+)?)\s*h$/i.exec(str);
   if (hr) return Number.parseFloat(hr[1]) * 3600;
+  const compound = COMPOUND_DURATION_RE.exec(str);
+  if (compound) {
+    const hours = Number(compound[1] ?? compound[4] ?? 0);
+    const minutes = Number(compound[2] ?? compound[6] ?? 0);
+    const seconds = Number.parseFloat(compound[3] ?? compound[5] ?? compound[7] ?? "0");
+    return hours * 3600 + minutes * 60 + seconds;
+  }
   const bare = Number.parseFloat(str);
   return Number.isFinite(bare) ? bare : null;
 }

tests/unit/account-fallback-retry-after-json.test.ts

@@ -30,3 +30,107 @@ test("parseRetryFromErrorText ignores past ISO retryAfter values in JSON bodies"
   const pastIso = new Date(Date.now() - 60_000).toISOString();
   assert.equal(parseRetryFromErrorText(JSON.stringify({ error: { retryAfter: pastIso } })), null);
 });
+
+// ── google.rpc.ErrorInfo quota-reset metadata ───────────────────────────────
+// Google RPC 429 bodies (Antigravity / Cloud Code) can carry the long-lived quota reset
+// only in `ErrorInfo.metadata` — `quotaResetDelay` (Go-style duration) and/or
+// `quotaResetTimeStamp` (ISO) — without a RetryInfo detail or a parseable prose hint.
+
+const { parseDelayString, parseDetailedRetryHintFromJsonBody, parseRetryHintFromJsonBody } =
+  await import("../../open-sse/services/retryAfterJson.ts");
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+function errorInfoBody(metadata: Record<string, unknown>, extraDetails: unknown[] = []): string {
+  return JSON.stringify({
+    error: {
+      code: 429,
+      message: "Resource has been exhausted (e.g. check quota).",
+      status: "RESOURCE_EXHAUSTED",
+      details: [
+        ...extraDetails,
+        {
+          "@type": "type.googleapis.com/google.rpc.ErrorInfo",
+          reason: "QUOTA_EXHAUSTED",
+          domain: "cloudcode-pa.googleapis.com",
+          metadata,
+        },
+      ],
+    },
+  });
+}
+
+test("parseDelayString accepts compound Go-style durations", () => {
+  assert.equal(parseDelayString("2h7m23s"), 7_643_000);
+  assert.equal(parseDelayString("151h58m18.015s"), 547_098_015);
+  assert.equal(parseDelayString("2h7m"), 7_620_000);
+  assert.equal(parseDelayString("45m3s"), 2_703_000);
+  assert.equal(parseDelayString("1h0m0.5s"), 3_600_500);
+});
+
+test("parseDelayString keeps the single-unit and bare-number grammar unchanged", () => {
+  assert.equal(parseDelayString("33s"), 33_000);
+  assert.equal(parseDelayString("26.66s"), 26_660);
+  assert.equal(parseDelayString("1500ms"), 1_500);
+  assert.equal(parseDelayString("2m"), 120_000);
+  assert.equal(parseDelayString("1h"), 3_600_000);
+  assert.equal(parseDelayString("547098.015s"), 547_098_015);
+  assert.equal(parseDelayString("12"), 12_000);
+  assert.equal(parseDelayString(""), null);
+  assert.equal(parseDelayString(null), null);
+  assert.equal(parseDelayString("not-a-duration"), null);
+});
+
+test("ErrorInfo.metadata.quotaResetDelay (compound duration) becomes a body retry hint", () => {
+  const body = errorInfoBody({ quotaResetDelay: "151h58m18.015s" });
+  assert.equal(parseRetryHintFromJsonBody(body, 30 * DAY_MS), 547_098_015);
+  assert.deepEqual(parseDetailedRetryHintFromJsonBody(body, 30 * DAY_MS), {
+    retryAfterMs: 547_098_015,
+    provenance: "body",
+  });
+});
+
+test("ErrorInfo.metadata.quotaResetDelay (seconds) is parsed and capped by maxMs", () => {
+  const body = errorInfoBody({ quotaResetDelay: "547098.015s" });
+  assert.equal(parseRetryHintFromJsonBody(body, 30 * DAY_MS), 547_098_015);
+  assert.equal(parseRetryHintFromJsonBody(body, DAY_MS), DAY_MS);
+});
+
+test("ErrorInfo.metadata.quotaResetTimeStamp in the future becomes a body retry hint", () => {
+  const future = new Date(Date.now() + 2 * 60 * 60 * 1000).toISOString();
+  const waitMs = parseRetryHintFromJsonBody(
+    errorInfoBody({ quotaResetTimeStamp: future }),
+    30 * DAY_MS
+  );
+  assert.ok(waitMs !== null, "expected a parsed wait time, got null");
+  assert.ok(Math.abs((waitMs as number) - 7_200_000) <= 2_000, `expected ~7.2e6ms, got ${waitMs}`);
+});
+
+test("ErrorInfo.metadata.quotaResetTimeStamp in the past is ignored", () => {
+  const past = new Date(Date.now() - 60_000).toISOString();
+  assert.equal(
+    parseRetryHintFromJsonBody(errorInfoBody({ quotaResetTimeStamp: past }), DAY_MS),
+    null
+  );
+});
+
+test("ErrorInfo without quota-reset metadata yields no retry hint", () => {
+  assert.equal(
+    parseRetryHintFromJsonBody(errorInfoBody({ service: "aiplatform.googleapis.com" }), DAY_MS),
+    null
+  );
+});
+
+test("RetryInfo still wins over ErrorInfo quota-reset metadata", () => {
+  const body = errorInfoBody({ quotaResetDelay: "151h58m18.015s" }, [
+    { "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay: "26s" },
+  ]);
+  assert.deepEqual(parseDetailedRetryHintFromJsonBody(body, 30 * DAY_MS), {
+    retryAfterMs: 26_000,
+    provenance: "google_rpc_retry_info",
+  });
+});
+
+test("parseRetryFromErrorText honors ErrorInfo quotaResetDelay in a 429 JSON body", () => {
+  assert.equal(parseRetryFromErrorText(errorInfoBody({ quotaResetDelay: "2h7m23s" })), 7_643_000);
+});

tests/unit/classify429.test.ts

@@ -513,3 +513,21 @@ test("classify429: Moonshot engine overloaded stays rate_limit", () => {
     "rate_limit"
   );
 });
+
+test("classify429: a compound Go-style RetryInfo.retryDelay is read as its full duration", () => {
+  // The delay grammar mirrors retryAfterJson.parseDelayString, which accepts compound
+  // durations ("2h7m23s"). Before, "2h7m23s" fell through to parseFloat → 2 seconds.
+  const cases = [
+    ["2h7m23s", "quota_exhausted"],
+    ["45m3s", "rate_limit"],
+  ] as const;
+  for (const [retryDelay, expected] of cases) {
+    const body = {
+      error: {
+        message: "You exceeded your current quota, please check your plan and billing details.",
+        details: [{ "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay }],
+      },
+    };
+    assert.equal(classify429({ status: 429, body }), expected, retryDelay);
+  }
+});