fix(core): clamp provider output-budget keys to the window in samplingParams

A samplingParams config carrying a provider-specific output-budget key
(max_completion_tokens for GPT-5/o-series, max_new_tokens) but no
max_tokens previously passed the key through verbatim, so its value
escaped the prompt + output <= window clamp — e.g. max_completion_tokens:
200000 on a 200K window with a 150K prompt.

Clamp the key's value in place to the remaining window (min with the
request maxOutputTokens) instead of injecting a separate max_tokens:
sending both keys double-specifies the output budget and o-series
rejects the pair. The value only shrinks when the window is tight; when
there is room it passes through unchanged, matching how max_tokens is
already treated.

packages/core/src/core/openaiContentGenerator/pipeline.test.ts

@@ -2973,10 +2973,11 @@ describe('ContentGenerationPipeline', () => {
       );
     });
 
-    it('should pass arbitrary samplingParams keys through verbatim (e.g. max_completion_tokens for GPT-5)', async () => {
+    it('should pass arbitrary samplingParams keys through verbatim when the window has room (e.g. max_completion_tokens for GPT-5)', async () => {
       // Arrange: user sets a GPT-5 / o-series shape in samplingParams.
       // None of these are typed fields; all must appear on the wire because
-      // samplingParams is the source of truth.
+      // samplingParams is the source of truth. maxOutputTokens (32000) leaves
+      // room above max_completion_tokens (4096), so the value is not clamped.
       mockContentGeneratorConfig.samplingParams = {
         max_completion_tokens: 4096,
         reasoning_effort: 'medium',
@@ -2987,7 +2988,7 @@ describe('ContentGenerationPipeline', () => {
       const request: GenerateContentParameters = {
         model: 'test-model',
         contents: [{ parts: [{ text: 'Hello' }], role: 'user' }],
-        config: { maxOutputTokens: 999 },
+        config: { maxOutputTokens: 32000 },
       };
       (mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue([]);
       (mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
@@ -3001,8 +3002,9 @@ describe('ContentGenerationPipeline', () => {
       // Act
       await pipeline.execute(request, 'prompt-id');
 
-      // Assert: the exact samplingParams keys reach the wire; max_tokens is NOT
-      // synthesized from request.config.maxOutputTokens.
+      // Assert: the exact samplingParams keys reach the wire unchanged; a
+      // separate max_tokens is NOT synthesized (that would double-specify the
+      // budget and o-series rejects the pair).
       const call = (mockClient.chat.completions.create as Mock).mock
         .calls[0][0];
       expect(call).toMatchObject({
@@ -3013,6 +3015,45 @@ describe('ContentGenerationPipeline', () => {
       expect(call).not.toHaveProperty('max_tokens');
     });
 
+    it('should clamp a provider output-budget key to the window without injecting max_tokens', async () => {
+      // Arrange: max_completion_tokens (200000) exceeds the window's remaining
+      // room (maxOutputTokens 50000). The value must be clamped in place so
+      // `prompt + output ≤ window` holds — but NO max_tokens is injected
+      // (o-series rejects both keys together).
+      mockContentGeneratorConfig.samplingParams = {
+        max_completion_tokens: 200000,
+        reasoning_effort: 'high',
+      } as ContentGeneratorConfig['samplingParams'];
+      pipeline = new ContentGenerationPipeline(mockConfig);
+
+      const request: GenerateContentParameters = {
+        model: 'test-model',
+        contents: [{ parts: [{ text: 'Hello' }], role: 'user' }],
+        config: { maxOutputTokens: 50000 },
+      };
+      (mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue([]);
+      (mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
+        new GenerateContentResponse(),
+      );
+      (mockClient.chat.completions.create as Mock).mockResolvedValue({
+        id: 'test',
+        choices: [{ message: { content: 'r' } }],
+      });
+
+      // Act
+      await pipeline.execute(request, 'prompt-id');
+
+      // Assert: provider key clamped to the window; other keys verbatim; no
+      // max_tokens added.
+      const call = (mockClient.chat.completions.create as Mock).mock
+        .calls[0][0];
+      expect(call).toMatchObject({
+        max_completion_tokens: 50000,
+        reasoning_effort: 'high',
+      });
+      expect(call).not.toHaveProperty('max_tokens');
+    });
+
     it('should inject the window-clamped max_tokens when samplingParams omits it and carries no provider output-budget key', async () => {
       // Arrange: samplingParams is set but specifies no output budget (no
       // max_tokens, no provider-specific key). The window clamp

packages/core/src/core/openaiContentGenerator/pipeline.ts

@@ -131,6 +131,30 @@ function hasProviderOutputBudgetKey(samplingParams: {
   );
 }
 
+/**
+ * Clamp any provider-specific output-budget key (e.g. `max_completion_tokens`)
+ * to the window's remaining room, mutating and returning the passed object.
+ * An output budget is subject to `prompt + output ≤ window` regardless of the
+ * key it travels under, so we shrink the key's value to `requestMaxTokens` when
+ * it exceeds it — but we clamp the value in place rather than injecting a
+ * separate `max_tokens`, which would double-specify the budget and be rejected
+ * by endpoints like the o-series. When there is room (or no clamp value is
+ * available), the user's value passes through unchanged.
+ */
+function clampProviderOutputBudgetKeys(
+  samplingParams: { [key: string]: unknown },
+  requestMaxTokens: number | undefined,
+): { [key: string]: unknown } {
+  if (typeof requestMaxTokens !== 'number') return samplingParams;
+  for (const key of PROVIDER_OUTPUT_BUDGET_KEYS) {
+    const value = samplingParams[key];
+    if (typeof value === 'number' && value > requestMaxTokens) {
+      samplingParams[key] = requestMaxTokens;
+    }
+  }
+  return samplingParams;
+}
+
 /**
  * Resolve the effective streaming inactivity timeout (ms). Precedence:
  * explicit `ContentGeneratorConfig.streamIdleTimeoutMs` (programmatic, wins —
@@ -812,15 +836,18 @@ export class ContentGenerationPipeline {
     // When samplingParams is set, its keys pass through to the wire verbatim.
     // This lets users target provider-specific parameter names
     // (e.g. `max_completion_tokens` for GPT-5 / o-series) without a client release.
-    // max_tokens is a ceiling, not an exemption from the window clamp: when both
-    // a config max_tokens and the (clamped) request maxOutputTokens are present
-    // the smaller wins, and when samplingParams omits max_tokens the clamped
-    // request value is injected — so `prompt + max_tokens ≤ window` holds for
-    // samplingParams users too, matching the Anthropic path. The injection is
-    // suppressed when samplingParams already carries a provider-specific
-    // output-budget key (e.g. `max_completion_tokens`): adding `max_tokens`
-    // alongside it would double the budget and some endpoints (o-series) reject
-    // the pair, so those configs stay verbatim.
+    // No output budget escapes the window clamp, whatever key it travels under:
+    //   - max_tokens is a ceiling, not an exemption — when both a config
+    //     max_tokens and the (clamped) request maxOutputTokens are present the
+    //     smaller wins; when samplingParams omits max_tokens the clamped request
+    //     value is injected.
+    //   - A provider-specific output-budget key (max_completion_tokens,
+    //     max_new_tokens) is clamped in place to the window instead — we do NOT
+    //     also inject max_tokens, since sending the pair double-specifies the
+    //     budget and some endpoints (o-series) reject it. Its value only shrinks
+    //     when the window is tight; when there is room it passes through as-is.
+    // So `prompt + max_tokens ≤ window` holds for samplingParams users too,
+    // matching the Anthropic path.
     if (configSamplingParams !== undefined) {
       const requestMaxTokens = request.config?.maxOutputTokens;
       const maxTokens =
@@ -832,7 +859,13 @@ export class ContentGenerationPipeline {
       if (maxTokens !== undefined) {
         return { ...configSamplingParams, max_tokens: maxTokens };
       }
-      return { ...configSamplingParams };
+      // maxTokens is undefined only when a provider-specific output-budget key
+      // is in use (or there is nothing to clamp against). Clamp that key's
+      // value to the window rather than injecting a separate max_tokens.
+      return clampProviderOutputBudgetKeys(
+        { ...configSamplingParams },
+        requestMaxTokens,
+      );
     }
 
     const params: Record<string, unknown> = {