From 47ddedcada2dc13a4c9c202dfaa27e3f0746ae64 Mon Sep 17 00:00:00 2001 From: Ali Khokhar Date: Sun, 5 Jul 2026 22:56:45 -0700 Subject: [PATCH] Fix NIM chat template downgrade (#997) ## Problem NVIDIA NIM Mistral-tokenizer models can reject chat-template controls with HTTP 400. FCC only removed `chat_template`, so requests that only had `chat_template_kwargs` still failed. ## Changes | Before | After | | --- | --- | | NIM retried chat-template errors by stripping only `extra_body.chat_template`. | NIM retries chat-template errors by stripping `extra_body.chat_template` and `extra_body.chat_template_kwargs`. | | Mistral-tokenizer models could fail after rejecting thinking chat-template kwargs. | Mistral-tokenizer models retry once without NIM chat-template controls. | | Reasoning-budget downgrades preserved thinking flags independently. | Reasoning-budget downgrades still preserve thinking flags independently. | | Package metadata stayed at `3.4.2`. | Package metadata is bumped to `3.4.3`. |

Greptile Summary

 This PR fixes the NVIDIA NIM chat-template retry downgrade. The main changes are: - Removes both `extra_body.chat_template` and `extra_body.chat_template_kwargs` before retrying chat-template 400 errors. - Adds tests for the full chat-template path and the kwargs-only regression case. - Adds helper coverage for unchanged request bodies. - Bumps package metadata from `3.4.2` to `3.4.3` and keeps `uv.lock` aligned.

Confidence Score: 5/5

Safe to merge with minimal risk. The production change is narrow and covered by targeted tests for both chat-template stripping paths. Version metadata and lockfile updates follow the repository guidance. No files require special attention.

T-Rex T-Rex Logs

 **What T-Rex did** - The initial focused chat-template test run was executed and showed 6 tests passed, while the outer shell wrapper returned nonzero due to a PIPESTATUS\[0\] substitution issue under /bin/sh after pytest completed. - A corrected wrapper rerun of the same focused chat-template tests was performed, and 6 tests passed with exit code 0. - Focused provider retry tests for chat\_template or reasoning\_budget were executed, and 4 tests passed with exit code 0. View all artifacts T-Rex Ran code and verified through T-Rex

Important Files Changed

 | Filename | Overview | |----------|----------| | providers/nvidia_nim/retry.py | Extends NIM chat-template retry downgrades to remove both `chat_template` and `chat_template_kwargs` while preserving unchanged-body behavior. | | tests/providers/test_nvidia_nim.py | Updates streaming retry coverage to assert the second request strips all chat-template controls, including the kwargs-only regression case. | | tests/providers/test_nvidia_nim_request.py | Adds request-body helper coverage for stripping `chat_template_kwargs` and returning `None` when no chat-template controls are present. | | pyproject.toml | Bumps package version from `3.4.2` to `3.4.3` for the production bug fix. | | uv.lock | Keeps the editable package version in the lockfile aligned with `pyproject.toml`. |

Sequence Diagram

 ```mermaid %%{init: {'theme': 'neutral'}}%% sequenceDiagram participant Client as Claude request participant Provider as NvidiaNimProvider participant NIM as NVIDIA NIM API Client->>Provider: stream_response(request) Provider->>NIM: create(extra_body with chat_template controls) NIM-->>Provider: HTTP 400 chat_template rejection Provider->>Provider: clone_body_without_chat_template() Provider->>Provider: remove chat_template and chat_template_kwargs Provider->>NIM: retry create(extra_body without chat-template controls) NIM-->>Provider: stream chunks Provider-->>Client: SSE response events ``` ```mermaid %%{init: {'theme': 'base', 'themeVariables': {"darkMode": true, "background": "#0d1117", "primaryColor": "#21262d", "primaryTextColor": "#e6edf3", "primaryBorderColor": "#8b949e", "lineColor": "#8b949e", "textColor": "#e6edf3", "edgeLabelBackground": "#161b22", "actorBkg": "#21262d", "actorBorder": "#8b949e", "actorTextColor": "#e6edf3", "actorLineColor": "#8b949e", "signalColor": "#8b949e", "signalTextColor": "#e6edf3", "noteBkgColor": "#373320", "noteBorderColor": "#d4a72c", "noteTextColor": "#f0e6c0", "labelBoxBkgColor": "#21262d", "labelBoxBorderColor": "#8b949e", "labelTextColor": "#e6edf3", "loopTextColor": "#e6edf3", "activationBkgColor": "#30363d", "activationBorderColor": "#8b949e"}}}%% sequenceDiagram participant Client as Claude request participant Provider as NvidiaNimProvider participant NIM as NVIDIA NIM API Client->>Provider: stream_response(request) Provider->>NIM: create(extra_body with chat_template controls) NIM-->>Provider: HTTP 400 chat_template rejection Provider->>Provider: clone_body_without_chat_template() Provider->>Provider: remove chat_template and chat_template_kwargs Provider->>NIM: retry create(extra_body without chat-template controls) NIM-->>Provider: stream chunks Provider-->>Client: SSE response events ```
Reviews (1): Last reviewed commit: ["Fix NIM chat template downgrade"](https://github.com/alishahryar1/free-claude-code/commit/fb2d152372356e22441bd52aca99b1dca4792b61) | [Re-trigger Greptile](https://app.greptile.com/api/retrigger?id=42001002) --- providers/nvidia_nim/retry.py | 11 +++-- pyproject.toml | 2 +- tests/providers/test_nvidia_nim.py | 50 +++++++++++++++++++++- tests/providers/test_nvidia_nim_request.py | 30 ++++++++++++- uv.lock | 2 +- 5 files changed, 85 insertions(+), 10 deletions(-) diff --git a/providers/nvidia_nim/retry.py b/providers/nvidia_nim/retry.py index 3b105fd0..335a0b6d 100644 --- a/providers/nvidia_nim/retry.py +++ b/providers/nvidia_nim/retry.py @@ -11,8 +11,8 @@ def clone_body_without_reasoning_budget(body: dict[str, Any]) -> dict[str, Any] def clone_body_without_chat_template(body: dict[str, Any]) -> dict[str, Any] | None: - """Clone a request body and strip only chat_template.""" - return _clone_strip_extra_body(body, _strip_chat_template_field) + """Clone a request body and strip NIM chat-template control fields.""" + return _clone_strip_extra_body(body, _strip_chat_template_fields) def clone_body_without_reasoning_content( @@ -51,8 +51,11 @@ def _strip_reasoning_budget_fields(extra_body: dict[str, Any]) -> bool: return removed -def _strip_chat_template_field(extra_body: dict[str, Any]) -> bool: - return extra_body.pop("chat_template", None) is not None +def _strip_chat_template_fields(extra_body: dict[str, Any]) -> bool: + removed = extra_body.pop("chat_template", None) is not None + if extra_body.pop("chat_template_kwargs", None) is not None: + removed = True + return removed def _strip_message_reasoning_content(body: dict[str, Any]) -> bool: diff --git a/pyproject.toml b/pyproject.toml index f294f105..db759cdd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "free-claude-code" -version = "3.4.2" +version = "3.4.3" description = "Middleware between Claude Code CLI (Anthropic API) and NVIDIA NIM" readme = "README.md" requires-python = ">=3.14.0" diff --git a/tests/providers/test_nvidia_nim.py b/tests/providers/test_nvidia_nim.py index 83467f8d..82a19a74 100644 --- a/tests/providers/test_nvidia_nim.py +++ b/tests/providers/test_nvidia_nim.py @@ -428,12 +428,58 @@ async def test_stream_response_retries_without_chat_template(provider_config): assert "reasoning_budget" not in first_extra assert "chat_template" not in second_extra - assert second_extra["chat_template_kwargs"] == { + assert "chat_template_kwargs" not in second_extra + assert "reasoning_budget" not in second_extra + + event_text = "".join(events) + assert "event: error" not in event_text + assert "OK" in event_text + + +@pytest.mark.asyncio +async def test_stream_response_retries_without_chat_template_kwargs_issue_993( + provider_config, +): + provider = NvidiaNimProvider(provider_config, nim_settings=NimSettings()) + req = MockRequest(model="mistralai/mistral-small-4-119b-2603") + + mock_chunk = MagicMock() + mock_chunk.choices = [ + MagicMock( + delta=MagicMock(content="OK", reasoning_content=""), + finish_reason="stop", + ) + ] + mock_chunk.usage = MagicMock(completion_tokens=2) + + async def mock_stream(): + yield mock_chunk + + first_error = _make_bad_request_error( + "chat_template is not supported for Mistral tokenizers." + ) + + with patch.object( + provider._client.chat.completions, "create", new_callable=AsyncMock + ) as mock_create: + mock_create.side_effect = [first_error, mock_stream()] + + events = [e async for e in provider.stream_response(req)] + + assert mock_create.await_count == 2 + + first_extra = mock_create.call_args_list[0].kwargs["extra_body"] + second_kwargs = mock_create.call_args_list[1].kwargs + + assert "chat_template" not in first_extra + assert first_extra["chat_template_kwargs"] == { "thinking": True, "enable_thinking": True, "reasoning_budget": 100, } - assert "reasoning_budget" not in second_extra + second_extra = second_kwargs.get("extra_body") or {} + assert "chat_template" not in second_extra + assert "chat_template_kwargs" not in second_extra event_text = "".join(events) assert "event: error" not in event_text diff --git a/tests/providers/test_nvidia_nim_request.py b/tests/providers/test_nvidia_nim_request.py index 276d5464..22acbd0c 100644 --- a/tests/providers/test_nvidia_nim_request.py +++ b/tests/providers/test_nvidia_nim_request.py @@ -318,13 +318,39 @@ class TestBuildRequestBody: assert cloned is not None assert "chat_template" not in cloned["extra_body"] - assert cloned["extra_body"]["chat_template_kwargs"] == { + assert "chat_template_kwargs" not in cloned["extra_body"] + assert cloned["extra_body"]["ignore_eos"] is False + assert body["extra_body"]["chat_template"] == "custom_template" + assert body["extra_body"]["chat_template_kwargs"] == { "thinking": True, "enable_thinking": True, "reasoning_budget": 100, } + + def test_clone_body_without_chat_template_kwargs_only(self): + body = { + "model": "test", + "extra_body": { + "chat_template_kwargs": { + "thinking": True, + "enable_thinking": True, + "reasoning_budget": 100, + }, + "ignore_eos": False, + }, + } + + cloned = clone_body_without_chat_template(body) + + assert cloned is not None + assert "chat_template" not in cloned["extra_body"] + assert "chat_template_kwargs" not in cloned["extra_body"] assert cloned["extra_body"]["ignore_eos"] is False - assert body["extra_body"]["chat_template"] == "custom_template" + + def test_clone_body_without_chat_template_returns_none_when_unchanged(self): + body = {"model": "test", "extra_body": {"ignore_eos": False}} + + assert clone_body_without_chat_template(body) is None def test_no_chat_template_kwargs_when_thinking_disabled(self): req = MagicMock() diff --git a/uv.lock b/uv.lock index 21eb51a9..7b8728f7 100644 --- a/uv.lock +++ b/uv.lock @@ -561,7 +561,7 @@ wheels = [ [[package]] name = "free-claude-code" -version = "3.4.2" +version = "3.4.3" source = { editable = "." } dependencies = [ { name = "aiohttp" },