diff --git a/pyproject.toml b/pyproject.toml index 16fd9f833..e807318ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ studio = [ "frontend/.git*", "backend/requirements/**/*", "backend/plugins/**/*", + "backend/assets/**/*.jinja", "backend/core/data_recipe/oxc-validator/*.json", "backend/core/data_recipe/oxc-validator/*.mjs", ] diff --git a/studio/backend/assets/chat_templates/gemma-4-edge.jinja b/studio/backend/assets/chat_templates/gemma-4-edge.jinja new file mode 100644 index 000000000..026612723 --- /dev/null +++ b/studio/backend/assets/chat_templates/gemma-4-edge.jinja @@ -0,0 +1,397 @@ +{#- + Gemma 4 chat template (E2B / E4B edge variant), vendored for Unsloth Studio. + Source: google/gemma-4-31B-it HF discussion/PR #118 (adds the preserve_thinking + flag plus null-rendering, string-arguments validation, balanced turn tags, empty + messages handling, and OpenAI image_url/input_audio aliases). + Studio-local changes vs PR #118: + 1. preserve_thinking defaults to false (see SETUP block below). + 2. The empty "<|channel>thought\n" block on enable_thinking=false is + NOT emitted. Google ships a distinct template for E2B/E4B (google/gemma-4-E2B-it, + google/gemma-4-E4B-it) that omits it; only the 12b/26B-A4B/31B family emits it. + This file matches the E2B/E4B behavior; gemma-4.jinja keeps the larger-model one. + Applied to unsloth/gemma-4-E2B-it-GGUF and unsloth/gemma-4-E4B-it-GGUF so the + embedded GGUF template does not need re-downloading. +-#} +{%- macro format_parameters(properties, required, filter_keys=false) -%} + {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in properties | dictsort -%} + {%- set add_comma = false -%} + {%- if not filter_keys or key not in standard_keys -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {{ key }}:{ + {%- if value['description'] -%} + description:<|"|>{{ value['description'] }}<|"|> + {%- set add_comma = true -%} + {%- endif -%} + {%- if value['type'] | upper == 'STRING' -%} + {%- if value['enum'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + enum:{{ format_argument(value['enum']) }} + {%- endif -%} + {%- elif value['type'] | upper == 'ARRAY' -%} + {%- if value['items'] is mapping and value['items'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + items:{ + {%- set ns_items = namespace(found_first=false) -%} + {%- for item_key, item_value in value['items'] | dictsort -%} + {%- if item_value is not none -%} + {%- if ns_items.found_first %},{% endif -%} + {%- set ns_items.found_first = true -%} + {%- if item_key == 'properties' -%} + properties:{ + {%- if item_value is mapping -%} + {{- format_parameters(item_value, value['items']['required'] | default([])) -}} + {%- endif -%} + } + {%- elif item_key == 'required' -%} + required:[ + {%- for req_item in item_value -%} + <|"|>{{- req_item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- elif item_key == 'type' -%} + {%- if item_value is string -%} + type:{{ format_argument(item_value | upper) }} + {%- else -%} + type:{{ format_argument(item_value | map('upper') | list) }} + {%- endif -%} + {%- else -%} + {{ item_key }}:{{ format_argument(item_value) }} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + } + {%- endif -%} + {%- endif -%} + {%- if value['nullable'] %} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + nullable:true + {%- endif -%} + {%- if value['type'] | upper == 'OBJECT' -%} + {%- if value['properties'] is defined and value['properties'] is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value['properties'], value['required'] | default([])) -}} + } + {%- elif value is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}} + } + {%- endif -%} + {%- if value['required'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + required:[ + {%- for item in value['required'] | default([]) -%} + <|"|>{{- item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- endif -%} + {%- endif -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + type:<|"|>{{ value['type'] | upper }}<|"|>} + {%- endif -%} + {%- endfor -%} +{%- endmacro -%} +{%- macro format_function_declaration(tool_data) -%} + declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|> + {%- set params = tool_data['function']['parameters'] -%} + {%- if params -%} + ,parameters:{ + {%- if params['properties'] -%} + properties:{ {{- format_parameters(params['properties'], params['required']) -}} }, + {%- endif -%} + {%- if params['required'] -%} + required:[ + {%- for item in params['required'] -%} + <|"|>{{- item -}}<|"|> + {{- ',' if not loop.last -}} + {%- endfor -%} + ], + {%- endif -%} + {%- if params['type'] -%} + type:<|"|>{{- params['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + {%- if 'response' in tool_data['function'] -%} + {%- set response_declaration = tool_data['function']['response'] -%} + ,response:{ + {%- if response_declaration['description'] -%} + description:<|"|>{{- response_declaration['description'] -}}<|"|>, + {%- endif -%} + {%- if response_declaration['type'] | upper == 'OBJECT' -%} + type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + } +{%- endmacro -%} +{%- macro format_argument(argument, escape_keys=True) -%} + {%- if argument is none -%} + {{- 'null' -}} + {%- elif argument is string -%} + {{- '<|"|>' + argument + '<|"|>' -}} + {%- elif argument is boolean -%} + {{- 'true' if argument else 'false' -}} + {%- elif argument is mapping -%} + {{- '{' -}} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in argument | dictsort -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {%- if escape_keys -%} + {{- '<|"|>' + key + '<|"|>' -}} + {%- else -%} + {{- key -}} + {%- endif -%} + :{{- format_argument(value, escape_keys=escape_keys) -}} + {%- endfor -%} + {{- '}' -}} + {%- elif argument is sequence -%} + {{- '[' -}} + {%- for item in argument -%} + {{- format_argument(item, escape_keys=escape_keys) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- ']' -}} + {%- else -%} + {{- argument -}} + {%- endif -%} +{%- endmacro -%} +{%- macro strip_thinking(text) -%} + {%- set ns = namespace(result='') -%} + {%- for part in text.split('') -%} + {%- if '<|channel>' in part -%} + {%- set ns.result = ns.result + part.split('<|channel>')[0] -%} + {%- else -%} + {%- set ns.result = ns.result + part -%} + {%- endif -%} + {%- endfor -%} + {{- ns.result | trim -}} +{%- endmacro -%} + +{%- macro format_tool_response_block(tool_name, response) -%} + {{- '<|tool_response>' -}} + {%- if response is mapping -%} + {{- 'response:' + tool_name + '{' -}} + {%- for key, value in response | dictsort -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- '}' -}} + {%- else -%} + {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}} + {%- endif -%} + {{- '' -}} +{%- endmacro -%} + +{#- ===== SETUP ===== -#} +{%- set ns = namespace(prev_message_type=None, prev_non_tool_role=None) -%} +{%- set loop_messages = messages -%} +{%- set enable_thinking = enable_thinking | default(false) -%} +{#- Unsloth Studio: preserve_thinking defaults OFF (upstream PR #118 defaults true). -#} +{%- set preserve_thinking = preserve_thinking | default(false) -%} +{{- bos_token -}} +{#- Handle System/Tool Definitions Block -#} +{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%} + {{- '<|turn>system\n' -}} + {#- Inject Thinking token at the very top of the FIRST system turn -#} + {%- if enable_thinking -%} + {{- '<|think|>\n' -}} + {%- set ns.prev_message_type = 'think' -%} + {%- endif -%} + {%- if messages and messages[0]['role'] in ['system', 'developer'] -%} + {%- if messages[0]['content'] is string -%} + {{- messages[0]['content'] | trim -}} + {%- elif messages[0]['content'] is sequence -%} + {%- for item in messages[0]['content'] -%} + {{- item['text'] | trim + ' '-}} + {%- endfor -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} + {%- endif -%} + {%- if tools -%} + {%- for tool in tools %} + {{- '<|tool>' -}} + {{- format_function_declaration(tool) | trim -}} + {{- '' -}} + {%- endfor %} + {%- set ns.prev_message_type = 'tool' -%} + {%- endif -%} + {{- '\n' -}} +{%- endif %} + +{#- Pre-scan: find last user message index for reasoning guard -#} +{%- set ns_turn = namespace(last_user_idx=-1) -%} +{%- for i in range(loop_messages | length) -%} + {%- if loop_messages[i]['role'] == 'user' -%} + {%- set ns_turn.last_user_idx = i -%} + {%- endif -%} +{%- endfor -%} + +{#- Loop through messages -#} +{%- for message in loop_messages -%} + {%- if message['role'] != 'tool' -%} + {%- set ns.prev_message_type = None -%} + {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%} + {#- Detect continuation using tracked state - O(1) instead of O(n) backward scan -#} + {%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%} + {%- if not continue_same_model_turn -%} + {{- '<|turn>' + role + '\n' }} + {%- endif -%} + + {#- Render reasoning/reasoning_content as thinking channel (tool-call turns only) -#} + {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%} + {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or preserve_thinking -%} + {%- if thinking_text and thinking_gate and message.get('tool_calls') -%} + {{- '<|channel>thought\n' + thinking_text + '\n' -}} + {%- endif -%} + + {%- if message.get('tool_calls') -%} + {%- for tool_call in message.get('tool_calls') -%} + {%- set function = tool_call['function'] -%} + {{- '<|tool_call>call:' + function['name'] + '{' -}} + {%- if function['arguments'] is mapping -%} + {%- set ns_args = namespace(found_first=false) -%} + {%- for key, value in function['arguments'] | dictsort -%} + {%- if ns_args.found_first %},{% endif -%} + {%- set ns_args.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {%- elif function['arguments'] is none -%} + {%- else -%} + {{- raise_exception( + "chat_template: tool_calls[].function.arguments must be a " + "JSON object (mapping), not a string. Deserialize arguments " + "before passing to the template." + ) -}} + {%- endif -%} + {{- '}' -}} + {%- endfor -%} + {%- set ns.prev_message_type = 'tool_call' -%} + {%- endif -%} + + {%- set ns_tr_out = namespace(flag=false) -%} + {%- if message.get('tool_responses') -%} + {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#} + {%- for tool_response in message.get('tool_responses') -%} + {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endfor -%} + {%- elif message.get('tool_calls') -%} + {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#} + {%- set ns_tool_scan = namespace(stopped=false) -%} + {%- for k in range(loop.index0 + 1, loop_messages | length) -%} + {%- if ns_tool_scan.stopped -%} + {%- elif loop_messages[k]['role'] != 'tool' -%} + {%- set ns_tool_scan.stopped = true -%} + {%- else -%} + {%- set follow = loop_messages[k] -%} + {#- Resolve tool_call_id to function name -#} + {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%} + {%- for tc in message.get('tool_calls') -%} + {%- if tc.get('id') == follow.get('tool_call_id') -%} + {%- set ns_tname.name = tc['function']['name'] -%} + {%- endif -%} + {%- endfor -%} + {#- Handle content as string or content-parts array -#} + {%- set tool_body = follow.get('content') -%} + {%- if tool_body is string -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- elif tool_body is sequence and tool_body is not string -%} + {%- set ns_txt = namespace(s='') -%} + {%- for part in tool_body -%} + {%- if part.get('type') == 'text' -%} + {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%} + {%- endif -%} + {%- endfor -%} + {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}} + {%- for part in tool_body -%} + {%- if part.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif part.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif part.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- endif -%} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + + {%- set captured_content -%} + {%- if message.get('content') is string -%} + {%- if role == 'model' -%} + {{- strip_thinking(message['content']) -}} + {%- else -%} + {{- message['content'] | trim -}} + {%- endif -%} + {%- elif message.get('content') is sequence -%} + {%- for item in message['content'] -%} + {%- if item.get('type') == 'text' -%} + {%- if role == 'model' -%} + {{- strip_thinking(item['text']) -}} + {%- else -%} + {{- item['text'] | trim -}} + {%- endif -%} + {%- elif item.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif item.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif item.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endset -%} + + {{- captured_content -}} + {%- set has_content = captured_content | trim | length > 0 -%} + + {#- Forward-scan: find next non-tool message role for continuation detection -#} + {%- set next_nt = namespace(role=None, found=false) -%} + {%- for j in range(loop.index0 + 1, loop_messages | length) -%} + {%- if not next_nt.found -%} + {%- if loop_messages[j]['role'] != 'tool' -%} + {%- set next_nt.role = loop_messages[j]['role'] -%} + {%- set next_nt.found = true -%} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + + {%- set continues_into_next = ( + role == 'model' + and next_nt.role == 'assistant' + and not message.get('tool_calls') + and not ns_tr_out.flag + ) -%} + + {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%} + {{- '<|tool_response>' -}} + {%- elif continues_into_next -%} + {{- '\n' -}} + {%- elif not (ns_tr_out.flag and not has_content) -%} + {{- '\n' -}} + {%- endif -%} + + {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#} + {%- set ns.prev_non_tool_role = message['role'] -%} + {%- endif -%} +{%- endfor -%} + +{%- if add_generation_prompt -%} + {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%} + {{- '<|turn>model\n' -}} + {%- endif -%} + {#- E2B/E4B do NOT emit an empty thought block when enable_thinking is false + (unlike the 12b/26B-A4B/31B family); see header. -#} +{%- endif -%} diff --git a/studio/backend/assets/chat_templates/gemma-4.jinja b/studio/backend/assets/chat_templates/gemma-4.jinja new file mode 100644 index 000000000..65ab39df5 --- /dev/null +++ b/studio/backend/assets/chat_templates/gemma-4.jinja @@ -0,0 +1,397 @@ +{#- + Gemma 4 chat template, vendored for Unsloth Studio. + Source: google/gemma-4-31B-it HF discussion/PR #118 (adds the preserve_thinking + flag plus null-rendering, string-arguments validation, balanced turn tags, empty + messages handling, and OpenAI image_url/input_audio aliases). + Studio-local change: preserve_thinking defaults to false (see SETUP block below). + Applied to unsloth/gemma-4-*-GGUF models so the embedded GGUF template does not + need re-downloading. Keep in sync with upstream if PR #118 changes. +-#} +{%- macro format_parameters(properties, required, filter_keys=false) -%} + {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in properties | dictsort -%} + {%- set add_comma = false -%} + {%- if not filter_keys or key not in standard_keys -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {{ key }}:{ + {%- if value['description'] -%} + description:<|"|>{{ value['description'] }}<|"|> + {%- set add_comma = true -%} + {%- endif -%} + {%- if value['type'] | upper == 'STRING' -%} + {%- if value['enum'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + enum:{{ format_argument(value['enum']) }} + {%- endif -%} + {%- elif value['type'] | upper == 'ARRAY' -%} + {%- if value['items'] is mapping and value['items'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + items:{ + {%- set ns_items = namespace(found_first=false) -%} + {%- for item_key, item_value in value['items'] | dictsort -%} + {%- if item_value is not none -%} + {%- if ns_items.found_first %},{% endif -%} + {%- set ns_items.found_first = true -%} + {%- if item_key == 'properties' -%} + properties:{ + {%- if item_value is mapping -%} + {{- format_parameters(item_value, value['items']['required'] | default([])) -}} + {%- endif -%} + } + {%- elif item_key == 'required' -%} + required:[ + {%- for req_item in item_value -%} + <|"|>{{- req_item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- elif item_key == 'type' -%} + {%- if item_value is string -%} + type:{{ format_argument(item_value | upper) }} + {%- else -%} + type:{{ format_argument(item_value | map('upper') | list) }} + {%- endif -%} + {%- else -%} + {{ item_key }}:{{ format_argument(item_value) }} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + } + {%- endif -%} + {%- endif -%} + {%- if value['nullable'] %} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + nullable:true + {%- endif -%} + {%- if value['type'] | upper == 'OBJECT' -%} + {%- if value['properties'] is defined and value['properties'] is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value['properties'], value['required'] | default([])) -}} + } + {%- elif value is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}} + } + {%- endif -%} + {%- if value['required'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + required:[ + {%- for item in value['required'] | default([]) -%} + <|"|>{{- item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- endif -%} + {%- endif -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + type:<|"|>{{ value['type'] | upper }}<|"|>} + {%- endif -%} + {%- endfor -%} +{%- endmacro -%} +{%- macro format_function_declaration(tool_data) -%} + declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|> + {%- set params = tool_data['function']['parameters'] -%} + {%- if params -%} + ,parameters:{ + {%- if params['properties'] -%} + properties:{ {{- format_parameters(params['properties'], params['required']) -}} }, + {%- endif -%} + {%- if params['required'] -%} + required:[ + {%- for item in params['required'] -%} + <|"|>{{- item -}}<|"|> + {{- ',' if not loop.last -}} + {%- endfor -%} + ], + {%- endif -%} + {%- if params['type'] -%} + type:<|"|>{{- params['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + {%- if 'response' in tool_data['function'] -%} + {%- set response_declaration = tool_data['function']['response'] -%} + ,response:{ + {%- if response_declaration['description'] -%} + description:<|"|>{{- response_declaration['description'] -}}<|"|>, + {%- endif -%} + {%- if response_declaration['type'] | upper == 'OBJECT' -%} + type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + } +{%- endmacro -%} +{%- macro format_argument(argument, escape_keys=True) -%} + {%- if argument is none -%} + {{- 'null' -}} + {%- elif argument is string -%} + {{- '<|"|>' + argument + '<|"|>' -}} + {%- elif argument is boolean -%} + {{- 'true' if argument else 'false' -}} + {%- elif argument is mapping -%} + {{- '{' -}} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in argument | dictsort -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {%- if escape_keys -%} + {{- '<|"|>' + key + '<|"|>' -}} + {%- else -%} + {{- key -}} + {%- endif -%} + :{{- format_argument(value, escape_keys=escape_keys) -}} + {%- endfor -%} + {{- '}' -}} + {%- elif argument is sequence -%} + {{- '[' -}} + {%- for item in argument -%} + {{- format_argument(item, escape_keys=escape_keys) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- ']' -}} + {%- else -%} + {{- argument -}} + {%- endif -%} +{%- endmacro -%} +{%- macro strip_thinking(text) -%} + {%- set ns = namespace(result='') -%} + {%- for part in text.split('') -%} + {%- if '<|channel>' in part -%} + {%- set ns.result = ns.result + part.split('<|channel>')[0] -%} + {%- else -%} + {%- set ns.result = ns.result + part -%} + {%- endif -%} + {%- endfor -%} + {{- ns.result | trim -}} +{%- endmacro -%} + +{%- macro format_tool_response_block(tool_name, response) -%} + {{- '<|tool_response>' -}} + {%- if response is mapping -%} + {{- 'response:' + tool_name + '{' -}} + {%- for key, value in response | dictsort -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- '}' -}} + {%- else -%} + {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}} + {%- endif -%} + {{- '' -}} +{%- endmacro -%} + +{#- ===== SETUP ===== -#} +{%- set ns = namespace(prev_message_type=None, prev_non_tool_role=None) -%} +{%- set loop_messages = messages -%} +{%- set enable_thinking = enable_thinking | default(false) -%} +{#- Unsloth Studio: preserve_thinking defaults OFF (upstream PR #118 defaults true). -#} +{%- set preserve_thinking = preserve_thinking | default(false) -%} +{{- bos_token -}} +{#- Handle System/Tool Definitions Block -#} +{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%} + {{- '<|turn>system\n' -}} + {#- Inject Thinking token at the very top of the FIRST system turn -#} + {%- if enable_thinking -%} + {{- '<|think|>\n' -}} + {%- set ns.prev_message_type = 'think' -%} + {%- endif -%} + {%- if messages and messages[0]['role'] in ['system', 'developer'] -%} + {%- if messages[0]['content'] is string -%} + {{- messages[0]['content'] | trim -}} + {%- elif messages[0]['content'] is sequence -%} + {%- for item in messages[0]['content'] -%} + {{- item['text'] | trim + ' '-}} + {%- endfor -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} + {%- endif -%} + {%- if tools -%} + {%- for tool in tools %} + {{- '<|tool>' -}} + {{- format_function_declaration(tool) | trim -}} + {{- '' -}} + {%- endfor %} + {%- set ns.prev_message_type = 'tool' -%} + {%- endif -%} + {{- '\n' -}} +{%- endif %} + +{#- Pre-scan: find last user message index for reasoning guard -#} +{%- set ns_turn = namespace(last_user_idx=-1) -%} +{%- for i in range(loop_messages | length) -%} + {%- if loop_messages[i]['role'] == 'user' -%} + {%- set ns_turn.last_user_idx = i -%} + {%- endif -%} +{%- endfor -%} + +{#- Loop through messages -#} +{%- for message in loop_messages -%} + {%- if message['role'] != 'tool' -%} + {%- set ns.prev_message_type = None -%} + {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%} + {#- Detect continuation using tracked state - O(1) instead of O(n) backward scan -#} + {%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%} + {%- if not continue_same_model_turn -%} + {{- '<|turn>' + role + '\n' }} + {%- endif -%} + + {#- Render reasoning/reasoning_content as thinking channel (tool-call turns only) -#} + {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%} + {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or preserve_thinking -%} + {%- if thinking_text and thinking_gate and message.get('tool_calls') -%} + {{- '<|channel>thought\n' + thinking_text + '\n' -}} + {%- endif -%} + + {%- if message.get('tool_calls') -%} + {%- for tool_call in message.get('tool_calls') -%} + {%- set function = tool_call['function'] -%} + {{- '<|tool_call>call:' + function['name'] + '{' -}} + {%- if function['arguments'] is mapping -%} + {%- set ns_args = namespace(found_first=false) -%} + {%- for key, value in function['arguments'] | dictsort -%} + {%- if ns_args.found_first %},{% endif -%} + {%- set ns_args.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {%- elif function['arguments'] is none -%} + {%- else -%} + {{- raise_exception( + "chat_template: tool_calls[].function.arguments must be a " + "JSON object (mapping), not a string. Deserialize arguments " + "before passing to the template." + ) -}} + {%- endif -%} + {{- '}' -}} + {%- endfor -%} + {%- set ns.prev_message_type = 'tool_call' -%} + {%- endif -%} + + {%- set ns_tr_out = namespace(flag=false) -%} + {%- if message.get('tool_responses') -%} + {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#} + {%- for tool_response in message.get('tool_responses') -%} + {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endfor -%} + {%- elif message.get('tool_calls') -%} + {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#} + {%- set ns_tool_scan = namespace(stopped=false) -%} + {%- for k in range(loop.index0 + 1, loop_messages | length) -%} + {%- if ns_tool_scan.stopped -%} + {%- elif loop_messages[k]['role'] != 'tool' -%} + {%- set ns_tool_scan.stopped = true -%} + {%- else -%} + {%- set follow = loop_messages[k] -%} + {#- Resolve tool_call_id to function name -#} + {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%} + {%- for tc in message.get('tool_calls') -%} + {%- if tc.get('id') == follow.get('tool_call_id') -%} + {%- set ns_tname.name = tc['function']['name'] -%} + {%- endif -%} + {%- endfor -%} + {#- Handle content as string or content-parts array -#} + {%- set tool_body = follow.get('content') -%} + {%- if tool_body is string -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- elif tool_body is sequence and tool_body is not string -%} + {%- set ns_txt = namespace(s='') -%} + {%- for part in tool_body -%} + {%- if part.get('type') == 'text' -%} + {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%} + {%- endif -%} + {%- endfor -%} + {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}} + {%- for part in tool_body -%} + {%- if part.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif part.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif part.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- endif -%} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + + {%- set captured_content -%} + {%- if message.get('content') is string -%} + {%- if role == 'model' -%} + {{- strip_thinking(message['content']) -}} + {%- else -%} + {{- message['content'] | trim -}} + {%- endif -%} + {%- elif message.get('content') is sequence -%} + {%- for item in message['content'] -%} + {%- if item.get('type') == 'text' -%} + {%- if role == 'model' -%} + {{- strip_thinking(item['text']) -}} + {%- else -%} + {{- item['text'] | trim -}} + {%- endif -%} + {%- elif item.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif item.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif item.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endset -%} + + {{- captured_content -}} + {%- set has_content = captured_content | trim | length > 0 -%} + + {#- Forward-scan: find next non-tool message role for continuation detection -#} + {%- set next_nt = namespace(role=None, found=false) -%} + {%- for j in range(loop.index0 + 1, loop_messages | length) -%} + {%- if not next_nt.found -%} + {%- if loop_messages[j]['role'] != 'tool' -%} + {%- set next_nt.role = loop_messages[j]['role'] -%} + {%- set next_nt.found = true -%} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + + {%- set continues_into_next = ( + role == 'model' + and next_nt.role == 'assistant' + and not message.get('tool_calls') + and not ns_tr_out.flag + ) -%} + + {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%} + {{- '<|tool_response>' -}} + {%- elif continues_into_next -%} + {{- '\n' -}} + {%- elif not (ns_tr_out.flag and not has_content) -%} + {{- '\n' -}} + {%- endif -%} + + {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#} + {%- set ns.prev_non_tool_role = message['role'] -%} + {%- endif -%} +{%- endfor -%} + +{%- if add_generation_prompt -%} + {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%} + {{- '<|turn>model\n' -}} + {%- endif -%} + + {%- if not enable_thinking -%} + {#- Suppress thinking - but not when awaiting tool responses -#} + {%- if ns.prev_message_type != 'tool_call' -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} + {%- endif -%} +{%- endif -%} diff --git a/studio/backend/core/inference/chat_templates.py b/studio/backend/core/inference/chat_templates.py new file mode 100644 index 000000000..58f63ff61 --- /dev/null +++ b/studio/backend/core/inference/chat_templates.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Bundled chat-template selection for GGUF inference. + +Some shipped GGUF quants embed an older chat template. Rather than re-cutting and +asking users to re-download every quant, Studio can override the embedded template +at llama-server launch time with a bundled, up-to-date Jinja template for known +model families. The override is wired through the existing ``chat_template_override`` +-> ``--chat-template-file`` path in ``LlamaCppBackend.load_model``. + +Currently this covers ``unsloth/gemma-4-*-GGUF``, which gains the upstream PR #118 +``preserve_thinking`` flag (defaulted OFF here) so the Studio "Preserve thinking" +toggle appears while staying disabled by default. +""" + +import re +from functools import lru_cache +from pathlib import Path +from typing import Optional + +# assets live at /assets/chat_templates/. This module is at +# /core/inference/chat_templates.py, so walk up three parents to +# (mirrors utils/inference/inference_config.py). +_ASSETS_DIR = Path(__file__).parent.parent.parent / "assets" / "chat_templates" + +# unsloth/gemma-4--GGUF (case-insensitive). The "-GGUF" suffix is retained +# on ModelConfig.identifier for HF GGUF repos, so this matches E2B / E4B / 31B / +# 26B-A4B and any future unsloth/gemma-4-*-GGUF, while excluding gemma-3, +# non-Unsloth, and non-GGUF identifiers (e.g. the bf16 "unsloth/gemma-4-E2B-it"). +_GEMMA4_GGUF_RE = re.compile(r"^unsloth/gemma-4-.+-gguf$", re.IGNORECASE) + +# Google ships two distinct gemma-4 chat templates: E2B/E4B omit the empty +# "<|channel>thought" block on enable_thinking=false, while the +# 12b/26B-A4B/31B family emits it. Route the two GGUF families to the matching +# bundled template so each keeps its model's intended behavior. +_GEMMA4_EDGE_GGUF_RE = re.compile(r"^unsloth/gemma-4-e[24]b-it-gguf$", re.IGNORECASE) + +_GEMMA4_TEMPLATE_FILE = "gemma-4.jinja" # 12b / 26B-A4B / 31B +_GEMMA4_EDGE_TEMPLATE_FILE = "gemma-4-edge.jinja" # E2B / E4B + + +def _canonical_repo_id(model_identifier: str) -> str: + """Mirror ``ModelConfig.from_identifier``: a bare HF shorthand with no owner + (e.g. ``gemma-4-E2B-it-GGUF``) defaults to the ``unsloth/`` org. The resolver + runs on the raw ``request.model_path`` (before that canonicalization), so apply + the same rule here, otherwise shorthand loads would skip the override. + """ + mid = model_identifier.strip() + if mid and "/" not in mid: + mid = f"unsloth/{mid}" + return mid + + +def is_unsloth_gemma4_gguf(model_identifier: Optional[str]) -> bool: + """True for canonical ``unsloth/gemma-4-*-GGUF`` repo identifiers (and the + owner-less shorthand that resolves to the same Unsloth repo).""" + if not model_identifier: + return False + return bool(_GEMMA4_GGUF_RE.match(_canonical_repo_id(model_identifier))) + + +def is_unsloth_gemma4_edge_gguf(model_identifier: Optional[str]) -> bool: + """True for the E2B / E4B GGUF repos, which use the edge-variant template.""" + if not model_identifier: + return False + return bool(_GEMMA4_EDGE_GGUF_RE.match(_canonical_repo_id(model_identifier))) + + +def _gemma4_template_file(model_identifier: Optional[str]) -> Optional[str]: + """Return the bundled template filename for a gemma-4 GGUF id, else None.""" + if is_unsloth_gemma4_edge_gguf(model_identifier): + return _GEMMA4_EDGE_TEMPLATE_FILE + if is_unsloth_gemma4_gguf(model_identifier): + return _GEMMA4_TEMPLATE_FILE + return None + + +@lru_cache(maxsize=8) +def load_bundled_chat_template(name: str) -> str: + """Read a bundled chat-template asset by filename (cached for the process).""" + return (_ASSETS_DIR / name).read_text(encoding="utf-8") + + +def resolve_effective_chat_template_override( + *, + model_identifier: Optional[str], + user_override: Optional[str], +) -> Optional[str]: + """Resolve which chat-template text to launch llama-server with. + + Precedence: + 1. An explicit, non-empty user override always wins (advanced users). + 2. For ``unsloth/gemma-4-*-GGUF``, return the bundled gemma-4 template + (adds ``preserve_thinking``, default off) so the embedded GGUF template + is overridden without re-downloading quants. E2B/E4B get the edge + variant; 12b/26B-A4B/31B get the standard one. + 3. Otherwise ``None`` -> llama-server renders the GGUF's embedded template. + + The result is fed to ``LlamaCppBackend.load_model(chat_template_override=...)`` + and must be computed before the route-level reload-dedup check so the live + backend state and the incoming request compare consistently. + """ + if user_override and user_override.strip(): + return user_override + template_file = _gemma4_template_file(model_identifier) + if template_file is not None: + return load_bundled_chat_template(template_file) + return None diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 486cff1bc..b5d5328fb 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -3966,6 +3966,7 @@ class LlamaCppBackend: self._chat_template_file = tempfile.NamedTemporaryFile( mode = "w", + encoding = "utf-8", suffix = ".jinja", delete = False, prefix = "unsloth_chat_template_", @@ -3987,6 +3988,13 @@ class LlamaCppBackend: thinking_default = False self._reasoning_default = thinking_default reasoning_kw = self._reasoning_kwargs(thinking_default) + # preserve_thinking is an independent kwarg. Default it OFF + # at launch so direct OpenAI-compatible callers that omit the + # field match the UI's default-off behavior (the bundled + # gemma-4 template also defaults it false; the frontend sends + # preserve_thinking per request once toggled on). + if self._supports_preserve_thinking: + reasoning_kw["preserve_thinking"] = False cmd.extend( [ "--chat-template-kwargs", diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index ab8097b65..fa708d0e3 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -714,6 +714,7 @@ from state.tool_approvals import resolve_tool_decision from core.inference.key_exchange import decrypt_api_key from core.inference.providers import get_provider_info, get_base_url from core.inference.external_provider import ExternalProviderClient +from core.inference.chat_templates import resolve_effective_chat_template_override from storage import providers_db from utils.utils import safe_error_detail, log_and_http_error @@ -1180,9 +1181,19 @@ def _should_strip_split_mode(request: LoadRequest, backend_extra: Optional[list[ ) -def _request_matches_loaded_settings(request: LoadRequest, llama_backend: LlamaCppBackend) -> bool: +def _request_matches_loaded_settings( + request: LoadRequest, + llama_backend: LlamaCppBackend, + effective_chat_template_override: Optional[str] = None, +) -> bool: """True iff every runtime setting on the request matches the loaded server. - Caller has already checked model+variant+is_loaded. See #5401.""" + Caller has already checked model+variant+is_loaded. See #5401. + + ``effective_chat_template_override`` is the resolved template that will be + launched (user override, else a bundled family template such as the + gemma-4 override), so the dedup compares against what the backend actually + holds rather than the raw request field. Defaults to the request field for + callers that do not resolve a bundled override.""" # Compare requested n_ctx (not effective) so VRAM-cap doesn't mask an # Auto-vs-explicit slider flip. if request.max_seq_length != llama_backend.requested_n_ctx: @@ -1222,7 +1233,12 @@ def _request_matches_loaded_settings(request: LoadRequest, llama_backend: LlamaC if backend_mode in ("mtp", "mtp+ngram") and request.spec_draft_n_max is not None: if int(request.spec_draft_n_max) != (llama_backend.spec_draft_n_max or 0): return False - if (request.chat_template_override or None) != (llama_backend.chat_template_override or None): + _effective_cto = ( + effective_chat_template_override + if effective_chat_template_override is not None + else request.chat_template_override + ) + if (_effective_cto or None) != (llama_backend.chat_template_override or None): return False # llama_extra_args=None means "inherit"; only an explicit differing list # forces a reload. On the inherit path, refuse to match if stored extras @@ -1348,6 +1364,17 @@ async def load_model( # Version switching is handled by the subprocess-based inference # backend -- no ensure_transformers_version() needed here. + # Resolve the effective chat-template override once, up front: an + # explicit user override, else a bundled family template (e.g. the + # gemma-4 override that ships preserve_thinking without re-downloading + # quants), else None. Used for both the reload-dedup check below and the + # load_model calls, so the live backend state and the incoming request + # compare against the same template text. + effective_chat_template_override = resolve_effective_chat_template_override( + model_identifier = model_identifier, + user_override = request.chat_template_override, + ) + # ── Already-loaded check: skip reload if the exact model is active ── backend = get_inference_backend() llama_backend = get_llama_cpp_backend() @@ -1365,7 +1392,9 @@ async def load_model( and llama_backend.model_identifier and llama_backend.model_identifier.lower() == model_identifier.lower() # Match runtime settings so Apply isn't dropped (#5401). - and _request_matches_loaded_settings(request, llama_backend) + and _request_matches_loaded_settings( + request, llama_backend, effective_chat_template_override + ) # Skip if a prior audio probe failed -- let load_model retry. and getattr(llama_backend, "_audio_probed", True) ): @@ -1530,7 +1559,13 @@ async def load_model( else: # Strip only the groups whose first-class field was set by # the caller, so an inherited --chat-template-file survives - # an Apply that omits chat_template_override. + # an Apply that omits chat_template_override. A bundled family + # template (e.g. the gemma-4 override) is an effective + # first-class template setting even when the raw request + # omits chat_template_override, so strip the inherited + # --chat-template-file in that case too -- otherwise the stale + # extra arg (appended last) shadows the bundled template while + # Studio reports the bundled template's capabilities. fields_set = getattr(request, "model_fields_set", set()) stripped = strip_shadowing_flags( llama_backend.extra_args, @@ -1539,7 +1574,10 @@ async def load_model( strip_spec = ( "speculative_type" in fields_set or "spec_draft_n_max" in fields_set ), - strip_template = "chat_template_override" in fields_set, + strip_template = ( + "chat_template_override" in fields_set + or effective_chat_template_override is not None + ), strip_split_mode = _should_strip_split_mode( request, llama_backend.extra_args ), @@ -1573,7 +1611,7 @@ async def load_model( model_identifier = config.identifier, is_vision = config.is_vision, n_ctx = request.max_seq_length, - chat_template_override = request.chat_template_override, + chat_template_override = effective_chat_template_override, cache_type_kv = request.cache_type_kv, speculative_type = request.speculative_type, spec_draft_n_max = request.spec_draft_n_max, @@ -2148,6 +2186,21 @@ async def get_status(current_subject: str = Depends(get_current_subject)): _display_model_id = os.path.basename(_model_id) _inference_cfg = load_inference_config(_model_id) if _model_id else None _audio_type = getattr(llama_backend, "_audio_type", None) + # Don't surface Studio's auto-applied bundled family template (e.g. the + # gemma-4 override) as a user-authored override: the frontend adopts + # status.chat_template_override as editable state and would otherwise + # re-send it as an explicit override for a later, unrelated model. Only + # expose a genuine user override. + _reported_chat_template_override = llama_backend.chat_template_override + _auto_chat_template_override = resolve_effective_chat_template_override( + model_identifier = _model_id, + user_override = None, + ) + if ( + _auto_chat_template_override is not None + and _reported_chat_template_override == _auto_chat_template_override + ): + _reported_chat_template_override = None return InferenceStatusResponse( active_model = _display_model_id, model_identifier = None if _native_grant_backed else _model_id, @@ -2174,7 +2227,7 @@ async def get_status(current_subject: str = Depends(get_current_subject)): max_context_length = llama_backend.max_context_length, native_context_length = llama_backend.native_context_length, cache_type_kv = llama_backend.cache_type_kv, - chat_template_override = llama_backend.chat_template_override, + chat_template_override = _reported_chat_template_override, speculative_type = llama_backend.requested_spec_mode, spec_draft_n_max = llama_backend.spec_draft_n_max, tensor_parallel = llama_backend.tensor_parallel, diff --git a/studio/backend/tests/test_gemma4_chat_template_override.py b/studio/backend/tests/test_gemma4_chat_template_override.py new file mode 100644 index 000000000..f726741aa --- /dev/null +++ b/studio/backend/tests/test_gemma4_chat_template_override.py @@ -0,0 +1,345 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Auto-override of the chat template for ``unsloth/gemma-4-*-GGUF``. + +Studio ships a bundled ``gemma-4.jinja`` (PR #118 based, ``preserve_thinking`` +defaulted off) and applies it to gemma-4 GGUF loads via the existing +``chat_template_override`` -> ``--chat-template-file`` path, so users do not need +to re-download quants. Pins the family matcher, the resolver precedence, the +bundled asset's reasoning/tool capabilities (which drive the "Preserve thinking" +UI toggle), the Jinja gate behaviour, and the reload-dedup interaction. +""" + +from __future__ import annotations + +import importlib.util +import sys +import types as _types +from pathlib import Path + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +import pytest + +# ── chat_templates is dependency-light: load it directly so the pure-logic +# tests run without the studio venv / core.inference package side effects. ── +_CT_PATH = Path(_BACKEND_DIR) / "core" / "inference" / "chat_templates.py" +_ct_spec = importlib.util.spec_from_file_location("_gemma4_ct_test", _CT_PATH) +chat_templates = importlib.util.module_from_spec(_ct_spec) +_ct_spec.loader.exec_module(chat_templates) + +is_unsloth_gemma4_gguf = chat_templates.is_unsloth_gemma4_gguf +resolve_effective_chat_template_override = chat_templates.resolve_effective_chat_template_override +load_bundled_chat_template = chat_templates.load_bundled_chat_template +is_unsloth_gemma4_edge_gguf = chat_templates.is_unsloth_gemma4_edge_gguf + +BUNDLED = load_bundled_chat_template("gemma-4.jinja") # 12b / 26B-A4B / 31B +EDGE = load_bundled_chat_template("gemma-4-edge.jinja") # E2B / E4B + + +# ── Stubs so core.inference.llama_cpp imports without the full studio venv ── +def _stub_modules_ctx(): + """patch.dict context that stubs the heavy deps llama_cpp pulls in at import, + but only those NOT already importable (real httpx / structlog are kept when + present, e.g. in CI), and removes the stubs on exit so other tests are not + polluted.""" + from unittest.mock import patch + + _loggers_stub = _types.ModuleType("loggers") + _loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name) + _structlog_stub = _types.ModuleType("structlog") + _structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub") + _httpx_stub = _types.ModuleType("httpx") + for _exc in ( + "ConnectError", + "TimeoutException", + "ReadTimeout", + "ReadError", + "RemoteProtocolError", + "CloseError", + ): + setattr(_httpx_stub, _exc, type(_exc, (Exception,), {})) + _httpx_stub.Timeout = type("T", (), {"__init__": lambda s, *a, **k: None}) + _httpx_stub.Client = type( + "C", + (), + { + "__init__": lambda s, **kw: None, + "__enter__": lambda s: s, + "__exit__": lambda s, *a: None, + }, + ) + overrides = { + name: stub + for name, stub in ( + ("loggers", _loggers_stub), + ("structlog", _structlog_stub), + ("httpx", _httpx_stub), + ) + if name not in sys.modules + } + return patch.dict(sys.modules, overrides) + + +def _detect_reasoning_flags(): + with _stub_modules_ctx(): + from core.inference.llama_cpp import detect_reasoning_flags + return detect_reasoning_flags + + +# ── Family matcher ─────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "model_id,expected", + [ + ("unsloth/gemma-4-E2B-it-GGUF", True), + ("unsloth/gemma-4-E4B-it-GGUF", True), + ("unsloth/gemma-4-31B-it-GGUF", True), + ("unsloth/gemma-4-26B-A4B-it-GGUF", True), + ("UNSLOTH/GEMMA-4-E2B-IT-GGUF", True), # case-insensitive + ("gemma-4-E2B-it-GGUF", True), # owner-less shorthand -> unsloth/ + ("gemma-4-31B-it-GGUF", True), # owner-less shorthand -> unsloth/ + ("unsloth/gemma-4-E2B-it", False), # bf16, not GGUF + ("unsloth/gemma-3-4b-it-GGUF", False), # gemma 3 + ("google/gemma-4-31B-it-GGUF", False), # not unsloth + ("unsloth/Qwen3.5-9B-MTP-GGUF", False), + ("/home/user/models/gemma-4-E2B.Q4_K_M.gguf", False), # local path + ("", False), + (None, False), + ], +) +def test_is_unsloth_gemma4_gguf(model_id, expected): + assert is_unsloth_gemma4_gguf(model_id) is expected + + +# ── Resolver precedence ────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "model_id,expected_edge", + [ + ("unsloth/gemma-4-E2B-it-GGUF", True), + ("unsloth/gemma-4-E4B-it-GGUF", True), + ("UNSLOTH/GEMMA-4-E4B-IT-GGUF", True), + ("unsloth/gemma-4-12b-it-GGUF", False), + ("unsloth/gemma-4-26B-A4B-it-GGUF", False), + ("unsloth/gemma-4-31B-it-GGUF", False), + ("unsloth/gemma-3-4b-it-GGUF", False), + ], +) +def test_is_unsloth_gemma4_edge_gguf(model_id, expected_edge): + assert is_unsloth_gemma4_edge_gguf(model_id) is expected_edge + + +def test_resolver_returns_edge_template_for_e2b_e4b(): + for mid in ("unsloth/gemma-4-E2B-it-GGUF", "unsloth/gemma-4-E4B-it-GGUF"): + out = resolve_effective_chat_template_override(model_identifier = mid, user_override = None) + assert out == EDGE + assert out != BUNDLED + + +def test_resolver_handles_owner_less_shorthand(): + # ModelConfig.from_identifier prefixes unsloth/ for bare ids; the resolver + # runs before that, so it must apply the same normalization. + assert ( + resolve_effective_chat_template_override( + model_identifier = "gemma-4-E2B-it-GGUF", user_override = None + ) + == EDGE + ) + assert ( + resolve_effective_chat_template_override( + model_identifier = "gemma-4-31B-it-GGUF", user_override = None + ) + == BUNDLED + ) + + +def test_resolver_returns_standard_template_for_larger_models(): + for mid in ( + "unsloth/gemma-4-12b-it-GGUF", + "unsloth/gemma-4-26B-A4B-it-GGUF", + "unsloth/gemma-4-31B-it-GGUF", + ): + out = resolve_effective_chat_template_override(model_identifier = mid, user_override = None) + assert out == BUNDLED + + +def test_resolver_user_override_wins(): + out = resolve_effective_chat_template_override( + model_identifier = "unsloth/gemma-4-E2B-it-GGUF", user_override = "MY TEMPLATE" + ) + assert out == "MY TEMPLATE" + + +def test_resolver_blank_override_falls_back_to_bundled(): + out = resolve_effective_chat_template_override( + model_identifier = "unsloth/gemma-4-31B-it-GGUF", user_override = " " + ) + assert out == BUNDLED + + +def test_resolver_none_for_non_gemma(): + assert ( + resolve_effective_chat_template_override( + model_identifier = "unsloth/Llama-3.2-1B-Instruct-GGUF", user_override = None + ) + is None + ) + + +# ── Bundled asset content + capability classification ──────────────── + + +@pytest.mark.parametrize("tpl", [BUNDLED, EDGE]) +def test_bundled_template_has_preserve_thinking_defaulted_off(tpl): + assert "preserve_thinking" in tpl + assert "preserve_thinking | default(false)" in tpl + + +@pytest.mark.parametrize("name", ["gemma-4.jinja", "gemma-4-edge.jinja"]) +def test_bundled_templates_are_ascii(name): + # The temp file written for --chat-template-file must encode on any locale. + # Keeping the bundled templates ASCII avoids UnicodeEncodeError on non-UTF-8 + # Windows locales (cp932/cp1252) regardless of the writer's encoding. + text = load_bundled_chat_template(name) + non_ascii = sorted({c for c in text if ord(c) > 127}) + assert not non_ascii, f"{name} has non-ASCII chars: {non_ascii}" + + +@pytest.mark.parametrize("tpl", [BUNDLED, EDGE]) +def test_detect_reasoning_flags_on_bundled_template(tpl): + detect_reasoning_flags = _detect_reasoning_flags() + flags = detect_reasoning_flags(tpl, "unsloth/gemma-4-E2B-it-GGUF") + assert flags["supports_reasoning"] is True + assert flags["reasoning_style"] == "enable_thinking" + assert flags["reasoning_always_on"] is False + # This is what makes the "Preserve thinking" toggle appear in the UI. + assert flags["supports_preserve_thinking"] is True + assert flags["supports_tools"] is True + + +def test_edge_template_omits_empty_thought_block_on_thinking_off(): + """E2B/E4B must NOT emit the empty <|channel>thought block when + thinking is disabled; the larger-model template must. This is the only + intended difference between the two bundled templates.""" + EMPTY = "<|channel>thought\n" + msgs = [{"role": "user", "content": "hi"}] + edge_off = _render_with(EDGE, msgs, enable_thinking = False) + std_off = _render_with(BUNDLED, msgs, enable_thinking = False) + assert EMPTY not in edge_off, "edge (E2B/E4B) should not emit empty thought block" + assert EMPTY in std_off, "standard (12b/26B/31B) should emit empty thought block" + # With thinking ON neither appends the empty block at the prompt tail. + assert EMPTY not in _render_with(EDGE, msgs, enable_thinking = True) + + +# ── Jinja gate behaviour (off = omit prior reasoning, on = keep) ───── + + +def _render_with(tpl, messages, **kw): + pytest.importorskip("jinja2") # transitive via transformers; skip in minimal envs + from jinja2 import Environment, BaseLoader + + def raise_exception(msg): + raise RuntimeError(msg) + + env = Environment(loader = BaseLoader()) + return env.from_string(tpl).render( + messages = messages, + bos_token = "", + raise_exception = raise_exception, + add_generation_prompt = True, + **kw, + ) + + +def _render(messages, **kw): + return _render_with(BUNDLED, messages, **kw) + + +def _convo_with_prior_tool_reasoning(): + # Assistant tool-call turn with reasoning, BEFORE the last user message. + return [ + {"role": "user", "content": "q1"}, + { + "role": "assistant", + "reasoning_content": "SECRET_THOUGHT", + "tool_calls": [{"id": "c1", "function": {"name": "f", "arguments": {"x": 1}}}], + }, + {"role": "tool", "tool_call_id": "c1", "content": "42"}, + {"role": "user", "content": "q2"}, + ] + + +def test_preserve_thinking_off_omits_prior_reasoning(): + # default(false): kwarg unset -> prior reasoning dropped before last user turn. + assert "SECRET_THOUGHT" not in _render(_convo_with_prior_tool_reasoning()) + + +def test_preserve_thinking_on_keeps_prior_reasoning(): + assert "SECRET_THOUGHT" in _render(_convo_with_prior_tool_reasoning(), preserve_thinking = True) + + +def test_enable_thinking_gates_think_token(): + assert "<|think|>" in _render([{"role": "user", "content": "hi"}], enable_thinking = True) + assert "<|think|>" not in _render([{"role": "user", "content": "hi"}], enable_thinking = False) + + +# ── Reload dedup interaction (why the route resolves the effective override) ── + + +def test_already_in_target_state_consistent_with_bundled_override(): + """The backend dedup compares the incoming override against the live one. + + The route resolves the bundled template up front so a re-load that omits + ``chat_template_override`` still matches (no spurious reload), while a raw + ``None`` would not. + """ + LlamaCppBackend = _import_backend() + + class _FakeProcess: + def terminate(self): ... + def wait(self, timeout = None): + return 0 + + def kill(self): ... + def poll(self): + return 0 + + backend = LlamaCppBackend() + backend._process = _FakeProcess() + backend._healthy = True + backend._model_identifier = "unsloth/gemma-4-E2B-it-GGUF" + backend._hf_variant = "Q4_K_M" + backend._requested_n_ctx = 8192 + backend._cache_type_kv = None + backend._speculative_type = None + backend._requested_spec_mode = "auto" + backend._chat_template_override = BUNDLED # live server launched with the bundle + backend._is_vision = False + backend._extra_args = None + backend._gguf_path = None + + common = dict( + model_identifier = "unsloth/gemma-4-E2B-it-GGUF", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = None, + extra_args = None, + is_vision = False, + ) + # Effective (resolved bundled) override -> already loaded, no reload. + assert backend._already_in_target_state(chat_template_override = BUNDLED, **common) is True + # Raw None (unresolved) -> false match, would force a needless reload. + assert backend._already_in_target_state(chat_template_override = None, **common) is False + + +def _import_backend(): + with _stub_modules_ctx(): + from core.inference.llama_cpp import LlamaCppBackend + return LlamaCppBackend