diff --git a/common/arg.cpp b/common/arg.cpp index 1f70d0ad4..3e63ce0a6 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -3646,6 +3646,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } } ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING")); + add_opt(common_arg( + {"--reasoning-effort"}, "LEVEL", + "reasoning effort level given to the chat template: 'default' to keep the template default,\n" + "or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)", + [](common_params & params, const std::string & value) { + if (value == "default") { + params.default_template_kwargs.erase("reasoning_effort"); + } else { + params.default_template_kwargs["reasoning_effort"] = json(value).dump(); + } + } + ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING_EFFORT")); add_opt(common_arg( {"--reasoning-budget"}, "N", "token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)", diff --git a/common/chat.cpp b/common/chat.cpp index faf51dcd2..a45c51fa0 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -920,6 +920,10 @@ static std::string common_chat_template_direct_apply_impl( bool enabled = inp["preserve_reasoning"].get(); jinja::caps_apply_preserve_reasoning(ctx, enabled); } + if (inp.contains("reasoning_effort") && inp["reasoning_effort"].is_string() && !inp["reasoning_effort"].empty()) { + std::string reasoning_effort = inp["reasoning_effort"].get(); + jinja::caps_apply_reasoning_effort(ctx, reasoning_effort); + } jinja::global_from_json(ctx, inp, inputs.mark_input); diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cpp index cdd7ccfa2..fb97909ed 100644 --- a/common/jinja/caps.cpp +++ b/common/jinja/caps.cpp @@ -17,7 +17,7 @@ namespace jinja { using caps_json_fn = std::function; using caps_ctx_fn = std::function; -using caps_analyze_fn = std::function; +using caps_analyze_fn = std::function; void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) { ctx.set_val("preserve_thinking", mk_val(enabled)); @@ -26,6 +26,12 @@ void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) { ctx.set_val("drop_thinking", mk_val(!enabled)); } +void caps_apply_reasoning_effort(jinja::context & ctx, const std::string & effort) { + value var = mk_val(effort); // bind to the same value for stats + ctx.set_val("reasoning_effort", var); + ctx.set_val("reasoning_strength", var); +} + static void caps_try_execute(jinja::program & prog, const caps_json_fn & messages_fn, const caps_ctx_fn & ctx_fn, @@ -62,7 +68,7 @@ static void caps_try_execute(jinja::program & prog, // ignore exceptions during capability analysis } - analyze_fn(success, messages, tools, result); + analyze_fn(ctx, success, messages, tools, result); } // for debugging only @@ -87,6 +93,7 @@ std::map caps::to_map() const { {"supports_parallel_tool_calls", supports_parallel_tool_calls}, {"supports_system_role", supports_system_role}, {"supports_preserve_reasoning", supports_preserve_reasoning}, + {"supports_reasoning_effort", supports_reasoning_effort}, {"supports_object_arguments", supports_object_arguments}, }; } @@ -124,7 +131,7 @@ caps caps_get(jinja::program & prog) { }, nullptr, // ctx_fn nullptr, // tools_fn - [&](bool success, value & messages, value &, const std::string &) { + [&](context &, bool success, value & messages, value &, const std::string &) { auto & content = messages->at(0)->at("content"); caps_print_stats(content, "messages[0].content"); if (has_op(content, "selectattr") || has_op(content, "array_access")) { @@ -158,7 +165,7 @@ caps caps_get(jinja::program & prog) { }, nullptr, // ctx_fn nullptr, // tools_fn - [&](bool, value & messages, value &, const std::string &) { + [&](context &, bool, value & messages, value &, const std::string &) { auto & content = messages->at(0)->at("content"); caps_print_stats(content, "messages[0].content"); if (!content->stats.used) { @@ -234,7 +241,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value & tools, const std::string &) { + [&](context &, bool success, value & messages, value & tools, const std::string &) { if (!success) { return; // Nothing can be inferred } @@ -327,7 +334,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value & tools, const std::string &) { + [&](context &, bool success, value & messages, value & tools, const std::string &) { if (!success) { result.supports_tool_calls = false; result.supports_tools = false; @@ -429,7 +436,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value &, const std::string &) { + [&](context &, bool success, value & messages, value &, const std::string &) { if (!success) { result.supports_parallel_tool_calls = false; return; @@ -486,7 +493,7 @@ caps caps_get(jinja::program & prog) { caps_apply_preserve_reasoning(ctx, true); }, nullptr, // tools_fn - [&](bool, value &, value &, const std::string & output) { + [&](context &, bool, value &, value &, const std::string & output) { // note: we cannot use stats here because the reasoning_content may be used for "if" condition test, but not actually outputted in the final result if (output.find(reasoning_placeholder) != std::string::npos) { result.supports_preserve_reasoning = true; @@ -494,6 +501,32 @@ caps caps_get(jinja::program & prog) { } ); + JJ_DEBUG("%s\n", ">>> Running capability check: reasoning effort"); + + // case: reasoning effort level + caps_try_execute( + prog, + [&]() { + // messages + return json::array({ + { + {"role", "user"}, + {"content", "User message"} + }, + }); + }, + [&](context & ctx) { + ctx.set_val("enable_thinking", mk_val(true)); + caps_apply_reasoning_effort(ctx, "low"); + }, + nullptr, // tools_fn + [&](context & ctx, bool, value &, value &, const std::string &) { + value effort = ctx.get_val("reasoning_effort"); + caps_print_stats(effort, "reasoning_effort"); + result.supports_reasoning_effort = effort->stats.used; + } + ); + JJ_DEBUG("%s\n", result.to_string().c_str()); return result; diff --git a/common/jinja/caps.h b/common/jinja/caps.h index a290cd7da..b81dd95f2 100644 --- a/common/jinja/caps.h +++ b/common/jinja/caps.h @@ -16,6 +16,9 @@ struct caps { // supports preserve reasoning trace in the full history, not just the last assistant message bool supports_preserve_reasoning = false; + // supports reasoning effort levels + bool supports_reasoning_effort = false; + // one of the 2 content capabilities must be true bool supports_string_content = true; bool supports_typed_content = false; @@ -32,5 +35,6 @@ struct caps { caps caps_get(jinja::program & prog); void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled); +void caps_apply_reasoning_effort(jinja::context & ctx, const std::string & effort); } // namespace jinja diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index de75ad330..51ad85484 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -6955,6 +6955,24 @@ static void test_reasoning_budget_message_per_request() { } } +static void test_reasoning_effort_caps() { + LOG_DBG("%s\n", __func__); + + auto assert_supports_effort = [](const std::string & path, bool expected) { + auto tmpls = read_templates(path); + assert_equals(expected, common_chat_templates_get_caps(tmpls.get()).at("supports_reasoning_effort")); + }; + + assert_supports_effort("models/templates/deepseek-ai-DeepSeek-V4.jinja", true); + assert_supports_effort("models/templates/muse-glimmer.jinja", true); + assert_supports_effort("models/templates/tencent-Hy3.jinja", true); + assert_supports_effort("models/templates/openai-gpt-oss-120b.jinja", true); + assert_supports_effort("models/templates/upstage-Solar-Open-100B.jinja", true); + assert_supports_effort("models/templates/Cohere2MoE.jinja", true); + assert_supports_effort("models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja", false); + assert_supports_effort("models/templates/Qwen-Qwen3-0.6B.jinja", false); +} + static void test_msg_diffs_compute() { LOG_DBG("%s\n", __func__); { @@ -7114,6 +7132,7 @@ int main(int argc, char ** argv) { test_deepseek_v4_thinking_retention(); test_deepseek_v4_tool_result_ordering(); test_template_generation_prompt(); + test_reasoning_effort_caps(); test_reasoning_budget_tokens_per_request(); test_reasoning_budget_message_per_request(); test_template_output_peg_parsers(detailed_debug); diff --git a/tools/cli/README.md b/tools/cli/README.md index 880c4a540..b3543ed4d 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -170,6 +170,7 @@ | `--jinja, --no-jinja` | whether to use jinja template engine for chat (default: enabled)
(env: LLAMA_ARG_JINJA) | | `--reasoning-format FORMAT` | controls whether thought tags are allowed and/or extracted from the response, and in which format they're returned; one of:
- none: leaves thoughts unparsed in `message.content`
- deepseek: puts thoughts in `message.reasoning_content`
- deepseek-legacy: keeps `` tags in `message.content` while also populating `message.reasoning_content`
(default: auto)
(env: LLAMA_ARG_THINK) | | `-rea, --reasoning [on\|off\|auto]` | Use reasoning/thinking in the chat ('on', 'off', or 'auto', default: 'auto' (detect from template))
(env: LLAMA_ARG_REASONING) | +| `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,
or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)
(env: LLAMA_ARG_REASONING_EFFORT) | | `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)
(env: LLAMA_ARG_THINK_BUDGET) | | `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)
(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) | | `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | diff --git a/tools/completion/README.md b/tools/completion/README.md index c2e52ac06..3bc2d823e 100644 --- a/tools/completion/README.md +++ b/tools/completion/README.md @@ -251,6 +251,7 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `--jinja, --no-jinja` | whether to use jinja template engine for chat (default: disabled)
(env: LLAMA_ARG_JINJA) | | `--reasoning-format FORMAT` | controls whether thought tags are allowed and/or extracted from the response, and in which format they're returned; one of:
- none: leaves thoughts unparsed in `message.content`
- deepseek: puts thoughts in `message.reasoning_content`
- deepseek-legacy: keeps `` tags in `message.content` while also populating `message.reasoning_content`
(default: auto)
(env: LLAMA_ARG_THINK) | | `-rea, --reasoning [on\|off\|auto]` | Use reasoning/thinking in the chat ('on', 'off', or 'auto', default: 'auto' (detect from template))
(env: LLAMA_ARG_REASONING) | +| `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,
or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)
(env: LLAMA_ARG_REASONING_EFFORT) | | `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)
(env: LLAMA_ARG_THINK_BUDGET) | | `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)
(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) | | `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | diff --git a/tools/server/README.md b/tools/server/README.md index a2ab872b4..ec3a33e0c 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -226,6 +226,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--jinja, --no-jinja` | whether to use jinja template engine for chat (default: enabled)
(env: LLAMA_ARG_JINJA) | | `--reasoning-format FORMAT` | controls whether thought tags are allowed and/or extracted from the response, and in which format they're returned; one of:
- none: leaves thoughts unparsed in `message.content`
- deepseek: puts thoughts in `message.reasoning_content`
- deepseek-legacy: keeps `` tags in `message.content` while also populating `message.reasoning_content`
(default: auto)
(env: LLAMA_ARG_THINK) | | `-rea, --reasoning [on\|off\|auto]` | Use reasoning/thinking in the chat ('on', 'off', or 'auto', default: 'auto' (detect from template))
(env: LLAMA_ARG_REASONING) | +| `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,
or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)
(env: LLAMA_ARG_REASONING_EFFORT) | | `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)
(env: LLAMA_ARG_THINK_BUDGET) | | `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)
(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) | | `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | @@ -1250,7 +1251,7 @@ The `response_format` parameter supports both plain JSON output (e.g. `{"type": `chat_template_kwargs`: Allows sending additional parameters to the json templating system. For example: `{"enable_thinking": false}` -`reasoning_effort`: If set to `none`, reasoning will be disabled for this request. Other values (e.g., `low`, `max`) have no effect on reasoning. +`reasoning_effort`: If `none`, reasoning/thinking is disabled. Otherwise, the value is made available to the jinja template. `reasoning_format`: The reasoning format to be parsed. If set to `none`, it will output the raw generated text. diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 5ff7685bb..7ed486528 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1292,12 +1292,15 @@ json oaicompat_chat_params_parse( throw std::invalid_argument("invalid type for \"enable_thinking\" (expected boolean, got string)"); } - // Parse also the OAI "reasoning_effort": "none" specific value + // Parse the OAI "reasoning_effort" field; "none" disables reasoning. if (body.contains("reasoning_effort")) { auto reasoning_effort = json_value(body, "reasoning_effort", std::string("")); if (reasoning_effort == "none") { inputs.enable_thinking = false; - } // other reasoning_effort values are model-specific and not yet handled + inputs.chat_template_kwargs.erase("reasoning_effort"); + } else if (!reasoning_effort.empty()) { + inputs.chat_template_kwargs["reasoning_effort"] = json(reasoning_effort).dump(); + } } inputs.force_pure_content = opt.force_pure_content;