diff --git a/Makefile b/Makefile index 051ea092e..bd9646830 100644 --- a/Makefile +++ b/Makefile @@ -754,7 +754,7 @@ music_default.o: otherarch/acestep/music_adapter.cpp $(CXX) $(CXXFLAGS) -c $< -o $@ # idiotic "for easier compilation" -GPTTYPE_ADAPTER = gpttype_adapter.cpp otherarch/llama_v2.cpp otherarch/llama_v3.cpp src/llama.cpp src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-iswa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model.cpp src/llama-quant.cpp src/llama-hparams.cpp otherarch/gptj_v1.cpp otherarch/gptj_v2.cpp otherarch/gptj_v3.cpp otherarch/gpt2_v1.cpp otherarch/gpt2_v2.cpp otherarch/gpt2_v3.cpp otherarch/rwkv_v2.cpp otherarch/rwkv_v3.cpp otherarch/neox_v2.cpp otherarch/neox_v3.cpp otherarch/mpt_v3.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h +GPTTYPE_ADAPTER = gpttype_adapter.cpp common/chat.cpp otherarch/llama_v2.cpp otherarch/llama_v3.cpp src/llama.cpp src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-iswa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model.cpp src/llama-quant.cpp src/llama-hparams.cpp otherarch/gptj_v1.cpp otherarch/gptj_v2.cpp otherarch/gptj_v3.cpp otherarch/gpt2_v1.cpp otherarch/gpt2_v2.cpp otherarch/gpt2_v3.cpp otherarch/rwkv_v2.cpp otherarch/rwkv_v3.cpp otherarch/neox_v2.cpp otherarch/neox_v3.cpp otherarch/mpt_v3.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h gpttype_adapter_failsafe.o: $(GPTTYPE_ADAPTER) $(CXX) $(CXXFLAGS) $(FAILSAFE_FLAGS) -c $< -o $@ gpttype_adapter.o: $(GPTTYPE_ADAPTER) diff --git a/expose.cpp b/expose.cpp index 4ca2fea72..969027840 100644 --- a/expose.cpp +++ b/expose.cpp @@ -347,6 +347,25 @@ extern "C" return chat_template.c_str(); } + static std::string parsed_tool_calls = ""; + const char* parse_chat_tool_calls(const char * generated_text, + const char * tools_json, + const char * chat_template, + const char * chat_template_kwargs_json, + const char * tool_choice, + bool parallel_tool_calls, + bool is_partial) { + parsed_tool_calls = gpttype_parse_chat_tool_calls( + generated_text ? generated_text : "", + tools_json ? tools_json : "", + chat_template ? chat_template : "", + chat_template_kwargs_json ? chat_template_kwargs_json : "", + tool_choice ? tool_choice : "", + parallel_tool_calls, + is_partial); + return parsed_tool_calls.c_str(); + } + const char* get_pending_output() { return gpttype_get_pending_output().c_str(); } diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index db8c5d3bf..e8d283eca 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -36,6 +36,7 @@ #include "llama_v2.cpp" #include "llama_v3.cpp" #include "src/llama.cpp" +#include "common/chat.cpp" #include "gptj_v1.cpp" #include "gptj_v2.cpp" #include "gptj_v3.cpp" @@ -4725,6 +4726,95 @@ std::string gpttype_get_chat_template() return std::string(model_template.data(), model_template.size() - 1); } +std::string gpttype_parse_chat_tool_calls(const std::string & generated_text, + const std::string & tools_json, + const std::string & chat_template, + const std::string & chat_template_kwargs_json, + const std::string & tool_choice, + bool parallel_tool_calls, + bool is_partial) +{ + try + { + if(generated_text.empty() || tools_json.empty()) + { + return ""; + } + + json tools = json::parse(tools_json); + if(!tools.is_array() || tools.empty()) + { + return ""; + } + + std::string tmpl_src = chat_template.empty() ? gpttype_get_chat_template() : chat_template; + if(tmpl_src.empty()) + { + return ""; + } + + auto tmpls = common_chat_templates_init(nullptr, tmpl_src); + + common_chat_msg msg; + msg.role = "user"; + msg.content = "tool parser warmup"; + + common_chat_templates_inputs inputs; + inputs.use_jinja = true; + inputs.messages = { msg }; + inputs.tools = common_chat_tools_parse_oaicompat(tools); + inputs.tool_choice = common_chat_tool_choice_parse_oaicompat(tool_choice.empty() ? "auto" : tool_choice); + inputs.parallel_tool_calls = parallel_tool_calls; + inputs.add_generation_prompt = true; + + if(!chat_template_kwargs_json.empty()) + { + json kwargs = json::parse(chat_template_kwargs_json); + if(kwargs.is_object()) + { + for(const auto & item : kwargs.items()) + { + inputs.chat_template_kwargs[item.key()] = item.value().dump(); + } + } + } + + common_chat_params chat_params = common_chat_templates_apply(tmpls.get(), inputs); + if(chat_params.parser.empty()) + { + return ""; + } + + common_chat_parser_params parser_params(chat_params); + parser_params.parse_tool_calls = true; + parser_params.parser.load(chat_params.parser); + + common_chat_msg parsed = common_chat_parse(generated_text, is_partial, parser_params); + if(parsed.tool_calls.empty()) + { + return ""; + } + + json tool_calls = parsed.to_json_oaicompat().value("tool_calls", json::array()); + return tool_calls.dump(); + } + catch(const std::exception & e) + { + if(debugmode == 1 && !is_quiet) + { + printf("\nNative tool parser failed: %s\n", e.what()); + } + } + catch(...) + { + if(debugmode == 1 && !is_quiet) + { + printf("\nNative tool parser failed with unknown error.\n"); + } + } + return ""; +} + std::vector gpttype_get_token_arr(const std::string & input, bool addbos) { std::vector toks; diff --git a/koboldcpp.py b/koboldcpp.py index 5c0f16123..05b7d59fd 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -968,6 +968,9 @@ def init_library(): handle.token_count.restype = token_count_outputs handle.get_pending_output.restype = ctypes.c_char_p handle.get_chat_template.restype = ctypes.c_char_p + handle.parse_chat_tool_calls.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_bool, ctypes.c_bool] + handle.parse_chat_tool_calls.restype = ctypes.c_char_p + handle.calc_new_state_kv.restype = ctypes.c_size_t handle.calc_new_state_tokencount.restype = ctypes.c_size_t handle.calc_old_state_kv.argtypes = [ctypes.c_int] @@ -3667,6 +3670,47 @@ def toolcall_to_normalized_json(text,start_tag,end_tag,required_match_txt): #con return text #fallback +def native_parse_toolcall_tags(text: str, genparams: dict) -> list: + global cached_chat_template, cached_jinja_kwargs + if not text or not genparams.get('using_openai_tools', False): + return [] + tools = genparams.get('tools', []) + if not tools: + return [] + try: + kwargs = dict(cached_jinja_kwargs or {}) + user_kwargs = genparams.get("chat_template_kwargs") + if isinstance(user_kwargs, dict): + kwargs.update(user_kwargs) + if "reasoning_effort" in genparams and genparams["reasoning_effort"] is not None: + kwargs["reasoning_effort"] = genparams["reasoning_effort"] + + tool_choice = genparams.get("tool_choice", "auto") + if isinstance(tool_choice, dict): + tool_choice = "required" + elif tool_choice is None: + tool_choice = "auto" + else: + tool_choice = str(tool_choice) + + raw = handle.parse_chat_tool_calls( + text.encode("UTF-8"), + json.dumps(tools).encode("UTF-8"), + (cached_chat_template or "").encode("UTF-8"), + json.dumps(kwargs).encode("UTF-8"), + tool_choice.encode("UTF-8"), + True, + False) + if not raw: + return [] + parsed = json.loads(ctypes.string_at(raw).decode("UTF-8", "ignore")) + if not isinstance(parsed, list): + return [] + parsed = [normalize_tool_call_resp(obj) for obj in parsed if isinstance(obj, dict)] + return coerce_tool_argtypes(parsed, tools) + except Exception: + return [] + def repack_toolcall_tags(text: str, original_tools:list): global thinkformats, tool_call_pairs tool_calls = [] @@ -5168,8 +5212,11 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler): if api_format == 4 or api_format == 2 or api_format == 8 or api_format == 9: using_openai_tools = genparams.get('using_openai_tools', False) if using_openai_tools: - # first, check and potentially segment multiple tags for multi-tool calls - tool_calls = repack_toolcall_tags(recvtxt,genparams.get('tools', [])) + # first, let llama.cpp's chat parser handle known template-specific tool formats + tool_calls = native_parse_toolcall_tags(recvtxt, genparams) + # fallback: check and potentially segment multiple tags for multi-tool calls + if not tool_calls: + tool_calls = repack_toolcall_tags(recvtxt,genparams.get('tools', [])) if tool_calls and len(tool_calls)>0: flat = [] for obj in tool_calls: diff --git a/model_adapter.h b/model_adapter.h index 467fa7bde..baca4a3f1 100644 --- a/model_adapter.h +++ b/model_adapter.h @@ -84,6 +84,13 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in generation_outputs gpttype_generate(const generation_inputs inputs); bool gpttype_generate_abort(); std::string gpttype_get_chat_template(); +std::string gpttype_parse_chat_tool_calls(const std::string & generated_text, + const std::string & tools_json, + const std::string & chat_template, + const std::string & chat_template_kwargs_json, + const std::string & tool_choice, + bool parallel_tool_calls, + bool is_partial); bool gpttype_batch_generate_enabled(); int gpttype_batch_generate_submit(const generation_inputs inputs); bool gpttype_batch_generate_has_finished(int request_id);