vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 09:04:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adapter fixes (#1659)

Browse source 
* test adapters

* add assistant_gen adapter key

* add support for chat templates stored as .jinja files

* removed mistakenly commited gated-tokenizers link

* autoguess: Harmony: add missing newline prefixes to system_end
This commit is contained in:
kallewoof 2025-08-10 17:19:50 +09:00 • committed by GitHub
parent 57db0ce9cd
commit 204739e7f1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 33 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

5
kcpp_adapters/AutoGuess.json
View file

@ -107,6 +107,7 @@
"user_start": "[INST] ",
"user_end": "",
"assistant_start": "[/INST] ",
"assistant_gen": "[/INST]",
"assistant_end": "</s>"
}
}, {
@ -173,6 +174,7 @@
"user_start": "<|bom|><|user|> ",
"user_end": "<|eom|>",
"assistant_start": "<|bom|><|assistant|> ",
"assistant_gen": "<|bom|><|assistant|>",
"assistant_end": "<|eom|>"
}
}, {
@ -191,7 +193,7 @@
"name": "OpenAI Harmony",
"adapter": {
"system_start": "<|start|>developer<|message|># Instructions\n\n",
"system_end": "<|end|>",
"system_end": "\n\n<|end|>",
"user_start": "<|start|>user<|message|>",
"user_end": "<|end|>",
"assistant_start": "<|start|>assistant<|channel|>final<|message|>",
@ -206,6 +208,7 @@
"user_start": "User: ",
"user_end": "\n\n",
"assistant_start": "Assistant: ",
"assistant_gen": "Assistant:",
"assistant_end": "\n\n"
}
}, {

9
koboldcpp.py
View file

@ -2419,6 +2419,7 @@ ws ::= | " " | "\n" [ \t]{0,20}
user_message_end = adapter_obj.get("user_end", "")
assistant_message_start = adapter_obj.get("assistant_start", "\n### Response:\n")
assistant_message_end = adapter_obj.get("assistant_end", "")
assistant_message_gen = adapter_obj.get("assistant_gen", assistant_message_start)
tools_message_start = adapter_obj.get("tools_start", "\nTool Results:\n")
tools_message_end = adapter_obj.get("tools_end", "")
images_added = []
@ -2531,7 +2532,7 @@ ws ::= | " " | "\n" [ \t]{0,20}
elif message['role'] == "tool":
messages_string += tools_message_end
messages_string += assistant_message_start
messages_string += assistant_message_gen
genparams["prompt"] = messages_string
if len(images_added)>0:
genparams["images"] = images_added
@ -2552,7 +2553,8 @@ ws ::= | " " | "\n" [ \t]{0,20}
adapter_obj = {} if chatcompl_adapter is None else chatcompl_adapter
user_message_start = adapter_obj.get("user_start", "### Instruction:")
assistant_message_start = adapter_obj.get("assistant_start", "### Response:")
genparams["prompt"] = f"{user_message_start} In one sentence, write a descriptive caption for this image.\n{assistant_message_start}"
assistant_message_gen = adapter_obj.get("assistant_gen", assistant_message_start)
genparams["prompt"] = f"{user_message_start} In one sentence, write a descriptive caption for this image.\n{assistant_message_gen}"
elif api_format==6:
detokstr = ""
@ -2560,12 +2562,13 @@ ws ::= | " " | "\n" [ \t]{0,20}
adapter_obj = {} if chatcompl_adapter is None else chatcompl_adapter
user_message_start = adapter_obj.get("user_start", "\n\n### Instruction:\n")
assistant_message_start = adapter_obj.get("assistant_start", "\n\n### Response:\n")
assistant_message_gen = adapter_obj.get("assistant_gen", assistant_message_start)
try:
detokstr = detokenize_ids(tokids)
except Exception as e:
utfprint("Ollama Context Error: " + str(e))
ollamasysprompt = genparams.get('system', "")
ollamabodyprompt = f"{detokstr}{user_message_start}{genparams.get('prompt', '')}{assistant_message_start}"
ollamabodyprompt = f"{detokstr}{user_message_start}{genparams.get('prompt', '')}{assistant_message_gen}"
ollamaopts = genparams.get('options', {})
if genparams.get('stop',[]) is not None:
genparams["stop_sequence"] = genparams.get('stop', [])

22
tests/test_autoguess.py
View file

@ -21,8 +21,8 @@ AUTOGUESS_MAPPING = {
"Google Gemma 3n": "lmstudio-community/gemma-3n-E4B-it-MLX-bf16",
"Llama 3.x": "Steelskull/L3.3-Shakudo-70b",
"Llama 4": "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",
"Mistral V7 (with system prompt)": "Doctor-Shotgun/MS3.2-24B-Magnum-Diamond",
"Mistral V3": "mistralai/Mistral-7B-Instruct-v0.3",
"Mistral Tekken": "Doctor-Shotgun/MS3.2-24B-Magnum-Diamond",
"Mistral Non-Tekken": "mistralai/Mistral-7B-Instruct-v0.3",
"GLM-4": "THUDM/glm-4-9b-chat-hf",
"Phi 3.5": "microsoft/Phi-3.5-mini-instruct",
"Phi 4 (mini)": "microsoft/Phi-4-mini-instruct",
@ -31,12 +31,13 @@ AUTOGUESS_MAPPING = {
"Jamba": "ai21labs/Jamba-tiny-dev",
"Dots": "rednote-hilab/dots.llm1.inst",
"RWKV World": "fla-hub/rwkv7-1.5B-world",
"OpenAI Harmony": "openai/gpt-oss-120b",
"Mistral (Generic)": "mistralai/Mistral-Nemo-Instruct-2407",
"ChatML (Generic)": "NewEden/Gemma-27B-chatml",
}
AUTOGUESS_SKIP_ADAPTER_TESTS = {
"Mistral V3": {"system"}, # Poor system support
"Mistral Non-Tekken": {"system"}, # Poor system support
"Mistral (Generic)": {"system"}, # Poor system support
}
@ -58,10 +59,12 @@ def get_tokenizer_config_for_huggingface_model_id(huggingface_model_id: str):
with open(fname) as f:
return json.load(f)
for filename in ["tokenizer_config.json", "chat_template.json"]:
for filename in ["tokenizer_config.json", "chat_template.json", "chat_template.jinja"]:
url = f"https://huggingface.co/{huggingface_model_id}/resolve/main/{filename}"
response = requests.get(url)
if response.status_code == 200:
if url.endswith(".jinja"):
return {"chat_template": response.text}
v = json.loads(response.text)
if 'chat_template' in v:
return v
@ -113,9 +116,9 @@ def test_tokenizer_with_adapter(tokenizer, adapter: dict[str, str], skip: set) -
expect = system("SyS-tEm")
templated = templ([{"role": "system", "content": "SyS-tEm"}, {"role": "user", "content": "user"}])
if expect not in templated:
return False, f"system role missing expected fragment {expect.replace("\n", "\\n")}: {templated.replace("\n", "\\n")}"
return False, f"system role missing expected fragment\n\tadapter: {expect.replace("\n", "\\n")}\n\ttokenizer: {templated.replace("\n", "\\n")}"
# Test user/asst/usernvidia/Llama-4-Scout-17B-16E-Instruct-FP8
# Test user/asst/user
expect = [
user("user_1"),
assistant("asst_1"),
@ -129,17 +132,21 @@ def test_tokenizer_with_adapter(tokenizer, adapter: dict[str, str], skip: set) -
rem = templated
for sub in expect:
if sub not in rem:
return False, f"missing expected fragment {sub.replace("\n", "\\n")}: {rem.replace("\n", "\\n")}"
return False, f"missing expected fragment\n\tadapter: {sub.replace("\n", "\\n")}\n\ttokenizer: {rem.replace("\n", "\\n")}"
rem = rem.split(sub, 1)[1]
except jinja2.exceptions.TemplateError as e:
return False, f"template error: {e}"
return True, None
filter = sys.argv[1] if len(sys.argv) > 1 else None
failures = 0
seen = set()
namefmt = "{name:<" + str(max(len(name) for name in AUTOGUESS_MAPPING.keys())) + "}"
hmifmt = "{huggingface_model_id:<" + str(max(len(huggingface_model_id) for huggingface_model_id in AUTOGUESS_MAPPING.values())) + "}"
for name, huggingface_model_id in AUTOGUESS_MAPPING.items():
if filter and filter not in name:
continue
seen.add(name)
if huggingface_model_id == "***UNKNOWN***":
print(namefmt.format(name=name) + " = " + namefmt.format(name="***UNKNOWN***") + " : PENDING")
@ -162,6 +169,7 @@ for name, huggingface_model_id in AUTOGUESS_MAPPING.items():
print(namefmt.format(name=name) + " = " + namefmt.format(name=matched) + " : " + ("OK " if adaptercheck and name == matched else reason if not adaptercheck else "FAILURE") + " " + hmifmt.format(huggingface_model_id=huggingface_model_id) + " " + sub_template)
failures += name != matched or not adaptercheck
if filter is None:
for entry in autoguess:
if entry['name'] not in seen:
print(namefmt.format(name=entry['name']) + " MISSING MAPPING")