Merge branch 'upstream' into concedo_experimental

This commit is contained in:
Concedo 2025-04-08 21:46:30 +08:00
commit ebf924c5d1
25 changed files with 672 additions and 73 deletions

View file

@ -1851,7 +1851,8 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
tokenizer_pre == "megrez") {
pre_type = LLAMA_VOCAB_PRE_TYPE_QWEN2;
} else if (
tokenizer_pre == "gpt-4o") {
tokenizer_pre == "gpt-4o" ||
tokenizer_pre == "llama4") {
pre_type = LLAMA_VOCAB_PRE_TYPE_GPT4O;
clean_spaces = false;
} else if (