vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-19 05:15:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

common: support --models-dir loading MTP assistant models (#24431)

Browse source 
* common: support --models-dir loading MTP assistant models

* common: preset: check for MTP models with strict prefix

* common: preset: Take advantage of PR #27005

* handle other draft types

* drop eagle3

* clean up

---------

Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
Eric Zhang 2026-08-15 19:17:35 +08:00 • committed by GitHub
parent 27df9199d1
commit 5f754ea0e2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 33 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

36
common/preset.cpp
View file

@ -365,8 +365,25 @@ struct local_model {
std::string name;
std::string path;
std::string path_mmproj;
std::string path_draft;
};
// TODO @ngxson: handle "eagle3-" when it's supported by common_speculative_types_from_gguf()
static const char * draft_prefixes[] = { "mtp-", "dspark-", "dflash-" };
static bool is_mmproj_file(const std::string & fname) {
return fname.find("mmproj") != std::string::npos;
}
static bool is_draft_file(const std::string & fname) {
for (const auto & prefix : draft_prefixes) {
if (fname.rfind(prefix, 0) == 0) {
return true;
}
}
return false;
}
common_presets common_preset_context::load_from_models_dir(const std::string & models_dir) const {
if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {
throw std::runtime_error(string_format("error: '%s' does not exist or is not a directory\n", models_dir.c_str()));
@ -378,10 +395,15 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
common_file_info model_file;
common_file_info first_shard_file;
common_file_info mmproj_file;
common_file_info draft_file;
for (const auto & file : files) {
if (string_ends_with(file.name, ".gguf")) {
if (file.name.find("mmproj") != std::string::npos) {
if (is_mmproj_file(file.name)) {
mmproj_file = file;
} else if (is_draft_file(file.name)) {
if (draft_file.path.empty()) {
draft_file = file; // first sidecar found wins
}
} else if (file.name.find("-00001-of-") != std::string::npos) {
first_shard_file = file;
} else {
@ -393,7 +415,8 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
local_model model{
/* name */ name,
/* path */ first_shard_file.path.empty() ? model_file.path : first_shard_file.path,
/* path_mmproj */ mmproj_file.path // can be empty
/* path_mmproj */ mmproj_file.path, // can be empty
/* path_draft */ draft_file.path // can be empty
};
if (!model.path.empty()) {
models.push_back(model);
@ -405,13 +428,17 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
if (file.is_dir) {
scan_subdir(file.path, file.name);
} else if (string_ends_with(file.name, ".gguf")) {
if (is_mmproj_file(file.name) || is_draft_file(file.name)) {
continue; // companion file, cannot be loaded as a model on its own
}
// single file model
std::string name = file.name;
string_replace_all(name, ".gguf", "");
local_model model{
/* name */ name,
/* path */ file.path,
/* path_mmproj */ ""
/* path_mmproj */ "",
/* path_draft */ ""
};
models.push_back(model);
}
@ -426,6 +453,9 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
if (!model.path_mmproj.empty()) {
preset.set_option(*this, "LLAMA_ARG_MMPROJ", model.path_mmproj);
}
if (!model.path_draft.empty()) {
preset.set_option(*this, "LLAMA_ARG_SPEC_DRAFT_MODEL", model.path_draft);
}
out[preset.name] = preset;
}