vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 09:04:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix for mamba processing

Browse source
This commit is contained in:
Concedo 2024-03-13 13:27:46 +08:00
parent ba950716a9
commit 47c42fd45c
4 changed files with 24 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
colab.ipynb
View file

@ -54,7 +54,7 @@
"ForceRebuild = False #@param {type:\"boolean\"}\r\n",
"#@markdown <hr>\r\n",
"LoadLLaVAmmproj = False #@param {type:\"boolean\"}\r\n",
"LLaVAmmproj = \"https://huggingface.co/concedo/koboldcpp-mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\" #@param [\"https://huggingface.co/concedo/koboldcpp-mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\",\"https://huggingface.co/concedo/koboldcpp-mmproj/resolve/main/mistral-7b-mmproj-v1.5-Q4_1.gguf\",\"https://huggingface.co/concedo/koboldcpp-mmproj/resolve/main/llama-7b-mmproj-v1.5-Q4_0.gguf\"]{allow-input: true}\r\n",
"LLaVAmmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\" #@param [\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/mistral-7b-mmproj-v1.5-Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-7b-mmproj-v1.5-Q4_0.gguf\"]{allow-input: true}\r\n",
"VCommand = \"\"\r\n",
"#@markdown <hr>\r\n",
"LoadImgModel = False #@param {type:\"boolean\"}\r\n",

20
gpttype_adapter.cpp
View file

@ -58,6 +58,7 @@ static std::string current_grammar = "";
//return val: 0=fail, 1=(original ggml, alpaca), 2=(ggmf), 3=(ggjt)
static FileFormat file_format = FileFormat::BADFORMAT;
static FileFormatExtraMeta file_format_meta;
static gpt_vocab vocab;
static int32_t n_vocab = 0;
@ -736,12 +737,13 @@ static int GetBatchSize(int desiredBlasBatchSize,FileFormat in_file_format)
return desiredBlasBatchSize;
}
ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format, FileFormatExtraMeta file_format_meta)
ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format, FileFormatExtraMeta in_file_format_meta)
{
ggml_time_init();
kcpp_params = new gpt_params(); //allocate on heap to avoid linux segfault. yes this leaks memory.
file_format = in_file_format;
file_format_meta = in_file_format_meta;
kcpp_params->n_threads = inputs.threads;
kcpp_params->n_threads_batch = inputs.blasthreads;
bool isGguf = (file_format == FileFormat::GGUF_GENERIC);
@ -1828,9 +1830,23 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
std::fill(last_n_tokens.begin(), last_n_tokens.end(), 0);
n_past = 0;
if (file_format == FileFormat::RWKV_1 || file_format==FileFormat::RWKV_2)
bool is_mamba = (file_format == FileFormat::GGUF_GENERIC && file_format_meta.model_architecture==GGUFArch::ARCH_MAMBA);
if (file_format == FileFormat::RWKV_1 || file_format==FileFormat::RWKV_2 || is_mamba)
{
ContextFastForward(current_context_tokens, embd_inp, n_past, last_n_tokens, nctx, smartcontext, false, true);
if(is_mamba)
{
if(n_past==0)
{
llama_kv_cache_clear(llama_ctx_v4);
}
else if(embd_inp.size()==0)
{
embd_inp.push_back(current_context_tokens[current_context_tokens.size()-1]);
n_past -= 1;
}
}
}
else
{

4
model_adapter.cpp
View file

@ -293,6 +293,10 @@ void print_tok_vec(std::vector<float> &embd)
{
fileformatmeta->model_architecture = GGUFArch::ARCH_FALCON;
}
else if(modelarch=="mamba")
{
fileformatmeta->model_architecture = GGUFArch::ARCH_MAMBA;
}
}
gguf_free(ctx);
}

1
model_adapter.h
View file

@ -55,6 +55,7 @@ enum GGUFArch
ARCH_DEFAULT = 0, //used for llama and other generic gguf
ARCH_FALCON = 1,
ARCH_PHI = 2,
ARCH_MAMBA = 3,
};
struct FileFormatExtraMeta