vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-05-20 09:25:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

detokenize add special token ids

Browse source
This commit is contained in:
Concedo 2026-05-17 17:04:47 +08:00
parent c1514e328b
commit de2c5f1cef
3 changed files with 20 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

4
expose.cpp
View file

@ -362,14 +362,14 @@ extern "C"
}
static std::string detokenized_str = ""; //just share a static object for detokenizing
const char * detokenize(const token_count_outputs input)
const char * detokenize(const detokenize_inputs input)
{
std::vector<int> input_arr;
for(int i=0;i<input.count;++i)
{
input_arr.push_back(input.ids[i]);
}
detokenized_str = gpttype_detokenize(input_arr,false);
detokenized_str = gpttype_detokenize(input_arr,input.special);
return detokenized_str.c_str();
}