vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 09:34:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

workaround for deepseek not working

Browse source
This commit is contained in:
Concedo 2024-07-05 22:39:53 +08:00
parent 6b0756506b
commit 388a2aff00
3 changed files with 17 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

12
src/unicode.cpp
View file

@ -189,9 +189,19 @@ static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {
return map;
}
static bool unicode_wstring_from_utf8_failed_once = false;
static inline std::wstring unicode_wstring_from_utf8(const std::string & s) {
std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
return conv.from_bytes(s);
try {
return conv.from_bytes(s);
} catch(const std::exception & e) {
if(!unicode_wstring_from_utf8_failed_once)
{
unicode_wstring_from_utf8_failed_once = true;
printf("\nunicode_wstring_from_utf8 failed: %s\n", e.what());
}
return L"";
}
}
static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string> & bpe_words) {