vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-15 03:19:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

wstring convert for mpt

Browse source
This commit is contained in:
Concedo 2023-06-24 11:43:42 +08:00
parent 6d718525c4
commit 0485fa65a2
5 changed files with 30 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

6
otherarch/utils.h
View file

@ -34,6 +34,12 @@ void utreplace(std::string & str, const std::string & needle, const std::string
// poor-man's JSON parsing
std::map<std::string, int32_t> json_parse(const std::string & fname);
std::string convert_to_utf8(const std::wstring & input);
std::wstring convert_to_wstring(const std::string & input);
void gpt_split_words(std::string str, std::vector<std::string>& words);
// split text into tokens
//
// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53