vrr/prima.cpp
2
0
Fork 0
mirror of https://github.com/Lizonghang/prima.cpp.git synced 2025-09-10 18:34:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

unicode : avoid char32_t (#7957)

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2024-06-16 14:51:40 +03:00 • committed by GitHub
parent 6fe1c62741
commit 52399254b3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 13 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

2
llama.cpp
View file

@ -13246,7 +13246,7 @@ struct llm_tokenizer_wpm {
const std::vector<uint32_t> cpts_nfd = unicode_cpts_normalize_nfd(unicode_cpts_from_utf8(text));
std::vector<std::string> words(1, "");
for (const char32_t cpt : cpts_nfd) {
for (const uint32_t cpt : cpts_nfd) {
const auto flags = unicode_cpt_flags(cpt);
if (flags.is_whitespace) {