diff --git a/otherarch/acestep/bpe.h b/otherarch/acestep/bpe.h index fe5ebc6d1..6da28c0f8 100644 --- a/otherarch/acestep/bpe.h +++ b/otherarch/acestep/bpe.h @@ -263,6 +263,8 @@ static std::vector gpt2_pre_tokenize(const std::string &text) { return chunks; } +namespace kcpp_workaround_multiple_BPETokenizer { + // BPE tokenizer struct struct BPETokenizer { std::unordered_map vocab; // token_str -> id @@ -273,6 +275,9 @@ struct BPETokenizer { std::vector id_to_str; // id -> token_str (reverse vocab) }; +} +using kcpp_workaround_multiple_BPETokenizer::BPETokenizer; + // Minimal JSON parser for vocab.json ({"str": int, ...}) static bool load_vocab_json(const std::string &path, std::unordered_map &vocab) { FILE *f = fopen(path.c_str(), "rb");