mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-05-19 16:31:59 +00:00
bad9b61064
* sd: remove sampler alias handling from the C++ layer It's already handled at the Python layer. * sd: sync to master-580-7d33d4b * sd: sync to master-582-7023fc4
10 lines
No EOL
367 B
C++
10 lines
No EOL
367 B
C++
#ifndef __SD_TOKENIZERS_BPE_TOKENIZE_UTIL_H__
|
|
#define __SD_TOKENIZERS_BPE_TOKENIZE_UTIL_H__
|
|
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
std::vector<std::string> token_split(const std::string& text);
|
|
std::vector<std::string> split_with_special_tokens(const std::string& text, const std::vector<std::string>& special_tokens);
|
|
|
|
#endif // __SD_TOKENIZERS_BPE_TOKENIZE_UTIL_H__
|