mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-05-10 04:00:53 +00:00
f282362414
Squashed commits: [92df28061] added qwen image support (+1 squashed commits) Squashed commits: [1485c71ed] wip adding qwen image
10 lines
No EOL
307 B
C++
10 lines
No EOL
307 B
C++
#ifndef __TOKENIZE_UTIL__
|
|
#define __TOKENIZE_UTIL__
|
|
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
std::vector<std::string> token_split(const std::string& text);
|
|
std::vector<std::string> split_with_special_tokens(const std::string& text, const std::vector<std::string>& special_tokens);
|
|
|
|
#endif // __TOKENIZE_UTIL__
|