vrr/agent-zero
2
0
Fork 0
mirror of https://github.com/agent0ai/agent-zero.git synced 2026-05-04 22:50:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

projects preps

Browse source
This commit is contained in:
frdel 2025-10-23 13:08:27 +02:00
parent 2611de3212
commit c4bf352d9c
9 changed files with 236 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

2
python/helpers/tokens.py
View file

@ -13,7 +13,7 @@ def count_tokens(text: str, encoding_name="cl100k_base") -> int:
encoding = tiktoken.get_encoding(encoding_name)
# Encode the text and count the tokens
tokens = encoding.encode(text)
tokens = encoding.encode(text, disallowed_special=())
token_count = len(tokens)
return token_count