vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 01:24:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

convert : fix broken sentencepiece vocab (#14416)

Browse source
This commit is contained in:
Sigbjørn Skjæret 2025-06-27 10:42:19 +02:00 • committed by GitHub
parent 8846aace49
commit f667f1e624
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 5 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

10
convert_hf_to_gguf.py
View file

@ -936,7 +936,11 @@ class TextModel(ModelBase):
scores: list[float] = [-10000.0] * vocab_size scores: list[float] = [-10000.0] * vocab_size
toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size
for token_id in range(vocab_size): for token_id in range(tokenizer.vocab_size()):
if token_id >= vocab_size:
logger.warning(f'ignore tokens from {token_id}: id is out of range, max={vocab_size - 1}')
break
piece = tokenizer.IdToPiece(token_id) piece = tokenizer.IdToPiece(token_id)
text = piece.encode("utf-8") text = piece.encode("utf-8")
score = tokenizer.GetScore(token_id) score = tokenizer.GetScore(token_id)
@ -951,10 +955,6 @@ class TextModel(ModelBase):
elif tokenizer.IsByte(token_id): elif tokenizer.IsByte(token_id):
toktype = SentencePieceTokenTypes.BYTE toktype = SentencePieceTokenTypes.BYTE
if token_id >= vocab_size:
logger.warning(f'ignore tokens from {token_id}: id is out of range, max={vocab_size - 1}')
break
tokens[token_id] = text tokens[token_id] = text
scores[token_id] = score scores[token_id] = score
toktypes[token_id] = toktype toktypes[token_id] = toktype