vrr/ProxyAI
2
0
Fork 0
mirror of https://github.com/carlrobertoh/ProxyAI.git synced 2026-05-13 07:02:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

test: add test coverage for code completions

Browse source
This commit is contained in:
Carl-Robert Linnupuu 2024-09-01 17:21:57 +03:00
parent c83d9eff00
commit a546217155
5 changed files with 163 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/main/java/ee/carlrobert/codegpt/EncodingManager.java
View file

@ -20,6 +20,9 @@ import java.util.stream.Stream;
@Service
public final class EncodingManager {
private static final String SPECIAL_START = "<|";
private static final String SPECIAL_END = "|>";
private static final Logger LOG = Logger.getInstance(EncodingManager.class);
private final EncodingRegistry registry = Encodings.newDefaultEncodingRegistry();
@ -76,7 +79,8 @@ public final class EncodingManager {
* @return The truncated text.
*/
public String truncateText(String text, int maxTokens, boolean fromStart) {
var tokens = encoding.encode(text);
var textWithSpecialEncodingsRemoved = text.replace(SPECIAL_START, "").replace(SPECIAL_END, "");
var tokens = encoding.encode(textWithSpecialEncodingsRemoved);
int tokensToRetrieve = Math.min(maxTokens, tokens.size());
int startIndex = fromStart ? 0 : tokens.size() - tokensToRetrieve;
var truncatedList =