vrr/ProxyAI
2
0
Fork 0
mirror of https://github.com/carlrobertoh/ProxyAI.git synced 2026-05-12 14:10:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Revert "feat: code completion improvements"

Browse source 
This reverts commit abc8dc8d07.
This commit is contained in:
Carl-Robert Linnupuu 2024-02-05 16:28:18 +02:00
parent abc8dc8d07
commit 7f586da0c1
8 changed files with 177 additions and 109 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/main/java/ee/carlrobert/codegpt/EncodingManager.java
View file

@ -9,7 +9,6 @@ import com.knuddels.jtokkit.api.EncodingRegistry;
import com.knuddels.jtokkit.api.EncodingType;
import ee.carlrobert.codegpt.conversations.Conversation;
import ee.carlrobert.llm.client.openai.completion.request.OpenAIChatCompletionMessage;
import java.util.List;
@Service
public final class EncodingManager {
@ -53,19 +52,4 @@ public final class EncodingManager {
return 0;
}
}
/**
* Truncates the given text to the given number of tokens.
*
* @param text The text to truncate.
* @param maxTokens The maximum number of tokens to keep.
* @param fromStart Whether to truncate from the start or the end of the text.
* @return The truncated text.
*/
public String truncateText(String text, int maxTokens, boolean fromStart) {
List<Integer> tokens = encoding.encode(text);
int tokensToRetrieve = Math.min(maxTokens, tokens.size());
int startIndex = fromStart ? 0 : tokens.size() - tokensToRetrieve;
return encoding.decode(tokens.subList(startIndex, startIndex + tokensToRetrieve));
}
}