diff --git a/internal/llm/embed/cl100k_base.tiktoken b/internal/llm/bpe_data/cl100k_base.tiktoken similarity index 100% rename from internal/llm/embed/cl100k_base.tiktoken rename to internal/llm/bpe_data/cl100k_base.tiktoken diff --git a/internal/llm/embedded_loader.go b/internal/llm/embedded_loader.go index 83687bc..898ffbc 100644 --- a/internal/llm/embedded_loader.go +++ b/internal/llm/embedded_loader.go @@ -10,10 +10,10 @@ import ( tiktoken "github.com/pkoukk/tiktoken-go" ) -//go:embed embed/*.tiktoken +//go:embed bpe_data/*.tiktoken var embedFS embed.FS -const embedPrefix = "embed/" +const embedPrefix = "bpe_data/" // initEmbeddedLoader configures tiktoken to use embedded BPE data instead of fetching from network. // Call this once during application startup, before any GetEncoding/EncodingForModel calls.