From 6fb0b8d30cc893809d930dbf56fdfe01ad8d045f Mon Sep 17 00:00:00 2001
From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com>
Date: Wed, 10 Apr 2024 13:56:35 +0200
Subject: [PATCH] fix: Catch AssertionError in countTokens() (#445)

---
 src/main/java/ee/carlrobert/codegpt/EncodingManager.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/ee/carlrobert/codegpt/EncodingManager.java b/src/main/java/ee/carlrobert/codegpt/EncodingManager.java
index ab3b7d80..44aef5fb 100644
--- a/src/main/java/ee/carlrobert/codegpt/EncodingManager.java
+++ b/src/main/java/ee/carlrobert/codegpt/EncodingManager.java
@@ -59,9 +59,10 @@ public final class EncodingManager {
 
   public int countTokens(String text) {
     try {
+      // #444: Cl100kParser.split() throws AssertionError "Input is not UTF-8: "
       return encoding.countTokens(text);
-    } catch (Exception ex) {
-      LOG.warn(ex);
+    } catch (Exception | Error ex) {
+      LOG.warn("Could not count tokens for: " + text, ex);
       return 0;
     }
   }