diff --git a/src/main/java/ee/carlrobert/codegpt/EncodingManager.java b/src/main/java/ee/carlrobert/codegpt/EncodingManager.java index ab3b7d80..44aef5fb 100644 --- a/src/main/java/ee/carlrobert/codegpt/EncodingManager.java +++ b/src/main/java/ee/carlrobert/codegpt/EncodingManager.java @@ -59,9 +59,10 @@ public final class EncodingManager { public int countTokens(String text) { try { + // #444: Cl100kParser.split() throws AssertionError "Input is not UTF-8: " return encoding.countTokens(text); - } catch (Exception ex) { - LOG.warn(ex); + } catch (Exception | Error ex) { + LOG.warn("Could not count tokens for: " + text, ex); return 0; } }