From e3a247f99edab070cb3bfd668b402b4fea47e8cf Mon Sep 17 00:00:00 2001
From: Dex <i@dex.moe>
Date: Thu, 9 Jul 2026 00:39:02 +0100
Subject: [PATCH] perf(core): add pure-ASCII fast path to text token estimation
 (#6551)

estimateTextTokens scanned every string char-by-char via charCodeAt to
classify ASCII vs non-ASCII code units. For pure-ASCII text (code,
English prose - the common case) a single regex scan using V8's
optimized string search replaces the JS loop, and the mixed-text path
now counts only non-ASCII units, deriving the ASCII count from the
length. The token formula is unchanged, so results are byte-identical
for every input; verified exhaustively over all 65536 single code units
plus 20k randomized mixed strings against the previous implementation.

Median of 6 solo benchmark runs over a deterministic mixed fixture set:
51.9ms -> 32.2ms (-38%, 1.61x).
---
 .../request-tokenizer/textTokenizer.test.ts   | 31 +++++++++++++++++++
 .../utils/request-tokenizer/textTokenizer.ts  | 16 ++++++----
 2 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/packages/core/src/utils/request-tokenizer/textTokenizer.test.ts b/packages/core/src/utils/request-tokenizer/textTokenizer.test.ts
index 003e97941f..a0f24a7005 100644
--- a/packages/core/src/utils/request-tokenizer/textTokenizer.test.ts
+++ b/packages/core/src/utils/request-tokenizer/textTokenizer.test.ts
@@ -262,6 +262,37 @@ describe('TextTokenizer', () => {
     });
   });
 
+  describe('ASCII/non-ASCII boundary', () => {
+    it('should treat DEL (U+007F) as ASCII and U+0080 as non-ASCII', async () => {
+      // '\x7F' = 1 ASCII char: 1 / 4 = 0.25 -> ceil = 1
+      expect(await tokenizer.calculateTokens('\x7F')).toBe(1);
+      // '\u0080' = 1 non-ASCII char: 1 * 1.1 = 1.1 -> ceil = 2
+      expect(await tokenizer.calculateTokens('\u0080')).toBe(2);
+    });
+
+    it('should count pure-ASCII text of any length as ceil(length / 4)', async () => {
+      for (const len of [1, 3, 4, 5, 4096, 4097]) {
+        const text = 'a'.repeat(len);
+        expect(await tokenizer.calculateTokens(text)).toBe(Math.ceil(len / 4));
+      }
+    });
+
+    it('should stay consistent when a single non-ASCII char joins long ASCII text', async () => {
+      const ascii = 'x'.repeat(1000);
+      // 1000 / 4 = 250
+      expect(await tokenizer.calculateTokens(ascii)).toBe(250);
+      // 1000 / 4 + 1 * 1.1 = 251.1 -> ceil = 252, wherever the char sits
+      expect(await tokenizer.calculateTokens(ascii + '中')).toBe(252);
+      expect(await tokenizer.calculateTokens('中' + ascii)).toBe(252);
+    });
+
+    it('should count surrogate pairs as two non-ASCII units within mixed text', async () => {
+      const text = 'abcd🚀'; // 4 ASCII + 2 UTF-16 units
+      // 4 / 4 + 2 * 1.1 = 3.2 -> ceil = 4
+      expect(await tokenizer.calculateTokens(text)).toBe(4);
+    });
+  });
+
   describe('large inputs', () => {
     it('should handle very long text', async () => {
       const longText = 'a'.repeat(200000); // 200k characters
diff --git a/packages/core/src/utils/request-tokenizer/textTokenizer.ts b/packages/core/src/utils/request-tokenizer/textTokenizer.ts
index 7b5be98519..dac298b6b3 100644
--- a/packages/core/src/utils/request-tokenizer/textTokenizer.ts
+++ b/packages/core/src/utils/request-tokenizer/textTokenizer.ts
@@ -4,6 +4,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+const NON_ASCII_RE = /[\u0080-\uffff]/;
+
 /**
  * Text tokenizer for calculating text tokens using character-based estimation.
  *
@@ -19,17 +21,19 @@ export function estimateTextTokens(text: string): number {
     return 0;
   }
 
-  let asciiChars = 0;
-  let nonAsciiChars = 0;
+  // Fast path: pure-ASCII text (code, English prose). A single regex scan
+  // uses V8's optimized string search instead of a per-character JS loop.
+  if (!NON_ASCII_RE.test(text)) {
+    return Math.ceil(text.length / 4);
+  }
 
+  let nonAsciiChars = 0;
   for (let i = 0; i < text.length; i++) {
-    const charCode = text.charCodeAt(i);
-    if (charCode < 128) {
-      asciiChars++;
-    } else {
+    if (text.charCodeAt(i) >= 128) {
       nonAsciiChars++;
     }
   }
+  const asciiChars = text.length - nonAsciiChars;
 
   const tokens = asciiChars / 4 + nonAsciiChars * 1.1;
   return Math.ceil(tokens);