chore: remove tiktoken dependency and replace with character-based token estimation

This commit is contained in:
tanzhenxin 2026-01-17 22:39:18 +08:00
parent ff5ea3c6d7
commit 35c865968f
22 changed files with 329 additions and 595 deletions

View file

@ -38,14 +38,15 @@
"dependencies": {
"@google/genai": "1.30.0",
"@iarna/toml": "^2.2.5",
"@qwen-code/qwen-code-core": "file:../core",
"@modelcontextprotocol/sdk": "^1.25.1",
"@qwen-code/qwen-code-core": "file:../core",
"@types/update-notifier": "^6.0.8",
"ansi-regex": "^6.2.2",
"command-exists": "^1.2.9",
"comment-json": "^4.2.5",
"diff": "^7.0.0",
"dotenv": "^17.1.0",
"extract-zip": "^2.0.1",
"fzf": "^0.5.2",
"glob": "^10.5.0",
"highlight.js": "^11.11.1",
@ -65,7 +66,6 @@
"strip-json-comments": "^3.1.1",
"tar": "^7.5.2",
"undici": "^6.22.0",
"extract-zip": "^2.0.1",
"update-notifier": "^7.3.1",
"wrap-ansi": "9.0.2",
"yargs": "^17.7.2",
@ -74,6 +74,7 @@
"devDependencies": {
"@babel/runtime": "^7.27.6",
"@google/gemini-cli-test-utils": "file:../test-utils",
"@qwen-code/qwen-code-test-utils": "file:../test-utils",
"@testing-library/react": "^16.3.0",
"@types/archiver": "^6.0.3",
"@types/command-exists": "^1.2.3",
@ -92,8 +93,7 @@
"pretty-format": "^30.0.2",
"react-dom": "^19.1.0",
"typescript": "^5.3.3",
"vitest": "^3.1.1",
"@qwen-code/qwen-code-test-utils": "file:../test-utils"
"vitest": "^3.1.1"
},
"engines": {
"node": ">=20"