diff --git a/.changeset/fix-utf8-text-binary-detection.md b/.changeset/fix-utf8-text-binary-detection.md new file mode 100644 index 000000000..b7ce5e169 --- /dev/null +++ b/.changeset/fix-utf8-text-binary-detection.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Fix UTF-8 text files containing Chinese or emoji being misdetected as binary, so log files preview correctly in the web UI. diff --git a/packages/agent-core-v2/src/_base/text/encoding.ts b/packages/agent-core-v2/src/_base/text/encoding.ts index 714f05251..32df9743d 100644 --- a/packages/agent-core-v2/src/_base/text/encoding.ts +++ b/packages/agent-core-v2/src/_base/text/encoding.ts @@ -23,6 +23,13 @@ export type UtfTextEncoding = 'utf-8' | 'utf-16le' | 'utf-16be'; +export interface TextClassification { + readonly isBinary: boolean; + readonly encoding: UtfTextEncoding; +} + +export const FS_BINARY_NONPRINTABLE_FRACTION = 0.3; + export interface TextEncodingDetection { /** * Detected encoding. `'utf-8'` when no signal points elsewhere (also the @@ -51,16 +58,7 @@ const UTF16BE_BOM = [0xfe, 0xff] as const; const UTF16LE_BOM = [0xff, 0xfe] as const; const UTF8_BOM = [0xef, 0xbb, 0xbf] as const; -/** - * Detect the encoding of a text file from its leading bytes. - * - * Known limitation inherited from the reference implementation: a BOM-less - * UTF-16 file whose content carries no zero bytes at all (e.g. purely CJK - * text) is reported as `'utf-8'`; strict UTF-8 decoding of it will then fail - * or produce garbage. Notepad and most editors write a BOM, so this is rare - * in practice. - */ -export function detectTextEncoding(sample: Uint8Array): TextEncodingDetection { +function sniffTextEncoding(sample: Uint8Array): TextEncodingDetection { // Always trust a BOM first. if (sample.length >= 2) { const b0 = sample[0]!; @@ -101,6 +99,67 @@ export function detectTextEncoding(sample: Uint8Array): TextEncodingDetection { return { encoding: 'utf-8', seemsBinary: true }; } +export function classifyTextSample(sample: Uint8Array): TextClassification { + const sniffed = sniffTextEncoding(sample); + if (sniffed.seemsBinary || sniffed.encoding !== 'utf-8') { + return { isBinary: sniffed.seemsBinary, encoding: sniffed.encoding }; + } + if (sample.includes(0)) { + return { isBinary: true, encoding: 'utf-8' }; + } + let end = sample.length; + for (let i = Math.max(0, sample.length - 3); i < sample.length; i++) { + const b = sample[i]!; + const expected = + b >= 0xc2 && b <= 0xdf ? 2 : b >= 0xe0 && b <= 0xef ? 3 : b >= 0xf0 && b <= 0xf4 ? 4 : 0; + if (expected === 0 || i + expected <= sample.length) continue; + let validPrefix = true; + for (let j = i + 1; j < sample.length; j++) { + const cb = sample[j]!; + if (cb < 0x80 || cb > 0xbf) { + validPrefix = false; + break; + } + } + if (validPrefix) { + end = i; + break; + } + } + let text: string; + try { + text = new TextDecoder('utf-8', { fatal: true }).decode(sample.subarray(0, end)); + } catch { + return { isBinary: true, encoding: 'utf-8' }; + } + let nonPrintable = 0; + let total = 0; + for (const ch of text) { + const cp = ch.codePointAt(0)!; + total++; + if (cp === 9 || cp === 10 || cp === 13) continue; + if (cp < 32 || (cp >= 0x7f && cp <= 0x9f)) nonPrintable++; + } + if (total > 0 && nonPrintable / total > FS_BINARY_NONPRINTABLE_FRACTION) { + return { isBinary: true, encoding: 'utf-8' }; + } + return { isBinary: false, encoding: 'utf-8' }; +} + +/** + * Detect the encoding of a text file from its leading bytes. + * + * Known limitation inherited from the reference implementation: a BOM-less + * UTF-16 file whose content carries no zero bytes at all (e.g. purely CJK + * text) is reported as `'utf-8'`; strict UTF-8 decoding of it will then fail + * or produce garbage. Notepad and most editors write a BOM, so this is rare + * in practice. + */ +export function detectTextEncoding(sample: Uint8Array): TextEncodingDetection { + const classification = classifyTextSample(sample); + return { encoding: classification.encoding, seemsBinary: classification.isBinary }; +} + /** * Decode bytes in a detected UTF encoding to a JS string. Malformed * sequences are replaced (non-fatal) and a leading BOM is stripped. diff --git a/packages/agent-core-v2/src/_base/utils/fileMeta.ts b/packages/agent-core-v2/src/_base/utils/fileMeta.ts index f4dced47b..6ea99aec8 100644 --- a/packages/agent-core-v2/src/_base/utils/fileMeta.ts +++ b/packages/agent-core-v2/src/_base/utils/fileMeta.ts @@ -11,8 +11,11 @@ import { extname } from 'node:path'; +import { classifyTextSample } from '#/_base/text/encoding'; + +export { FS_BINARY_NONPRINTABLE_FRACTION } from '#/_base/text/encoding'; + export const FS_BINARY_SAMPLE_BYTES = 4096; -export const FS_BINARY_NONPRINTABLE_FRACTION = 0.3; export interface FileMetaStat { readonly size: number; @@ -21,16 +24,7 @@ export interface FileMetaStat { } export function detectBinary(buf: Uint8Array): boolean { - if (buf.length === 0) return false; - let nonPrintable = 0; - for (let i = 0; i < buf.length; i++) { - const b = buf[i]!; - if (b === 0) return true; - if (b === 9 || b === 10 || b === 13) continue; - if (b >= 32 && b <= 126) continue; - nonPrintable++; - } - return nonPrintable / buf.length > FS_BINARY_NONPRINTABLE_FRACTION; + return classifyTextSample(buf).isBinary; } export function countLines(text: string): number { diff --git a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts index 527f8459a..25283a769 100644 --- a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts +++ b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts @@ -56,11 +56,10 @@ const FsWireErrorCode = { } as const; import ignore, { type Ignore } from 'ignore'; -import { decodeUtfText, detectTextEncoding, type UtfTextEncoding } from '#/_base/text/encoding'; +import { classifyTextSample, decodeUtfText } from '#/_base/text/encoding'; import { buildEtag, countLines, - detectBinary, FS_BINARY_SAMPLE_BYTES, guessLanguageId, guessMime, @@ -259,20 +258,14 @@ export class WorkspaceFsService implements IWorkspaceFsService { const sampleSize = Math.min(FS_BINARY_SAMPLE_BYTES, st.size); const sample = sampleSize === 0 ? new Uint8Array() : await this.hostFs.readBytes(abs, sampleSize); - let isBinary = detectBinary(sample); - - // Trust encoding detection over the binary heuristic: a binary-looking - // sample can still be UTF-16 LE/BE text, and a BOM-marked UTF-16 file - // may not look binary at all (CJK-only content carries no zero bytes). - // Both are transcoded to UTF-8 so text clients can display them. - let transcodeEncoding: UtfTextEncoding | undefined; - if (req.encoding !== 'base64') { - const detection = detectTextEncoding(sample); - if (!detection.seemsBinary && detection.encoding !== 'utf-8') { - transcodeEncoding = detection.encoding; - isBinary = false; - } - } + const classification = classifyTextSample(sample); + const transcodeEncoding = + !classification.isBinary && classification.encoding !== 'utf-8' && req.encoding !== 'base64' + ? classification.encoding + : undefined; + const isBinary = + classification.isBinary || + (classification.encoding !== 'utf-8' && transcodeEncoding === undefined); if (isBinary && req.encoding === 'utf-8') { throw new Error2(ErrorCodes.FS_IS_BINARY, `file is binary: ${req.path}`, { @@ -448,7 +441,8 @@ export class WorkspaceFsService implements IWorkspaceFsService { const sampleSize = Math.min(FS_BINARY_SAMPLE_BYTES, st.size); const sample = sampleSize === 0 ? new Uint8Array() : await this.hostFs.readBytes(abs, sampleSize); - const isBinary = detectBinary(sample); + const classification = classifyTextSample(sample); + const isBinary = classification.isBinary || classification.encoding !== 'utf-8'; return { absolute: abs, relative: rel, diff --git a/packages/agent-core-v2/test/_base/text/encoding.test.ts b/packages/agent-core-v2/test/_base/text/encoding.test.ts index 1457b165a..826a77416 100644 --- a/packages/agent-core-v2/test/_base/text/encoding.test.ts +++ b/packages/agent-core-v2/test/_base/text/encoding.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { + classifyTextSample, decodeUtfText, detectTextEncoding, ENCODING_DETECTION_SAMPLE_BYTES, @@ -59,7 +60,7 @@ describe('detectTextEncoding', () => { it('limits the zero-byte heuristic to the leading sample window', () => { const sample = Buffer.alloc(ENCODING_DETECTION_SAMPLE_BYTES + 2, 0x61); sample[ENCODING_DETECTION_SAMPLE_BYTES + 1] = 0x00; - expect(detectTextEncoding(sample)).toEqual({ encoding: 'utf-8', seemsBinary: false }); + expect(detectTextEncoding(sample)).toEqual({ encoding: 'utf-8', seemsBinary: true }); }); it('flags zero bytes at both parities as binary', () => { @@ -78,6 +79,75 @@ describe('detectTextEncoding', () => { }); }); +describe('classifyTextSample', () => { + it('classifies UTF-8 multibyte text (CJK, emoji) as utf-8 text', () => { + const sample = Buffer.from('2026-08-16 INFO 启动完成 ✅\n处理请求 🚀 成功\n'.repeat(20), 'utf8'); + expect(classifyTextSample(sample)).toEqual({ isBinary: false, encoding: 'utf-8' }); + }); + + it('classifies an empty sample as utf-8 text', () => { + expect(classifyTextSample(new Uint8Array())).toEqual({ isBinary: false, encoding: 'utf-8' }); + }); + + it('classifies samples carrying NUL bytes as binary', () => { + expect( + classifyTextSample(Buffer.from([0x61, 0x62, 0x63, 0x00, 0x64, 0x65, 0x66])).isBinary, + ).toBe(true); + expect(classifyTextSample(Buffer.from([0x00, 0x00, 0x61, 0x62])).isBinary).toBe(true); + }); + + it('classifies control-char-heavy samples over the threshold as binary', () => { + const sample = Buffer.concat([Buffer.alloc(40, 0x1b), Buffer.alloc(60, 0x61)]); + expect(classifyTextSample(sample).isBinary).toBe(true); + }); + + it('keeps ANSI-colored log lines under the control-char threshold as text', () => { + const esc = String.fromCodePoint(0x1b); + const sample = Buffer.from(`${esc}[32mINFO${esc}[0m 启动完成 ✅\n`.repeat(10), 'utf8'); + expect(classifyTextSample(sample)).toEqual({ isBinary: false, encoding: 'utf-8' }); + }); + + it('classifies invalid UTF-8 without UTF-16 features as binary', () => { + expect(classifyTextSample(Buffer.from([0xd6, 0xd0, 0xc4, 0xe3, 0x31, 0x32]))).toEqual({ + isBinary: true, + encoding: 'utf-8', + }); + }); + + it('tolerates a multi-byte sequence truncated at the sample tail', () => { + const sample = Buffer.concat([Buffer.from('日志记录\n', 'utf8'), Buffer.from([0xe4, 0xb8])]); + expect(classifyTextSample(sample)).toEqual({ isBinary: false, encoding: 'utf-8' }); + }); + + it('treats a NUL byte beyond the UTF-16 parity window as binary', () => { + const sample = Buffer.concat([ + Buffer.alloc(600, 0x61), + Buffer.from([0x00]), + Buffer.alloc(100, 0x62), + ]); + expect(classifyTextSample(sample).isBinary).toBe(true); + }); + + it('rejects an impossible UTF-8 lead byte at the sample tail', () => { + const sample = Buffer.concat([Buffer.from('plain ascii log line\n'), Buffer.from([0xff])]); + expect(classifyTextSample(sample).isBinary).toBe(true); + }); + + it('rejects a tail lead byte not followed by continuation bytes', () => { + const sample = Buffer.concat([Buffer.from('plain ascii log line\n'), Buffer.from([0xe4, 0x41])]); + expect(classifyTextSample(sample).isBinary).toBe(true); + }); + + it('classifies UTF-16 BOM and zero-byte parity samples as text with the right encoding', () => { + const le = Buffer.concat([Buffer.from([0xff, 0xfe]), utf16Le('hello 你好')]); + expect(classifyTextSample(le)).toEqual({ isBinary: false, encoding: 'utf-16le' }); + expect(classifyTextSample(utf16Be('hello world, plain ascii'))).toEqual({ + isBinary: false, + encoding: 'utf-16be', + }); + }); +}); + describe('decodeUtfText', () => { it('decodes UTF-16 LE/BE and strips the BOM', () => { const le = Buffer.concat([Buffer.from([0xff, 0xfe]), utf16Le('你好\nworld')]); diff --git a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts index c1402d76e..47be101e7 100644 --- a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts +++ b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts @@ -95,10 +95,11 @@ function fakeFs( }; const lstatImpl = async (p: string) => { if (fileMap.has(p)) { + const c = fileMap.get(p)!; return { isFile: true, isDirectory: false, - size: fileMap.get(p)!.length, + size: Buffer.isBuffer(c) ? c.length : Buffer.byteLength(c), mtimeMs: 1000, ino: 1, }; @@ -789,6 +790,30 @@ describe('WorkspaceFsService.read', () => { expect(result.content).toBe(utf16.toString('base64')); }); + it('reads UTF-8 Chinese log content as text instead of throwing fs.is_binary', async () => { + const log = '2026-08-16 INFO 启动完成 ✅\n2026-08-16 INFO 处理请求 🚀 成功\n'.repeat(50); + const fs = makeSession({ 'app.log': log }, emptyHandler); + const result = await fs.read({ + path: 'app.log', + offset: 0, + length: 1024 * 1024, + encoding: 'utf-8', + }); + expect(result.content).toBe(log); + expect(result.encoding).toBe('utf-8'); + expect(result.is_binary).toBe(false); + expect(result.mime).toBe('text/plain'); + expect(result.truncated).toBe(false); + }); + + it('returns utf-8 rather than base64 for UTF-8 Chinese text in auto mode', async () => { + const fs = makeSession({ 'app.log': '中文日志 ✅\n' }, emptyHandler); + const result = await fs.read({ path: 'app.log', offset: 0, length: 1024, encoding: 'auto' }); + expect(result.content).toBe('中文日志 ✅\n'); + expect(result.encoding).toBe('utf-8'); + expect(result.is_binary).toBe(false); + }); + it('throws fs.is_directory for a directory', async () => { const fs = makeSession({ 'src/a.ts': '' }, emptyHandler); await expect( @@ -875,6 +900,12 @@ describe('WorkspaceFsService.resolveDownload', () => { expect(res.modifiedAt).toBeInstanceOf(Date); }); + it('resolves a UTF-8 Chinese log as text/plain', async () => { + const fs = makeSession({ 'app.log': '启动完成 ✅ 中文日志内容\n'.repeat(20) }, emptyHandler); + const res = await fs.resolveDownload('app.log'); + expect(res.mime).toBe('text/plain'); + }); + it('throws fs.is_directory for a directory', async () => { const fs = makeSession({ 'src/a.ts': '' }, emptyHandler); await expect(fs.resolveDownload('src')).rejects.toMatchObject({ code: 'fs.is_directory' }); diff --git a/packages/kap-server/src/routes/workspaceFs.ts b/packages/kap-server/src/routes/workspaceFs.ts index ff5902260..dad88cbe7 100644 --- a/packages/kap-server/src/routes/workspaceFs.ts +++ b/packages/kap-server/src/routes/workspaceFs.ts @@ -76,10 +76,10 @@ import { } from '@moonshot-ai/agent-core-v2/app/hostFolderBrowser/hostFolderBrowser'; import { buildEtag, - detectBinary, FS_BINARY_SAMPLE_BYTES, guessMime, } from '@moonshot-ai/agent-core-v2/_base/utils/fileMeta'; +import { classifyTextSample } from '@moonshot-ai/agent-core-v2/_base/text/encoding'; import { z } from 'zod'; import { errEnvelope, okEnvelope } from '../envelope'; @@ -289,7 +289,8 @@ async function handleFsContent( const sampleSize = Math.min(FS_BINARY_SAMPLE_BYTES, st.size); const sample = sampleSize === 0 ? new Uint8Array() : await hostFs.readBytes(abs, sampleSize); - isBinary = detectBinary(sample); + const classification = classifyTextSample(sample); + isBinary = classification.isBinary || classification.encoding !== 'utf-8'; } catch (err) { sendOsFsError(reply, requestId, err, path); return; diff --git a/packages/kap-server/test/workspaceFs.test.ts b/packages/kap-server/test/workspaceFs.test.ts index 50cd2e3be..99de88b1d 100644 --- a/packages/kap-server/test/workspaceFs.test.ts +++ b/packages/kap-server/test/workspaceFs.test.ts @@ -358,6 +358,17 @@ describe('server-v2 /api/v1 fs:content', () => { expect(res.headers.get('content-type')).toContain('text/plain'); }); + it('serves a UTF-8 Chinese .log file as text/plain', async () => { + const file = join(dir as string, 'server.log'); + const log = '2026-08-16 INFO 启动完成 ✅\n'.repeat(100); + await writeFile(file, log); + + const res = await getContent(file); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toContain('text/plain'); + expect(await res.text()).toBe(log); + }); + it('serves binary files byte-for-byte with an octet-stream fallback mime', async () => { const file = join(dir as string, 'blob.bin'); const original = Buffer.from([0x00, 0x01, 0x02, 0xff, 0xfe, 0x00, 0x10, 0x80]);