diff --git a/.changeset/large-file-reads.md b/.changeset/large-file-reads.md new file mode 100644 index 000000000..eaf25643b --- /dev/null +++ b/.changeset/large-file-reads.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Read large text files in bounded memory and read tail lines without scanning whole files. diff --git a/packages/agent-core/src/tools/builtin/file/read.ts b/packages/agent-core/src/tools/builtin/file/read.ts index 252eef3af..d6abd5764 100644 --- a/packages/agent-core/src/tools/builtin/file/read.ts +++ b/packages/agent-core/src/tools/builtin/file/read.ts @@ -83,6 +83,25 @@ type TextPreviewKaos = Kaos & { readTextPreview?: (path: string, n: number) => Promise; }; +interface TextFileScan { + totalLines: number; + endsWithNewline: boolean; + hasNul: boolean; + lineEndingFlags: LineEndingFlags; +} + +type RangeReadKaos = TextPreviewKaos & { + scanTextFile?: (path: string) => Promise; + readLineRange?: ( + path: string, + options: { startLine: number; maxLines: number; errors?: 'strict' | 'replace' | 'ignore' }, + ) => AsyncGenerator; + readTailLines?: ( + path: string, + options: { tailCount: number; errors?: 'strict' | 'replace' | 'ignore' }, + ) => AsyncGenerator; +}; + async function readTextHeader(kaos: TextPreviewKaos, path: string, n: number): Promise { if (kaos.readTextPreview !== undefined) { return kaos.readTextPreview(path, n); @@ -141,6 +160,41 @@ function renderedLineBytes(renderedLine: string, isFirst: boolean): number { return (isFirst ? 0 : 1) + Buffer.byteLength(renderedLine, 'utf8'); } +function renderEntries( + entries: readonly ReadLineEntry[], + lineEndingStyle: LineEndingStyle, +): { + renderedLines: string[]; + truncatedLineNumbers: number[]; + maxBytesReached: boolean; +} { + const renderedLines: string[] = []; + const truncatedLineNumbers: number[] = []; + let bytes = 0; + let maxBytesReached = false; + + for (const entry of entries) { + const rendered = renderLine(entry, lineEndingStyle); + const lineBytes = renderedLineBytes(rendered.line, renderedLines.length === 0); + if (renderedLines.length > 0 && bytes + lineBytes > MAX_BYTES) { + maxBytesReached = true; + break; + } + + if (rendered.wasTruncated) { + truncatedLineNumbers.push(entry.lineNo); + } + renderedLines.push(rendered.line); + bytes += lineBytes; + if (bytes >= MAX_BYTES) { + maxBytesReached = true; + break; + } + } + + return { renderedLines, truncatedLineNumbers, maxBytesReached }; +} + function isRegularFileMode(stMode: number): boolean { return (stMode & S_IFMT) === S_IFREG; } @@ -275,6 +329,36 @@ export class ReadTool implements BuiltinTool { effectiveLimit: number, requestedLines: number, ): Promise { + const rangeKaos = this.kaos as RangeReadKaos; + if (rangeKaos.scanTextFile !== undefined && rangeKaos.readLineRange !== undefined) { + const scan = await rangeKaos.scanTextFile(safePath); + if (scan.hasNul) { + return { isError: true, output: notReadableFileOutput(displayPath) }; + } + const selectedEntries: ReadLineEntry[] = []; + let lineNo = lineOffset; + for await (const rawLine of rangeKaos.readLineRange(safePath, { + startLine: lineOffset, + maxLines: effectiveLimit, + errors: 'strict', + })) { + selectedEntries.push({ lineNo, rawContent: stripTrailingLf(rawLine) }); + lineNo += 1; + } + const lineEndingStyle = lineEndingStyleFromFlags(scan.lineEndingFlags); + const rendered = renderEntries(selectedEntries, lineEndingStyle); + return this.finishReadResult({ + renderedLines: rendered.renderedLines, + truncatedLineNumbers: rendered.truncatedLineNumbers, + maxLinesReached: effectiveLimit >= MAX_LINES && lineOffset + MAX_LINES <= scan.totalLines, + maxBytesReached: rendered.maxBytesReached, + lineEndingStyle, + startLine: selectedEntries.length > 0 ? lineOffset : 0, + totalLines: scan.totalLines, + requestedLines, + }); + } + const selectedEntries: ReadLineEntry[] = []; const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; let currentLineNo = 0; @@ -311,37 +395,15 @@ export class ReadTool implements BuiltinTool { } const lineEndingStyle = lineEndingStyleFromFlags(flags); - const renderedLines: string[] = []; - const truncatedLineNumbers: number[] = []; - let bytes = 0; - let maxBytesReached = false; - - for (const entry of selectedEntries) { - const rendered = renderLine(entry, lineEndingStyle); - const lineBytes = renderedLineBytes(rendered.line, renderedLines.length === 0); - if (renderedLines.length > 0 && bytes + lineBytes > MAX_BYTES) { - maxBytesReached = true; - break; - } - - if (rendered.wasTruncated) { - truncatedLineNumbers.push(entry.lineNo); - } - renderedLines.push(rendered.line); - bytes += lineBytes; - if (bytes >= MAX_BYTES) { - maxBytesReached = true; - break; - } - } + const rendered = renderEntries(selectedEntries, lineEndingStyle); return this.finishReadResult({ - renderedLines, - truncatedLineNumbers, + renderedLines: rendered.renderedLines, + truncatedLineNumbers: rendered.truncatedLineNumbers, maxLinesReached, - maxBytesReached, + maxBytesReached: rendered.maxBytesReached, lineEndingStyle, - startLine: renderedLines.length > 0 ? lineOffset : 0, + startLine: selectedEntries.length > 0 ? lineOffset : 0, totalLines: currentLineNo, requestedLines, }); @@ -355,6 +417,33 @@ export class ReadTool implements BuiltinTool { requestedLines: number, ): Promise { const tailCount = Math.abs(lineOffset); + const rangeKaos = this.kaos as RangeReadKaos; + if (rangeKaos.scanTextFile !== undefined && rangeKaos.readTailLines !== undefined) { + const scan = await rangeKaos.scanTextFile(safePath); + if (scan.hasNul) { + return { isError: true, output: notReadableFileOutput(displayPath) }; + } + const rawLines: string[] = []; + for await (const rawLine of rangeKaos.readTailLines(safePath, { + tailCount, + errors: 'strict', + })) { + rawLines.push(rawLine); + } + const startLine = Math.max(1, scan.totalLines - rawLines.length + 1); + const entries = rawLines.map((rawLine, index) => ({ + lineNo: startLine + index, + rawContent: stripTrailingLf(rawLine), + })); + return this.finishTailEntries({ + entries, + lineEndingFlags: scan.lineEndingFlags, + effectiveLimit, + totalLines: scan.totalLines, + requestedLines, + }); + } + const entries: ReadLineEntry[] = []; const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; let currentLineNo = 0; @@ -374,8 +463,24 @@ export class ReadTool implements BuiltinTool { } } - const lineEndingStyle = lineEndingStyleFromFlags(flags); - let renderedCandidates = entries.slice(0, effectiveLimit).map((entry) => { + return this.finishTailEntries({ + entries, + lineEndingFlags: flags, + effectiveLimit, + totalLines: currentLineNo, + requestedLines, + }); + } + + private finishTailEntries(input: { + entries: readonly ReadLineEntry[]; + lineEndingFlags: LineEndingFlags; + effectiveLimit: number; + totalLines: number; + requestedLines: number; + }): ExecutableToolResult { + const lineEndingStyle = lineEndingStyleFromFlags(input.lineEndingFlags); + let renderedCandidates = input.entries.slice(0, input.effectiveLimit).map((entry) => { return { entry, rendered: renderLine(entry, lineEndingStyle) }; }); @@ -416,8 +521,8 @@ export class ReadTool implements BuiltinTool { maxBytesReached, lineEndingStyle, startLine: renderedCandidates[0]?.entry.lineNo ?? 0, - totalLines: currentLineNo, - requestedLines, + totalLines: input.totalLines, + requestedLines: input.requestedLines, }); } diff --git a/packages/agent-core/test/tools/read.test.ts b/packages/agent-core/test/tools/read.test.ts index 54e1fabdd..a3ba41615 100644 --- a/packages/agent-core/test/tools/read.test.ts +++ b/packages/agent-core/test/tools/read.test.ts @@ -636,6 +636,113 @@ describe('ReadTool', () => { expect(readText).not.toHaveBeenCalled(); }); + it('uses range reader when available without consuming readLines', async () => { + const content = Array.from({ length: 20 }, (_, i) => `line ${String(i + 1)}`).join('\n'); + const bytes = Buffer.from(content, 'utf8'); + const readLines = vi.fn(); + const readLineRange = vi.fn(async function* readLineRange( + _path: string, + options: { startLine: number; maxLines: number }, + ): AsyncGenerator { + for (let i = options.startLine; i < options.startLine + options.maxLines; i += 1) { + yield `line ${String(i)}\n`; + } + }); + const scanTextFile = vi.fn(async () => ({ + totalLines: 20, + endsWithNewline: false, + hasNul: false, + lineEndingFlags: { hasCrLf: false, hasLf: true, hasLoneCr: false }, + })); + const tool = new ReadTool( + createFakeKaos({ + stat: vi.fn().mockResolvedValue(REGULAR_FILE_STAT), + readBytes: vi.fn().mockImplementation(async (_path, n) => { + return n === undefined ? bytes : bytes.subarray(0, n); + }), + readLines, + scanTextFile, + readLineRange, + } as unknown as Partial), + PERMISSIVE_WORKSPACE, + ); + + const result = await executeTool(tool, context({ path: '/tmp/range.txt', line_offset: 5, n_lines: 3 })); + const output = toolContentString(result); + + expect(output).toContain('5\tline 5'); + expect(output).toContain('7\tline 7'); + expect(output).not.toContain('8\tline 8'); + expect(readLineRange).toHaveBeenCalledWith('/tmp/range.txt', { + startLine: 5, + maxLines: 3, + errors: 'strict', + }); + expect(readLines).not.toHaveBeenCalled(); + }); + + it('uses tail reader when available without consuming readLines', async () => { + const content = Array.from({ length: 20 }, (_, i) => `line ${String(i + 1)}`).join('\n'); + const bytes = Buffer.from(content, 'utf8'); + const readLines = vi.fn(); + const readTailLines = vi.fn(async function* readTailLines(): AsyncGenerator { + yield 'line 18\n'; + yield 'line 19\n'; + yield 'line 20'; + }); + const scanTextFile = vi.fn(async () => ({ + totalLines: 20, + endsWithNewline: false, + hasNul: false, + lineEndingFlags: { hasCrLf: false, hasLf: true, hasLoneCr: false }, + })); + const tool = new ReadTool( + createFakeKaos({ + stat: vi.fn().mockResolvedValue(REGULAR_FILE_STAT), + readBytes: vi.fn().mockImplementation(async (_path, n) => { + return n === undefined ? bytes : bytes.subarray(0, n); + }), + readLines, + scanTextFile, + readTailLines, + } as unknown as Partial), + PERMISSIVE_WORKSPACE, + ); + + const result = await executeTool(tool, context({ path: '/tmp/tail.txt', line_offset: -3 })); + const output = toolContentString(result); + + expect(output).toContain('18\tline 18'); + expect(output).toContain('20\tline 20'); + expect(readTailLines).toHaveBeenCalledWith('/tmp/tail.txt', { tailCount: 3, errors: 'strict' }); + expect(readLines).not.toHaveBeenCalled(); + }); + + it('short-circuits on scan NUL before range read', async () => { + const readLineRange = vi.fn(); + const tool = new ReadTool( + createFakeKaos({ + stat: vi.fn().mockResolvedValue(REGULAR_FILE_STAT), + readBytes: vi.fn().mockResolvedValue(Buffer.from('text')), + scanTextFile: vi.fn(async () => ({ + totalLines: 1, + endsWithNewline: false, + hasNul: true, + lineEndingFlags: { hasCrLf: false, hasLf: false, hasLoneCr: false }, + })), + readLineRange, + } as unknown as Partial), + PERMISSIVE_WORKSPACE, + ); + + const result = await executeTool(tool, context({ path: '/tmp/nul.txt' })); + const output = toolContentString(result); + + expect(result.isError).toBe(true); + expect(output).toContain('is not readable as UTF-8 text'); + expect(readLineRange).not.toHaveBeenCalled(); + }); + it('caps default reads at MAX_LINES', async () => { const content = Array.from({ length: MAX_LINES + 1 }, (_, i) => `line ${String(i + 1)}`).join( '\n', diff --git a/packages/kaos/src/internal.ts b/packages/kaos/src/internal.ts index 1a4742e18..6c89acf27 100644 --- a/packages/kaos/src/internal.ts +++ b/packages/kaos/src/internal.ts @@ -136,6 +136,7 @@ export function decodeTextWithErrors( data: Buffer, encoding: BufferEncoding, errors: 'strict' | 'replace' | 'ignore' = 'strict', + ignoreBOM: boolean = false, ): string { // Map Node's BufferEncoding names to Web TextDecoder labels where the two // diverge. Only UTF-family encodings participate in the strict/replace/ @@ -163,7 +164,7 @@ export function decodeTextWithErrors( } if (errors === 'strict') { - return new TextDecoder(webLabel, { fatal: true }).decode(data); + return new TextDecoder(webLabel, { fatal: true, ignoreBOM }).decode(data); } // 'ignore' must skip invalid input bytes/code units, not delete every @@ -174,7 +175,7 @@ export function decodeTextWithErrors( } // 'replace' β†’ substitute each invalid sequence with U+FFFD (default). - return new TextDecoder(webLabel, { fatal: false }).decode(data); + return new TextDecoder(webLabel, { fatal: false, ignoreBOM }).decode(data); } /** diff --git a/packages/kaos/src/local.ts b/packages/kaos/src/local.ts index 484f0b090..ba3c7f29d 100644 --- a/packages/kaos/src/local.ts +++ b/packages/kaos/src/local.ts @@ -22,6 +22,22 @@ import type { KaosProcess } from './process'; import type { StatResult } from './types'; const isWindows: boolean = process.platform === 'win32'; +const READ_CHUNK_SIZE = 64 * 1024; + +type TextDecodeErrors = 'strict' | 'replace' | 'ignore'; + +interface LineEndingFlags { + hasCrLf: boolean; + hasLf: boolean; + hasLoneCr: boolean; +} + +interface TextFileScan { + totalLines: number; + endsWithNewline: boolean; + hasNul: boolean; + lineEndingFlags: LineEndingFlags; +} /** * Build the `(dev, ino)` cycle-detection key used by `_globWalk`'s @@ -456,22 +472,173 @@ export class LocalKaos implements Kaos { async *readLines( path: string, - options?: { encoding?: BufferEncoding; errors?: 'strict' | 'replace' | 'ignore' }, + options?: { encoding?: BufferEncoding; errors?: TextDecodeErrors }, ): AsyncGenerator { const resolved = this._resolvePath(path); const encoding = options?.encoding ?? 'utf-8'; const errors = options?.errors ?? 'strict'; - const buf = await readFile(resolved); - const content = decodeTextWithErrors(buf, encoding, errors); - const lines = content.split('\n'); - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; - if (line === undefined) continue; - if (i < lines.length - 1) { - yield line + '\n'; - } else if (line !== '') { - yield line; + + if (!isUtf8Encoding(encoding)) { + const content = decodeTextWithErrors(await readFile(resolved), encoding, errors); + yield* splitLinesKeepingTerminator(content); + return; + } + + yield* this._readUtf8Lines(resolved, errors); + } + + async scanTextFile(path: string): Promise { + const resolved = this._resolvePath(path); + const fh = await open(resolved, 'r'); + try { + const buf = Buffer.alloc(READ_CHUNK_SIZE); + const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; + const validator = createUtf8Validator(); + let totalLines = 0; + let totalBytes = 0; + let endsWithNewline = false; + let hasNul = false; + let prevWasCr = false; + + while (true) { + const { bytesRead } = await fh.read(buf, 0, buf.length, null); + if (bytesRead === 0) break; + const chunk = buf.subarray(0, bytesRead); + validator.write(chunk); + for (let i = 0; i < chunk.length; i += 1) { + const byte = chunk[i]; + if (byte === undefined) continue; + if (byte === 0) hasNul = true; + if (byte === 0x0a) totalLines += 1; + } + prevWasCr = updateLineEndingFlagsFromBytes(flags, chunk, prevWasCr); + totalBytes += bytesRead; + endsWithNewline = chunk[bytesRead - 1] === 0x0a; } + + if (prevWasCr) flags.hasLoneCr = true; + validator.end(); + if (totalBytes > 0 && !endsWithNewline) totalLines += 1; + return { totalLines, endsWithNewline, hasNul, lineEndingFlags: flags }; + } finally { + await fh.close(); + } + } + + async *readLineRange( + path: string, + options: { startLine: number; maxLines: number; errors?: TextDecodeErrors }, + ): AsyncGenerator { + const resolved = this._resolvePath(path); + const errors = options.errors ?? 'strict'; + yield* this._readUtf8Lines(resolved, errors, { + startLine: options.startLine, + maxLines: options.maxLines, + }); + } + + async *readTailLines( + path: string, + options: { tailCount: number; errors?: TextDecodeErrors }, + ): AsyncGenerator { + if (options.tailCount <= 0) return; + const resolved = this._resolvePath(path); + const errors = options.errors ?? 'strict'; + const fh = await open(resolved, 'r'); + try { + const s = await fh.stat(); + if (s.size === 0) return; + + let pos = s.size; + let foundLf = 0; + let startOffset = 0; + let needLf = options.tailCount; + let sawTailBlock = false; + + while (pos > 0 && foundLf < needLf) { + const readSize = Math.min(READ_CHUNK_SIZE, pos); + pos -= readSize; + const buf = Buffer.alloc(readSize); + await fh.read(buf, 0, readSize, pos); + if (!sawTailBlock) { + sawTailBlock = true; + const endsWithNewline = buf[readSize - 1] === 0x0a; + needLf = endsWithNewline ? options.tailCount + 1 : options.tailCount; + } + for (let i = readSize - 1; i >= 0; i -= 1) { + const byte = buf[i]; + if (byte !== 0x0a) continue; + foundLf += 1; + if (foundLf === needLf) { + startOffset = pos + i + 1; + break; + } + } + } + + if (foundLf < needLf) startOffset = 0; + const data = await readRange(fh, startOffset, s.size - startOffset); + const text = decodeTextWithErrors(data, 'utf-8', errors, startOffset !== 0); + yield* splitLinesKeepingTerminator(text); + } finally { + await fh.close(); + } + } + + private async *_readUtf8Lines( + resolved: string, + errors: TextDecodeErrors, + range?: { startLine?: number; maxLines?: number }, + ): AsyncGenerator { + const startLine = range?.startLine ?? 1; + const maxLines = range?.maxLines ?? Number.POSITIVE_INFINITY; + const fh = await open(resolved, 'r'); + try { + const buf = Buffer.alloc(READ_CHUNK_SIZE); + let pending: Buffer[] = []; + let pendingOffset = 0; + let fileOffset = 0; + let lineNo = 1; + let yielded = 0; + + while (true) { + const { bytesRead } = await fh.read(buf, 0, buf.length, null); + if (bytesRead === 0) break; + const chunk = buf.subarray(0, bytesRead); + let lineStart = 0; + + for (let i = 0; i < chunk.length; i += 1) { + const byte = chunk[i]; + if (byte !== 0x0a) continue; + const piece = chunk.subarray(lineStart, i + 1); + const lineOffset = pending.length === 0 ? fileOffset + lineStart : pendingOffset; + const line = pending.length === 0 ? piece : Buffer.concat([...pending, piece]); + if (lineNo >= startLine) { + yield decodeTextWithErrors(line, 'utf-8', errors, lineOffset !== 0); + yielded += 1; + if (yielded >= maxLines) return; + } + pending = []; + lineStart = i + 1; + lineNo += 1; + } + + if (lineStart < chunk.length) { + const tail = Buffer.from(chunk.subarray(lineStart)); + if (pending.length === 0) pendingOffset = fileOffset + lineStart; + pending.push(tail); + } + fileOffset += bytesRead; + } + + if (pending.length > 0) { + const line = Buffer.concat(pending); + if (lineNo >= startLine) { + yield decodeTextWithErrors(line, 'utf-8', errors, pendingOffset !== 0); + } + } + } finally { + await fh.close(); } } @@ -597,6 +764,118 @@ export class LocalKaos implements Kaos { } } +function isUtf8Encoding(encoding: BufferEncoding): boolean { + return encoding === 'utf-8' || encoding === 'utf8'; +} + +function* splitLinesKeepingTerminator(text: string): Generator { + if (text.length === 0) return; + let start = 0; + for (let i = 0; i < text.length; i += 1) { + if (text.codePointAt(i) === 0x0a) { + yield text.slice(start, i + 1); + start = i + 1; + } + } + if (start < text.length) { + yield text.slice(start); + } +} + +function updateLineEndingFlagsFromBytes( + flags: LineEndingFlags, + chunk: Buffer, + prevWasCr: boolean, +): boolean { + for (let i = 0; i < chunk.length; i += 1) { + const byte = chunk[i]; + if (byte === undefined) continue; + if (byte === 0x0d) { + if (prevWasCr) flags.hasLoneCr = true; + prevWasCr = true; + } else if (byte === 0x0a) { + if (prevWasCr) { + flags.hasCrLf = true; + } else { + flags.hasLf = true; + } + prevWasCr = false; + } else { + if (prevWasCr) flags.hasLoneCr = true; + prevWasCr = false; + } + } + return prevWasCr; +} + +function createUtf8Validator(): { write(chunk: Buffer): void; end(): void } { + let needed = 0; + let lower = 0x80; + let upper = 0xbf; + + const fail = (): never => { + throw new TypeError('Invalid UTF-8 data'); + }; + + return { + write(chunk: Buffer): void { + for (let i = 0; i < chunk.length; i += 1) { + const byte = chunk[i]; + if (byte === undefined) continue; + if (needed === 0) { + if (byte <= 0x7f) continue; + if (byte >= 0xc2 && byte <= 0xdf) { + needed = 1; + } else if (byte === 0xe0) { + needed = 2; + lower = 0xa0; + } else if (byte >= 0xe1 && byte <= 0xec) { + needed = 2; + } else if (byte === 0xed) { + needed = 2; + upper = 0x9f; + } else if (byte >= 0xee && byte <= 0xef) { + needed = 2; + } else if (byte === 0xf0) { + needed = 3; + lower = 0x90; + } else if (byte >= 0xf1 && byte <= 0xf3) { + needed = 3; + } else if (byte === 0xf4) { + needed = 3; + upper = 0x8f; + } else { + fail(); + } + } else { + if (byte < lower || byte > upper) fail(); + lower = 0x80; + upper = 0xbf; + needed -= 1; + } + } + }, + end(): void { + if (needed !== 0) fail(); + }, + }; +} + +async function readRange( + fh: Awaited>, + start: number, + length: number, +): Promise { + const data = Buffer.alloc(length); + let offset = 0; + while (offset < length) { + const { bytesRead } = await fh.read(data, offset, length - offset, start + offset); + if (bytesRead === 0) break; + offset += bytesRead; + } + return offset === length ? data : data.subarray(0, offset); +} + // Wait for a freshly spawned ChildProcess to either emit 'spawn' (success) or // 'error' (ENOENT / EACCES / etc.). Until this resolves, callers should not // assume the child is running β€” they may otherwise write to the stdin of a diff --git a/packages/kaos/test/local.test.ts b/packages/kaos/test/local.test.ts index 01f192f88..cb2d23e93 100644 --- a/packages/kaos/test/local.test.ts +++ b/packages/kaos/test/local.test.ts @@ -176,6 +176,164 @@ describe('LocalKaos', () => { }); }); + describe('readLines streaming', () => { + async function collectLines(path: string, options?: Parameters[1]) { + const lines: string[] = []; + for await (const line of kaos.readLines(path, options)) { + lines.push(line); + } + return lines; + } + + it('preserves content exactly across representative line endings', async () => { + const fixtures: Array<[string, string]> = [ + ['multiline', 'line1\nline2\nline3\n'], + ['no trailing newline', 'line1\nline2'], + ['single line', 'only'], + ['single newline', '\n'], + ['empty', ''], + ['crlf', 'a\r\nb\r\n'], + ['lone cr', 'a\rB\n'], + ]; + for (const [name, content] of fixtures) { + const filePath = join(tempDir, `${name}.txt`); + await kaos.writeText(filePath, content); + expect((await collectLines(filePath)).join('')).toBe(content); + } + }); + + it('preserves multibyte characters and long single lines across chunk boundaries', async () => { + const filePath = join(tempDir, 'boundary.txt'); + const content = `${'a'.repeat(65535)}πŸ˜€\n${'x'.repeat(200000)}`; + await kaos.writeText(filePath, content); + await expect(collectLines(filePath)).resolves.toEqual([ + `${'a'.repeat(65535)}πŸ˜€\n`, + 'x'.repeat(200000), + ]); + }); + + it('preserves U+FEFF at the start of a non-first line', async () => { + const filePath = join(tempDir, 'bom-line.txt'); + const content = 'a\n\uFEFFb\n'; + await kaos.writeText(filePath, content); + await expect(collectLines(filePath)).resolves.toEqual(['a\n', '\uFEFFb\n']); + }); + + it('keeps utf16le and hex on the decode-then-split path', async () => { + const utf16Path = join(tempDir, 'utf16le.txt'); + await kaos.writeBytes(utf16Path, Buffer.from('a\n\u0A41\n', 'utf16le')); + await expect(collectLines(utf16Path, { encoding: 'utf16le' })).resolves.toEqual([ + 'a\n', + 'ੁ\n', + ]); + + const hexPath = join(tempDir, 'hex.txt'); + await kaos.writeBytes(hexPath, Buffer.from('a\nb')); + await expect(collectLines(hexPath, { encoding: 'hex' })).resolves.toEqual(['610a62']); + }); + + it('throws lazily when strict UTF-8 errors appear after the first line', async () => { + const filePath = join(tempDir, 'invalid-after-first-line.txt'); + await kaos.writeBytes(filePath, Buffer.concat([Buffer.from('ok\n', 'utf-8'), Buffer.from([0xff])])); + const gen = kaos.readLines(filePath); + await expect(gen.next()).resolves.toMatchObject({ value: 'ok\n', done: false }); + await expect(gen.next()).rejects.toThrow(); + }); + }); + + describe('scanTextFile', () => { + it('counts lines and classifies line endings', async () => { + const lf = join(tempDir, 'lf.txt'); + await kaos.writeText(lf, 'a\nb'); + await expect(kaos.scanTextFile(lf)).resolves.toMatchObject({ + totalLines: 2, + endsWithNewline: false, + hasNul: false, + lineEndingFlags: { hasCrLf: false, hasLf: true, hasLoneCr: false }, + }); + + const crlf = join(tempDir, 'crlf.txt'); + await kaos.writeText(crlf, 'a\r\nb\r\n'); + await expect(kaos.scanTextFile(crlf)).resolves.toMatchObject({ + totalLines: 2, + endsWithNewline: true, + lineEndingFlags: { hasCrLf: true, hasLf: false, hasLoneCr: false }, + }); + + const loneCr = join(tempDir, 'lone-cr.txt'); + await kaos.writeText(loneCr, 'a\rB\n'); + await expect(kaos.scanTextFile(loneCr)).resolves.toMatchObject({ + totalLines: 1, + lineEndingFlags: { hasCrLf: false, hasLf: true, hasLoneCr: true }, + }); + }); + + it('detects NUL and invalid UTF-8', async () => { + const nul = join(tempDir, 'nul.txt'); + await kaos.writeBytes(nul, Buffer.from('a\u0000b\n', 'utf-8')); + await expect(kaos.scanTextFile(nul)).resolves.toMatchObject({ hasNul: true }); + + const invalid = join(tempDir, 'invalid.txt'); + await kaos.writeBytes(invalid, Buffer.from([0xff])); + await expect(kaos.scanTextFile(invalid)).rejects.toThrow(); + }); + }); + + describe('readLineRange', () => { + async function collectRange(path: string, startLine: number, maxLines: number) { + const lines: string[] = []; + for await (const line of kaos.readLineRange(path, { startLine, maxLines })) { + lines.push(line); + } + return lines; + } + + it('reads only the requested line window', async () => { + const filePath = join(tempDir, 'range.txt'); + await kaos.writeText(filePath, 'a\nb\nc\nd\n'); + await expect(collectRange(filePath, 2, 2)).resolves.toEqual(['b\n', 'c\n']); + await expect(collectRange(filePath, 5, 2)).resolves.toEqual([]); + }); + + it('preserves U+FEFF at the start of a ranged non-first line', async () => { + const filePath = join(tempDir, 'range-bom.txt'); + await kaos.writeText(filePath, 'a\n\uFEFFb\n'); + await expect(collectRange(filePath, 2, 1)).resolves.toEqual(['\uFEFFb\n']); + }); + }); + + describe('readTailLines', () => { + async function collectTail(path: string, tailCount: number) { + const lines: string[] = []; + for await (const line of kaos.readTailLines(path, { tailCount })) { + lines.push(line); + } + return lines; + } + + it('reads last lines with and without trailing newline', async () => { + const trailing = join(tempDir, 'tail-trailing.txt'); + await kaos.writeText(trailing, 'a\nb\nc\n'); + await expect(collectTail(trailing, 2)).resolves.toEqual(['b\n', 'c\n']); + + const noTrailing = join(tempDir, 'tail-no-trailing.txt'); + await kaos.writeText(noTrailing, 'a\nb\nc'); + await expect(collectTail(noTrailing, 2)).resolves.toEqual(['b\n', 'c']); + }); + + it('returns the whole file when tailCount exceeds line count', async () => { + const filePath = join(tempDir, 'tail-short.txt'); + await kaos.writeText(filePath, 'a\nb\n'); + await expect(collectTail(filePath, 5)).resolves.toEqual(['a\n', 'b\n']); + }); + + it('preserves CRLF and U+FEFF in tail lines', async () => { + const filePath = join(tempDir, 'tail-crlf-bom.txt'); + await kaos.writeText(filePath, 'a\r\n\uFEFFb\r\n'); + await expect(collectTail(filePath, 1)).resolves.toEqual(['\uFEFFb\r\n']); + }); + }); + describe('readText errors parameter (Python compat)', () => { // A file with a valid UTF-8 prefix "δΈ­", an invalid standalone byte 0xff, // and a valid UTF-8 suffix "ζ–‡". Under strict decoding this throws.