mirror of
https://github.com/MoonshotAI/kimi-code.git
synced 2026-07-10 01:39:25 +00:00
perf(kaos): optimize large file reads (#971)
This commit is contained in:
7Sageer
GitHub
parent
6b68aa85e2
commit
b84704bff3
6 changed files with 699 additions and 44 deletions
5
.changeset/large-file-reads.md
Normal file
5
.changeset/large-file-reads.md
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
"@moonshot-ai/kimi-code": patch
|
||||
---
|
||||
|
||||
Read large text files in bounded memory and read tail lines without scanning whole files.
|
||||
|
|
@ -83,6 +83,25 @@ type TextPreviewKaos = Kaos & {
|
|||
readTextPreview?: (path: string, n: number) => Promise<Buffer>;
|
||||
};
|
||||
|
||||
interface TextFileScan {
|
||||
totalLines: number;
|
||||
endsWithNewline: boolean;
|
||||
hasNul: boolean;
|
||||
lineEndingFlags: LineEndingFlags;
|
||||
}
|
||||
|
||||
type RangeReadKaos = TextPreviewKaos & {
|
||||
scanTextFile?: (path: string) => Promise<TextFileScan>;
|
||||
readLineRange?: (
|
||||
path: string,
|
||||
options: { startLine: number; maxLines: number; errors?: 'strict' | 'replace' | 'ignore' },
|
||||
) => AsyncGenerator<string>;
|
||||
readTailLines?: (
|
||||
path: string,
|
||||
options: { tailCount: number; errors?: 'strict' | 'replace' | 'ignore' },
|
||||
) => AsyncGenerator<string>;
|
||||
};
|
||||
|
||||
async function readTextHeader(kaos: TextPreviewKaos, path: string, n: number): Promise<Buffer> {
|
||||
if (kaos.readTextPreview !== undefined) {
|
||||
return kaos.readTextPreview(path, n);
|
||||
|
|
@ -141,6 +160,41 @@ function renderedLineBytes(renderedLine: string, isFirst: boolean): number {
|
|||
return (isFirst ? 0 : 1) + Buffer.byteLength(renderedLine, 'utf8');
|
||||
}
|
||||
|
||||
function renderEntries(
|
||||
entries: readonly ReadLineEntry[],
|
||||
lineEndingStyle: LineEndingStyle,
|
||||
): {
|
||||
renderedLines: string[];
|
||||
truncatedLineNumbers: number[];
|
||||
maxBytesReached: boolean;
|
||||
} {
|
||||
const renderedLines: string[] = [];
|
||||
const truncatedLineNumbers: number[] = [];
|
||||
let bytes = 0;
|
||||
let maxBytesReached = false;
|
||||
|
||||
for (const entry of entries) {
|
||||
const rendered = renderLine(entry, lineEndingStyle);
|
||||
const lineBytes = renderedLineBytes(rendered.line, renderedLines.length === 0);
|
||||
if (renderedLines.length > 0 && bytes + lineBytes > MAX_BYTES) {
|
||||
maxBytesReached = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (rendered.wasTruncated) {
|
||||
truncatedLineNumbers.push(entry.lineNo);
|
||||
}
|
||||
renderedLines.push(rendered.line);
|
||||
bytes += lineBytes;
|
||||
if (bytes >= MAX_BYTES) {
|
||||
maxBytesReached = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { renderedLines, truncatedLineNumbers, maxBytesReached };
|
||||
}
|
||||
|
||||
function isRegularFileMode(stMode: number): boolean {
|
||||
return (stMode & S_IFMT) === S_IFREG;
|
||||
}
|
||||
|
|
@ -275,6 +329,36 @@ export class ReadTool implements BuiltinTool<ReadInput> {
|
|||
effectiveLimit: number,
|
||||
requestedLines: number,
|
||||
): Promise<ExecutableToolResult> {
|
||||
const rangeKaos = this.kaos as RangeReadKaos;
|
||||
if (rangeKaos.scanTextFile !== undefined && rangeKaos.readLineRange !== undefined) {
|
||||
const scan = await rangeKaos.scanTextFile(safePath);
|
||||
if (scan.hasNul) {
|
||||
return { isError: true, output: notReadableFileOutput(displayPath) };
|
||||
}
|
||||
const selectedEntries: ReadLineEntry[] = [];
|
||||
let lineNo = lineOffset;
|
||||
for await (const rawLine of rangeKaos.readLineRange(safePath, {
|
||||
startLine: lineOffset,
|
||||
maxLines: effectiveLimit,
|
||||
errors: 'strict',
|
||||
})) {
|
||||
selectedEntries.push({ lineNo, rawContent: stripTrailingLf(rawLine) });
|
||||
lineNo += 1;
|
||||
}
|
||||
const lineEndingStyle = lineEndingStyleFromFlags(scan.lineEndingFlags);
|
||||
const rendered = renderEntries(selectedEntries, lineEndingStyle);
|
||||
return this.finishReadResult({
|
||||
renderedLines: rendered.renderedLines,
|
||||
truncatedLineNumbers: rendered.truncatedLineNumbers,
|
||||
maxLinesReached: effectiveLimit >= MAX_LINES && lineOffset + MAX_LINES <= scan.totalLines,
|
||||
maxBytesReached: rendered.maxBytesReached,
|
||||
lineEndingStyle,
|
||||
startLine: selectedEntries.length > 0 ? lineOffset : 0,
|
||||
totalLines: scan.totalLines,
|
||||
requestedLines,
|
||||
});
|
||||
}
|
||||
|
||||
const selectedEntries: ReadLineEntry[] = [];
|
||||
const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false };
|
||||
let currentLineNo = 0;
|
||||
|
|
@ -311,37 +395,15 @@ export class ReadTool implements BuiltinTool<ReadInput> {
|
|||
}
|
||||
|
||||
const lineEndingStyle = lineEndingStyleFromFlags(flags);
|
||||
const renderedLines: string[] = [];
|
||||
const truncatedLineNumbers: number[] = [];
|
||||
let bytes = 0;
|
||||
let maxBytesReached = false;
|
||||
|
||||
for (const entry of selectedEntries) {
|
||||
const rendered = renderLine(entry, lineEndingStyle);
|
||||
const lineBytes = renderedLineBytes(rendered.line, renderedLines.length === 0);
|
||||
if (renderedLines.length > 0 && bytes + lineBytes > MAX_BYTES) {
|
||||
maxBytesReached = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (rendered.wasTruncated) {
|
||||
truncatedLineNumbers.push(entry.lineNo);
|
||||
}
|
||||
renderedLines.push(rendered.line);
|
||||
bytes += lineBytes;
|
||||
if (bytes >= MAX_BYTES) {
|
||||
maxBytesReached = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const rendered = renderEntries(selectedEntries, lineEndingStyle);
|
||||
|
||||
return this.finishReadResult({
|
||||
renderedLines,
|
||||
truncatedLineNumbers,
|
||||
renderedLines: rendered.renderedLines,
|
||||
truncatedLineNumbers: rendered.truncatedLineNumbers,
|
||||
maxLinesReached,
|
||||
maxBytesReached,
|
||||
maxBytesReached: rendered.maxBytesReached,
|
||||
lineEndingStyle,
|
||||
startLine: renderedLines.length > 0 ? lineOffset : 0,
|
||||
startLine: selectedEntries.length > 0 ? lineOffset : 0,
|
||||
totalLines: currentLineNo,
|
||||
requestedLines,
|
||||
});
|
||||
|
|
@ -355,6 +417,33 @@ export class ReadTool implements BuiltinTool<ReadInput> {
|
|||
requestedLines: number,
|
||||
): Promise<ExecutableToolResult> {
|
||||
const tailCount = Math.abs(lineOffset);
|
||||
const rangeKaos = this.kaos as RangeReadKaos;
|
||||
if (rangeKaos.scanTextFile !== undefined && rangeKaos.readTailLines !== undefined) {
|
||||
const scan = await rangeKaos.scanTextFile(safePath);
|
||||
if (scan.hasNul) {
|
||||
return { isError: true, output: notReadableFileOutput(displayPath) };
|
||||
}
|
||||
const rawLines: string[] = [];
|
||||
for await (const rawLine of rangeKaos.readTailLines(safePath, {
|
||||
tailCount,
|
||||
errors: 'strict',
|
||||
})) {
|
||||
rawLines.push(rawLine);
|
||||
}
|
||||
const startLine = Math.max(1, scan.totalLines - rawLines.length + 1);
|
||||
const entries = rawLines.map((rawLine, index) => ({
|
||||
lineNo: startLine + index,
|
||||
rawContent: stripTrailingLf(rawLine),
|
||||
}));
|
||||
return this.finishTailEntries({
|
||||
entries,
|
||||
lineEndingFlags: scan.lineEndingFlags,
|
||||
effectiveLimit,
|
||||
totalLines: scan.totalLines,
|
||||
requestedLines,
|
||||
});
|
||||
}
|
||||
|
||||
const entries: ReadLineEntry[] = [];
|
||||
const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false };
|
||||
let currentLineNo = 0;
|
||||
|
|
@ -374,8 +463,24 @@ export class ReadTool implements BuiltinTool<ReadInput> {
|
|||
}
|
||||
}
|
||||
|
||||
const lineEndingStyle = lineEndingStyleFromFlags(flags);
|
||||
let renderedCandidates = entries.slice(0, effectiveLimit).map((entry) => {
|
||||
return this.finishTailEntries({
|
||||
entries,
|
||||
lineEndingFlags: flags,
|
||||
effectiveLimit,
|
||||
totalLines: currentLineNo,
|
||||
requestedLines,
|
||||
});
|
||||
}
|
||||
|
||||
private finishTailEntries(input: {
|
||||
entries: readonly ReadLineEntry[];
|
||||
lineEndingFlags: LineEndingFlags;
|
||||
effectiveLimit: number;
|
||||
totalLines: number;
|
||||
requestedLines: number;
|
||||
}): ExecutableToolResult {
|
||||
const lineEndingStyle = lineEndingStyleFromFlags(input.lineEndingFlags);
|
||||
let renderedCandidates = input.entries.slice(0, input.effectiveLimit).map((entry) => {
|
||||
return { entry, rendered: renderLine(entry, lineEndingStyle) };
|
||||
});
|
||||
|
||||
|
|
@ -416,8 +521,8 @@ export class ReadTool implements BuiltinTool<ReadInput> {
|
|||
maxBytesReached,
|
||||
lineEndingStyle,
|
||||
startLine: renderedCandidates[0]?.entry.lineNo ?? 0,
|
||||
totalLines: currentLineNo,
|
||||
requestedLines,
|
||||
totalLines: input.totalLines,
|
||||
requestedLines: input.requestedLines,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -636,6 +636,113 @@ describe('ReadTool', () => {
|
|||
expect(readText).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('uses range reader when available without consuming readLines', async () => {
|
||||
const content = Array.from({ length: 20 }, (_, i) => `line ${String(i + 1)}`).join('\n');
|
||||
const bytes = Buffer.from(content, 'utf8');
|
||||
const readLines = vi.fn<Kaos['readLines']>();
|
||||
const readLineRange = vi.fn(async function* readLineRange(
|
||||
_path: string,
|
||||
options: { startLine: number; maxLines: number },
|
||||
): AsyncGenerator<string> {
|
||||
for (let i = options.startLine; i < options.startLine + options.maxLines; i += 1) {
|
||||
yield `line ${String(i)}\n`;
|
||||
}
|
||||
});
|
||||
const scanTextFile = vi.fn(async () => ({
|
||||
totalLines: 20,
|
||||
endsWithNewline: false,
|
||||
hasNul: false,
|
||||
lineEndingFlags: { hasCrLf: false, hasLf: true, hasLoneCr: false },
|
||||
}));
|
||||
const tool = new ReadTool(
|
||||
createFakeKaos({
|
||||
stat: vi.fn<Kaos['stat']>().mockResolvedValue(REGULAR_FILE_STAT),
|
||||
readBytes: vi.fn<Kaos['readBytes']>().mockImplementation(async (_path, n) => {
|
||||
return n === undefined ? bytes : bytes.subarray(0, n);
|
||||
}),
|
||||
readLines,
|
||||
scanTextFile,
|
||||
readLineRange,
|
||||
} as unknown as Partial<Kaos>),
|
||||
PERMISSIVE_WORKSPACE,
|
||||
);
|
||||
|
||||
const result = await executeTool(tool, context({ path: '/tmp/range.txt', line_offset: 5, n_lines: 3 }));
|
||||
const output = toolContentString(result);
|
||||
|
||||
expect(output).toContain('5\tline 5');
|
||||
expect(output).toContain('7\tline 7');
|
||||
expect(output).not.toContain('8\tline 8');
|
||||
expect(readLineRange).toHaveBeenCalledWith('/tmp/range.txt', {
|
||||
startLine: 5,
|
||||
maxLines: 3,
|
||||
errors: 'strict',
|
||||
});
|
||||
expect(readLines).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('uses tail reader when available without consuming readLines', async () => {
|
||||
const content = Array.from({ length: 20 }, (_, i) => `line ${String(i + 1)}`).join('\n');
|
||||
const bytes = Buffer.from(content, 'utf8');
|
||||
const readLines = vi.fn<Kaos['readLines']>();
|
||||
const readTailLines = vi.fn(async function* readTailLines(): AsyncGenerator<string> {
|
||||
yield 'line 18\n';
|
||||
yield 'line 19\n';
|
||||
yield 'line 20';
|
||||
});
|
||||
const scanTextFile = vi.fn(async () => ({
|
||||
totalLines: 20,
|
||||
endsWithNewline: false,
|
||||
hasNul: false,
|
||||
lineEndingFlags: { hasCrLf: false, hasLf: true, hasLoneCr: false },
|
||||
}));
|
||||
const tool = new ReadTool(
|
||||
createFakeKaos({
|
||||
stat: vi.fn<Kaos['stat']>().mockResolvedValue(REGULAR_FILE_STAT),
|
||||
readBytes: vi.fn<Kaos['readBytes']>().mockImplementation(async (_path, n) => {
|
||||
return n === undefined ? bytes : bytes.subarray(0, n);
|
||||
}),
|
||||
readLines,
|
||||
scanTextFile,
|
||||
readTailLines,
|
||||
} as unknown as Partial<Kaos>),
|
||||
PERMISSIVE_WORKSPACE,
|
||||
);
|
||||
|
||||
const result = await executeTool(tool, context({ path: '/tmp/tail.txt', line_offset: -3 }));
|
||||
const output = toolContentString(result);
|
||||
|
||||
expect(output).toContain('18\tline 18');
|
||||
expect(output).toContain('20\tline 20');
|
||||
expect(readTailLines).toHaveBeenCalledWith('/tmp/tail.txt', { tailCount: 3, errors: 'strict' });
|
||||
expect(readLines).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('short-circuits on scan NUL before range read', async () => {
|
||||
const readLineRange = vi.fn();
|
||||
const tool = new ReadTool(
|
||||
createFakeKaos({
|
||||
stat: vi.fn<Kaos['stat']>().mockResolvedValue(REGULAR_FILE_STAT),
|
||||
readBytes: vi.fn<Kaos['readBytes']>().mockResolvedValue(Buffer.from('text')),
|
||||
scanTextFile: vi.fn(async () => ({
|
||||
totalLines: 1,
|
||||
endsWithNewline: false,
|
||||
hasNul: true,
|
||||
lineEndingFlags: { hasCrLf: false, hasLf: false, hasLoneCr: false },
|
||||
})),
|
||||
readLineRange,
|
||||
} as unknown as Partial<Kaos>),
|
||||
PERMISSIVE_WORKSPACE,
|
||||
);
|
||||
|
||||
const result = await executeTool(tool, context({ path: '/tmp/nul.txt' }));
|
||||
const output = toolContentString(result);
|
||||
|
||||
expect(result.isError).toBe(true);
|
||||
expect(output).toContain('is not readable as UTF-8 text');
|
||||
expect(readLineRange).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('caps default reads at MAX_LINES', async () => {
|
||||
const content = Array.from({ length: MAX_LINES + 1 }, (_, i) => `line ${String(i + 1)}`).join(
|
||||
'\n',
|
||||
|
|
|
|||
|
|
@ -136,6 +136,7 @@ export function decodeTextWithErrors(
|
|||
data: Buffer,
|
||||
encoding: BufferEncoding,
|
||||
errors: 'strict' | 'replace' | 'ignore' = 'strict',
|
||||
ignoreBOM: boolean = false,
|
||||
): string {
|
||||
// Map Node's BufferEncoding names to Web TextDecoder labels where the two
|
||||
// diverge. Only UTF-family encodings participate in the strict/replace/
|
||||
|
|
@ -163,7 +164,7 @@ export function decodeTextWithErrors(
|
|||
}
|
||||
|
||||
if (errors === 'strict') {
|
||||
return new TextDecoder(webLabel, { fatal: true }).decode(data);
|
||||
return new TextDecoder(webLabel, { fatal: true, ignoreBOM }).decode(data);
|
||||
}
|
||||
|
||||
// 'ignore' must skip invalid input bytes/code units, not delete every
|
||||
|
|
@ -174,7 +175,7 @@ export function decodeTextWithErrors(
|
|||
}
|
||||
|
||||
// 'replace' → substitute each invalid sequence with U+FFFD (default).
|
||||
return new TextDecoder(webLabel, { fatal: false }).decode(data);
|
||||
return new TextDecoder(webLabel, { fatal: false, ignoreBOM }).decode(data);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -22,6 +22,22 @@ import type { KaosProcess } from './process';
|
|||
import type { StatResult } from './types';
|
||||
|
||||
const isWindows: boolean = process.platform === 'win32';
|
||||
const READ_CHUNK_SIZE = 64 * 1024;
|
||||
|
||||
type TextDecodeErrors = 'strict' | 'replace' | 'ignore';
|
||||
|
||||
interface LineEndingFlags {
|
||||
hasCrLf: boolean;
|
||||
hasLf: boolean;
|
||||
hasLoneCr: boolean;
|
||||
}
|
||||
|
||||
interface TextFileScan {
|
||||
totalLines: number;
|
||||
endsWithNewline: boolean;
|
||||
hasNul: boolean;
|
||||
lineEndingFlags: LineEndingFlags;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the `(dev, ino)` cycle-detection key used by `_globWalk`'s
|
||||
|
|
@ -456,22 +472,173 @@ export class LocalKaos implements Kaos {
|
|||
|
||||
async *readLines(
|
||||
path: string,
|
||||
options?: { encoding?: BufferEncoding; errors?: 'strict' | 'replace' | 'ignore' },
|
||||
options?: { encoding?: BufferEncoding; errors?: TextDecodeErrors },
|
||||
): AsyncGenerator<string> {
|
||||
const resolved = this._resolvePath(path);
|
||||
const encoding = options?.encoding ?? 'utf-8';
|
||||
const errors = options?.errors ?? 'strict';
|
||||
const buf = await readFile(resolved);
|
||||
const content = decodeTextWithErrors(buf, encoding, errors);
|
||||
const lines = content.split('\n');
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (line === undefined) continue;
|
||||
if (i < lines.length - 1) {
|
||||
yield line + '\n';
|
||||
} else if (line !== '') {
|
||||
yield line;
|
||||
|
||||
if (!isUtf8Encoding(encoding)) {
|
||||
const content = decodeTextWithErrors(await readFile(resolved), encoding, errors);
|
||||
yield* splitLinesKeepingTerminator(content);
|
||||
return;
|
||||
}
|
||||
|
||||
yield* this._readUtf8Lines(resolved, errors);
|
||||
}
|
||||
|
||||
async scanTextFile(path: string): Promise<TextFileScan> {
|
||||
const resolved = this._resolvePath(path);
|
||||
const fh = await open(resolved, 'r');
|
||||
try {
|
||||
const buf = Buffer.alloc(READ_CHUNK_SIZE);
|
||||
const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false };
|
||||
const validator = createUtf8Validator();
|
||||
let totalLines = 0;
|
||||
let totalBytes = 0;
|
||||
let endsWithNewline = false;
|
||||
let hasNul = false;
|
||||
let prevWasCr = false;
|
||||
|
||||
while (true) {
|
||||
const { bytesRead } = await fh.read(buf, 0, buf.length, null);
|
||||
if (bytesRead === 0) break;
|
||||
const chunk = buf.subarray(0, bytesRead);
|
||||
validator.write(chunk);
|
||||
for (let i = 0; i < chunk.length; i += 1) {
|
||||
const byte = chunk[i];
|
||||
if (byte === undefined) continue;
|
||||
if (byte === 0) hasNul = true;
|
||||
if (byte === 0x0a) totalLines += 1;
|
||||
}
|
||||
prevWasCr = updateLineEndingFlagsFromBytes(flags, chunk, prevWasCr);
|
||||
totalBytes += bytesRead;
|
||||
endsWithNewline = chunk[bytesRead - 1] === 0x0a;
|
||||
}
|
||||
|
||||
if (prevWasCr) flags.hasLoneCr = true;
|
||||
validator.end();
|
||||
if (totalBytes > 0 && !endsWithNewline) totalLines += 1;
|
||||
return { totalLines, endsWithNewline, hasNul, lineEndingFlags: flags };
|
||||
} finally {
|
||||
await fh.close();
|
||||
}
|
||||
}
|
||||
|
||||
async *readLineRange(
|
||||
path: string,
|
||||
options: { startLine: number; maxLines: number; errors?: TextDecodeErrors },
|
||||
): AsyncGenerator<string> {
|
||||
const resolved = this._resolvePath(path);
|
||||
const errors = options.errors ?? 'strict';
|
||||
yield* this._readUtf8Lines(resolved, errors, {
|
||||
startLine: options.startLine,
|
||||
maxLines: options.maxLines,
|
||||
});
|
||||
}
|
||||
|
||||
async *readTailLines(
|
||||
path: string,
|
||||
options: { tailCount: number; errors?: TextDecodeErrors },
|
||||
): AsyncGenerator<string> {
|
||||
if (options.tailCount <= 0) return;
|
||||
const resolved = this._resolvePath(path);
|
||||
const errors = options.errors ?? 'strict';
|
||||
const fh = await open(resolved, 'r');
|
||||
try {
|
||||
const s = await fh.stat();
|
||||
if (s.size === 0) return;
|
||||
|
||||
let pos = s.size;
|
||||
let foundLf = 0;
|
||||
let startOffset = 0;
|
||||
let needLf = options.tailCount;
|
||||
let sawTailBlock = false;
|
||||
|
||||
while (pos > 0 && foundLf < needLf) {
|
||||
const readSize = Math.min(READ_CHUNK_SIZE, pos);
|
||||
pos -= readSize;
|
||||
const buf = Buffer.alloc(readSize);
|
||||
await fh.read(buf, 0, readSize, pos);
|
||||
if (!sawTailBlock) {
|
||||
sawTailBlock = true;
|
||||
const endsWithNewline = buf[readSize - 1] === 0x0a;
|
||||
needLf = endsWithNewline ? options.tailCount + 1 : options.tailCount;
|
||||
}
|
||||
for (let i = readSize - 1; i >= 0; i -= 1) {
|
||||
const byte = buf[i];
|
||||
if (byte !== 0x0a) continue;
|
||||
foundLf += 1;
|
||||
if (foundLf === needLf) {
|
||||
startOffset = pos + i + 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (foundLf < needLf) startOffset = 0;
|
||||
const data = await readRange(fh, startOffset, s.size - startOffset);
|
||||
const text = decodeTextWithErrors(data, 'utf-8', errors, startOffset !== 0);
|
||||
yield* splitLinesKeepingTerminator(text);
|
||||
} finally {
|
||||
await fh.close();
|
||||
}
|
||||
}
|
||||
|
||||
private async *_readUtf8Lines(
|
||||
resolved: string,
|
||||
errors: TextDecodeErrors,
|
||||
range?: { startLine?: number; maxLines?: number },
|
||||
): AsyncGenerator<string> {
|
||||
const startLine = range?.startLine ?? 1;
|
||||
const maxLines = range?.maxLines ?? Number.POSITIVE_INFINITY;
|
||||
const fh = await open(resolved, 'r');
|
||||
try {
|
||||
const buf = Buffer.alloc(READ_CHUNK_SIZE);
|
||||
let pending: Buffer[] = [];
|
||||
let pendingOffset = 0;
|
||||
let fileOffset = 0;
|
||||
let lineNo = 1;
|
||||
let yielded = 0;
|
||||
|
||||
while (true) {
|
||||
const { bytesRead } = await fh.read(buf, 0, buf.length, null);
|
||||
if (bytesRead === 0) break;
|
||||
const chunk = buf.subarray(0, bytesRead);
|
||||
let lineStart = 0;
|
||||
|
||||
for (let i = 0; i < chunk.length; i += 1) {
|
||||
const byte = chunk[i];
|
||||
if (byte !== 0x0a) continue;
|
||||
const piece = chunk.subarray(lineStart, i + 1);
|
||||
const lineOffset = pending.length === 0 ? fileOffset + lineStart : pendingOffset;
|
||||
const line = pending.length === 0 ? piece : Buffer.concat([...pending, piece]);
|
||||
if (lineNo >= startLine) {
|
||||
yield decodeTextWithErrors(line, 'utf-8', errors, lineOffset !== 0);
|
||||
yielded += 1;
|
||||
if (yielded >= maxLines) return;
|
||||
}
|
||||
pending = [];
|
||||
lineStart = i + 1;
|
||||
lineNo += 1;
|
||||
}
|
||||
|
||||
if (lineStart < chunk.length) {
|
||||
const tail = Buffer.from(chunk.subarray(lineStart));
|
||||
if (pending.length === 0) pendingOffset = fileOffset + lineStart;
|
||||
pending.push(tail);
|
||||
}
|
||||
fileOffset += bytesRead;
|
||||
}
|
||||
|
||||
if (pending.length > 0) {
|
||||
const line = Buffer.concat(pending);
|
||||
if (lineNo >= startLine) {
|
||||
yield decodeTextWithErrors(line, 'utf-8', errors, pendingOffset !== 0);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await fh.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -597,6 +764,118 @@ export class LocalKaos implements Kaos {
|
|||
}
|
||||
}
|
||||
|
||||
function isUtf8Encoding(encoding: BufferEncoding): boolean {
|
||||
return encoding === 'utf-8' || encoding === 'utf8';
|
||||
}
|
||||
|
||||
function* splitLinesKeepingTerminator(text: string): Generator<string> {
|
||||
if (text.length === 0) return;
|
||||
let start = 0;
|
||||
for (let i = 0; i < text.length; i += 1) {
|
||||
if (text.codePointAt(i) === 0x0a) {
|
||||
yield text.slice(start, i + 1);
|
||||
start = i + 1;
|
||||
}
|
||||
}
|
||||
if (start < text.length) {
|
||||
yield text.slice(start);
|
||||
}
|
||||
}
|
||||
|
||||
function updateLineEndingFlagsFromBytes(
|
||||
flags: LineEndingFlags,
|
||||
chunk: Buffer,
|
||||
prevWasCr: boolean,
|
||||
): boolean {
|
||||
for (let i = 0; i < chunk.length; i += 1) {
|
||||
const byte = chunk[i];
|
||||
if (byte === undefined) continue;
|
||||
if (byte === 0x0d) {
|
||||
if (prevWasCr) flags.hasLoneCr = true;
|
||||
prevWasCr = true;
|
||||
} else if (byte === 0x0a) {
|
||||
if (prevWasCr) {
|
||||
flags.hasCrLf = true;
|
||||
} else {
|
||||
flags.hasLf = true;
|
||||
}
|
||||
prevWasCr = false;
|
||||
} else {
|
||||
if (prevWasCr) flags.hasLoneCr = true;
|
||||
prevWasCr = false;
|
||||
}
|
||||
}
|
||||
return prevWasCr;
|
||||
}
|
||||
|
||||
function createUtf8Validator(): { write(chunk: Buffer): void; end(): void } {
|
||||
let needed = 0;
|
||||
let lower = 0x80;
|
||||
let upper = 0xbf;
|
||||
|
||||
const fail = (): never => {
|
||||
throw new TypeError('Invalid UTF-8 data');
|
||||
};
|
||||
|
||||
return {
|
||||
write(chunk: Buffer): void {
|
||||
for (let i = 0; i < chunk.length; i += 1) {
|
||||
const byte = chunk[i];
|
||||
if (byte === undefined) continue;
|
||||
if (needed === 0) {
|
||||
if (byte <= 0x7f) continue;
|
||||
if (byte >= 0xc2 && byte <= 0xdf) {
|
||||
needed = 1;
|
||||
} else if (byte === 0xe0) {
|
||||
needed = 2;
|
||||
lower = 0xa0;
|
||||
} else if (byte >= 0xe1 && byte <= 0xec) {
|
||||
needed = 2;
|
||||
} else if (byte === 0xed) {
|
||||
needed = 2;
|
||||
upper = 0x9f;
|
||||
} else if (byte >= 0xee && byte <= 0xef) {
|
||||
needed = 2;
|
||||
} else if (byte === 0xf0) {
|
||||
needed = 3;
|
||||
lower = 0x90;
|
||||
} else if (byte >= 0xf1 && byte <= 0xf3) {
|
||||
needed = 3;
|
||||
} else if (byte === 0xf4) {
|
||||
needed = 3;
|
||||
upper = 0x8f;
|
||||
} else {
|
||||
fail();
|
||||
}
|
||||
} else {
|
||||
if (byte < lower || byte > upper) fail();
|
||||
lower = 0x80;
|
||||
upper = 0xbf;
|
||||
needed -= 1;
|
||||
}
|
||||
}
|
||||
},
|
||||
end(): void {
|
||||
if (needed !== 0) fail();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function readRange(
|
||||
fh: Awaited<ReturnType<typeof open>>,
|
||||
start: number,
|
||||
length: number,
|
||||
): Promise<Buffer> {
|
||||
const data = Buffer.alloc(length);
|
||||
let offset = 0;
|
||||
while (offset < length) {
|
||||
const { bytesRead } = await fh.read(data, offset, length - offset, start + offset);
|
||||
if (bytesRead === 0) break;
|
||||
offset += bytesRead;
|
||||
}
|
||||
return offset === length ? data : data.subarray(0, offset);
|
||||
}
|
||||
|
||||
// Wait for a freshly spawned ChildProcess to either emit 'spawn' (success) or
|
||||
// 'error' (ENOENT / EACCES / etc.). Until this resolves, callers should not
|
||||
// assume the child is running — they may otherwise write to the stdin of a
|
||||
|
|
|
|||
|
|
@ -176,6 +176,164 @@ describe('LocalKaos', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('readLines streaming', () => {
|
||||
async function collectLines(path: string, options?: Parameters<LocalKaos['readLines']>[1]) {
|
||||
const lines: string[] = [];
|
||||
for await (const line of kaos.readLines(path, options)) {
|
||||
lines.push(line);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
it('preserves content exactly across representative line endings', async () => {
|
||||
const fixtures: Array<[string, string]> = [
|
||||
['multiline', 'line1\nline2\nline3\n'],
|
||||
['no trailing newline', 'line1\nline2'],
|
||||
['single line', 'only'],
|
||||
['single newline', '\n'],
|
||||
['empty', ''],
|
||||
['crlf', 'a\r\nb\r\n'],
|
||||
['lone cr', 'a\rB\n'],
|
||||
];
|
||||
for (const [name, content] of fixtures) {
|
||||
const filePath = join(tempDir, `${name}.txt`);
|
||||
await kaos.writeText(filePath, content);
|
||||
expect((await collectLines(filePath)).join('')).toBe(content);
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves multibyte characters and long single lines across chunk boundaries', async () => {
|
||||
const filePath = join(tempDir, 'boundary.txt');
|
||||
const content = `${'a'.repeat(65535)}😀\n${'x'.repeat(200000)}`;
|
||||
await kaos.writeText(filePath, content);
|
||||
await expect(collectLines(filePath)).resolves.toEqual([
|
||||
`${'a'.repeat(65535)}😀\n`,
|
||||
'x'.repeat(200000),
|
||||
]);
|
||||
});
|
||||
|
||||
it('preserves U+FEFF at the start of a non-first line', async () => {
|
||||
const filePath = join(tempDir, 'bom-line.txt');
|
||||
const content = 'a\n\uFEFFb\n';
|
||||
await kaos.writeText(filePath, content);
|
||||
await expect(collectLines(filePath)).resolves.toEqual(['a\n', '\uFEFFb\n']);
|
||||
});
|
||||
|
||||
it('keeps utf16le and hex on the decode-then-split path', async () => {
|
||||
const utf16Path = join(tempDir, 'utf16le.txt');
|
||||
await kaos.writeBytes(utf16Path, Buffer.from('a\n\u0A41\n', 'utf16le'));
|
||||
await expect(collectLines(utf16Path, { encoding: 'utf16le' })).resolves.toEqual([
|
||||
'a\n',
|
||||
'ੁ\n',
|
||||
]);
|
||||
|
||||
const hexPath = join(tempDir, 'hex.txt');
|
||||
await kaos.writeBytes(hexPath, Buffer.from('a\nb'));
|
||||
await expect(collectLines(hexPath, { encoding: 'hex' })).resolves.toEqual(['610a62']);
|
||||
});
|
||||
|
||||
it('throws lazily when strict UTF-8 errors appear after the first line', async () => {
|
||||
const filePath = join(tempDir, 'invalid-after-first-line.txt');
|
||||
await kaos.writeBytes(filePath, Buffer.concat([Buffer.from('ok\n', 'utf-8'), Buffer.from([0xff])]));
|
||||
const gen = kaos.readLines(filePath);
|
||||
await expect(gen.next()).resolves.toMatchObject({ value: 'ok\n', done: false });
|
||||
await expect(gen.next()).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('scanTextFile', () => {
|
||||
it('counts lines and classifies line endings', async () => {
|
||||
const lf = join(tempDir, 'lf.txt');
|
||||
await kaos.writeText(lf, 'a\nb');
|
||||
await expect(kaos.scanTextFile(lf)).resolves.toMatchObject({
|
||||
totalLines: 2,
|
||||
endsWithNewline: false,
|
||||
hasNul: false,
|
||||
lineEndingFlags: { hasCrLf: false, hasLf: true, hasLoneCr: false },
|
||||
});
|
||||
|
||||
const crlf = join(tempDir, 'crlf.txt');
|
||||
await kaos.writeText(crlf, 'a\r\nb\r\n');
|
||||
await expect(kaos.scanTextFile(crlf)).resolves.toMatchObject({
|
||||
totalLines: 2,
|
||||
endsWithNewline: true,
|
||||
lineEndingFlags: { hasCrLf: true, hasLf: false, hasLoneCr: false },
|
||||
});
|
||||
|
||||
const loneCr = join(tempDir, 'lone-cr.txt');
|
||||
await kaos.writeText(loneCr, 'a\rB\n');
|
||||
await expect(kaos.scanTextFile(loneCr)).resolves.toMatchObject({
|
||||
totalLines: 1,
|
||||
lineEndingFlags: { hasCrLf: false, hasLf: true, hasLoneCr: true },
|
||||
});
|
||||
});
|
||||
|
||||
it('detects NUL and invalid UTF-8', async () => {
|
||||
const nul = join(tempDir, 'nul.txt');
|
||||
await kaos.writeBytes(nul, Buffer.from('a\u0000b\n', 'utf-8'));
|
||||
await expect(kaos.scanTextFile(nul)).resolves.toMatchObject({ hasNul: true });
|
||||
|
||||
const invalid = join(tempDir, 'invalid.txt');
|
||||
await kaos.writeBytes(invalid, Buffer.from([0xff]));
|
||||
await expect(kaos.scanTextFile(invalid)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('readLineRange', () => {
|
||||
async function collectRange(path: string, startLine: number, maxLines: number) {
|
||||
const lines: string[] = [];
|
||||
for await (const line of kaos.readLineRange(path, { startLine, maxLines })) {
|
||||
lines.push(line);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
it('reads only the requested line window', async () => {
|
||||
const filePath = join(tempDir, 'range.txt');
|
||||
await kaos.writeText(filePath, 'a\nb\nc\nd\n');
|
||||
await expect(collectRange(filePath, 2, 2)).resolves.toEqual(['b\n', 'c\n']);
|
||||
await expect(collectRange(filePath, 5, 2)).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it('preserves U+FEFF at the start of a ranged non-first line', async () => {
|
||||
const filePath = join(tempDir, 'range-bom.txt');
|
||||
await kaos.writeText(filePath, 'a\n\uFEFFb\n');
|
||||
await expect(collectRange(filePath, 2, 1)).resolves.toEqual(['\uFEFFb\n']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readTailLines', () => {
|
||||
async function collectTail(path: string, tailCount: number) {
|
||||
const lines: string[] = [];
|
||||
for await (const line of kaos.readTailLines(path, { tailCount })) {
|
||||
lines.push(line);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
it('reads last lines with and without trailing newline', async () => {
|
||||
const trailing = join(tempDir, 'tail-trailing.txt');
|
||||
await kaos.writeText(trailing, 'a\nb\nc\n');
|
||||
await expect(collectTail(trailing, 2)).resolves.toEqual(['b\n', 'c\n']);
|
||||
|
||||
const noTrailing = join(tempDir, 'tail-no-trailing.txt');
|
||||
await kaos.writeText(noTrailing, 'a\nb\nc');
|
||||
await expect(collectTail(noTrailing, 2)).resolves.toEqual(['b\n', 'c']);
|
||||
});
|
||||
|
||||
it('returns the whole file when tailCount exceeds line count', async () => {
|
||||
const filePath = join(tempDir, 'tail-short.txt');
|
||||
await kaos.writeText(filePath, 'a\nb\n');
|
||||
await expect(collectTail(filePath, 5)).resolves.toEqual(['a\n', 'b\n']);
|
||||
});
|
||||
|
||||
it('preserves CRLF and U+FEFF in tail lines', async () => {
|
||||
const filePath = join(tempDir, 'tail-crlf-bom.txt');
|
||||
await kaos.writeText(filePath, 'a\r\n\uFEFFb\r\n');
|
||||
await expect(collectTail(filePath, 1)).resolves.toEqual(['\uFEFFb\r\n']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readText errors parameter (Python compat)', () => {
|
||||
// A file with a valid UTF-8 prefix "中", an invalid standalone byte 0xff,
|
||||
// and a valid UTF-8 suffix "文". Under strict decoding this throws.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue