From 823c47bcc2419481d607b742041ae43f7d683dde Mon Sep 17 00:00:00 2001 From: Vanzeren <53075619+Vanzeren@users.noreply.github.com> Date: Mon, 6 Jul 2026 22:24:14 +0800 Subject: [PATCH] fix(backend): stop classifying UTF-16 markdown files as binary (#3966) * fix: detect utf16 markdown workspace diffs * fix: tighten binary detection in workspace diff scanner * fix: decode utf-8-sig before utf-8 to strip bom from diff content --- .../deerflow/workspace_changes/diff.py | 2 +- .../deerflow/workspace_changes/scanner.py | 57 ++++++++--- backend/tests/test_workspace_changes.py | 94 ++++++++++++++++++- 3 files changed, 137 insertions(+), 16 deletions(-) diff --git a/backend/packages/harness/deerflow/workspace_changes/diff.py b/backend/packages/harness/deerflow/workspace_changes/diff.py index 844a65a42..d0130e34c 100644 --- a/backend/packages/harness/deerflow/workspace_changes/diff.py +++ b/backend/packages/harness/deerflow/workspace_changes/diff.py @@ -175,7 +175,7 @@ def _snapshot_text(file: FileSnapshot | None) -> str | None: try: with open(file.text_path, encoding="utf-8") as cached: return cached.read() - except (OSError, UnicodeDecodeError): + except OSError: return None return None diff --git a/backend/packages/harness/deerflow/workspace_changes/scanner.py b/backend/packages/harness/deerflow/workspace_changes/scanner.py index d6461ae65..912c1e2fa 100644 --- a/backend/packages/harness/deerflow/workspace_changes/scanner.py +++ b/backend/packages/harness/deerflow/workspace_changes/scanner.py @@ -3,7 +3,7 @@ from __future__ import annotations import fnmatch import hashlib import os -import shutil +from codecs import BOM_UTF16_BE, BOM_UTF16_LE, getincrementaldecoder from pathlib import Path from .types import ( @@ -74,6 +74,7 @@ SENSITIVE_PATH_PATTERNS = ( ) SAMPLE_BYTES = 4096 +_UTF16_BOMS = (BOM_UTF16_LE, BOM_UTF16_BE) def is_sensitive_workspace_path(path: str) -> bool: @@ -193,16 +194,19 @@ def _snapshot_file( reason = "large" elif not should_include_text: text = None - elif text_cache_dir is not None: - text_path = str(_cache_text_file(host_file, virtual_path, text_cache_dir)) else: try: - text = host_file.read_text(encoding="utf-8") - except UnicodeDecodeError: - binary = True - reason = "binary" + raw = host_file.read_bytes() except OSError: return None + decoded = _decode_text_bytes(raw) + if decoded is None: + binary = True + reason = "binary" + elif text_cache_dir is not None: + text_path = str(_cache_text_file(decoded, virtual_path, text_cache_dir)) + else: + text = decoded return FileSnapshot( path=virtual_path, @@ -218,10 +222,10 @@ def _snapshot_file( ) -def _cache_text_file(source: Path, virtual_path: str, cache_dir: Path) -> Path: +def _cache_text_file(text: str, virtual_path: str, cache_dir: Path) -> Path: cache_name = hashlib.sha256(virtual_path.encode("utf-8")).hexdigest() target = cache_dir / cache_name - shutil.copyfile(source, target) + target.write_text(text, encoding="utf-8") return target @@ -238,11 +242,36 @@ def _sha256_file(path: Path) -> str: return digest.hexdigest() +def _decode_text_bytes(data: bytes) -> str | None: + for encoding in ("utf-8-sig", "utf-8"): + try: + return data.decode(encoding) + except UnicodeDecodeError: + continue + + if data.startswith(_UTF16_BOMS): + try: + return data.decode("utf-16") + except UnicodeDecodeError: + return None + + return None + + +def _sample_decodes_as_text(sample: bytes, encoding: str) -> bool: + try: + decoder = getincrementaldecoder(encoding)() + decoder.decode(sample, final=False) + except UnicodeDecodeError: + return False + return True + + def _looks_binary(sample: bytes) -> bool: + if sample.startswith(_UTF16_BOMS) and _sample_decodes_as_text(sample, "utf-16"): + return False if b"\x00" in sample: return True - try: - sample.decode("utf-8") - except UnicodeDecodeError: - return True - return False + if _sample_decodes_as_text(sample, "utf-8"): + return False + return True diff --git a/backend/tests/test_workspace_changes.py b/backend/tests/test_workspace_changes.py index 95519e66a..3a308c716 100644 --- a/backend/tests/test_workspace_changes.py +++ b/backend/tests/test_workspace_changes.py @@ -18,7 +18,7 @@ from deerflow.workspace_changes import ( scan_workspace_roots, ) from deerflow.workspace_changes.api import get_workspace_changes_response -from deerflow.workspace_changes.scanner import is_sensitive_workspace_path +from deerflow.workspace_changes.scanner import SAMPLE_BYTES, is_sensitive_workspace_path def _roots(tmp_path): @@ -70,6 +70,98 @@ def test_compare_snapshots_reports_text_file_changes(tmp_path): assert changes["/mnt/user-data/workspace/old.txt"].status == "deleted" +def test_compare_snapshots_treats_utf16_markdown_as_text(tmp_path): + roots = _roots(tmp_path) + workspace = roots[0].host_path + + before = scan_workspace_roots(roots) + (workspace / "guide.md").write_bytes("# 标题\n\nhello\n".encode("utf-16")) + after = scan_workspace_roots(roots) + + result = compare_snapshots(before, after) + + change = result.files[0] + assert change.path == "/mnt/user-data/workspace/guide.md" + assert change.binary is False + assert change.diff_unavailable_reason is None + assert "+# 标题" in change.diff + + +def test_compare_snapshots_reads_cached_utf16_markdown_baseline(tmp_path): + roots = _roots(tmp_path) + workspace = roots[0].host_path + cache_dir = tmp_path / "cache" + + (workspace / "guide.md").write_bytes("# 标题\n\nhello\n".encode("utf-16")) + before = scan_workspace_roots(roots, text_cache_dir=cache_dir) + (workspace / "guide.md").write_bytes("# 标题\n\nupdated\n".encode("utf-16")) + after = scan_workspace_roots(roots) + + result = compare_snapshots(before, after) + + change = result.files[0] + assert change.binary is False + assert change.diff_unavailable_reason is None + assert "-hello" in change.diff + assert "+updated" in change.diff + + +def test_compare_snapshots_treats_utf8_markdown_crossing_sample_boundary_as_text( + tmp_path, +): + roots = _roots(tmp_path) + workspace = roots[0].host_path + + before = scan_workspace_roots(roots) + content = ("a" * (SAMPLE_BYTES - 1)) + "你" + "\nrest\n" + (workspace / "guide.md").write_text(content, encoding="utf-8") + after = scan_workspace_roots(roots) + + result = compare_snapshots(before, after) + + change = result.files[0] + assert change.path == "/mnt/user-data/workspace/guide.md" + assert change.binary is False + assert "你" in change.diff + assert "+rest" in change.diff + + +def test_compare_snapshots_strips_utf8_bom(tmp_path): + roots = _roots(tmp_path) + workspace = roots[0].host_path + + before = scan_workspace_roots(roots) + content = "# 标题\n\nhello\n".encode() + (workspace / "guide.md").write_bytes(b"\xef\xbb\xbf" + content) + after = scan_workspace_roots(roots) + + result = compare_snapshots(before, after) + + change = result.files[0] + assert change.path == "/mnt/user-data/workspace/guide.md" + assert change.binary is False + assert change.diff_unavailable_reason is None + assert "\ufeff" not in change.diff + assert "+# 标题" in change.diff + + +def test_compare_snapshots_keeps_nul_bytes_classified_as_binary(tmp_path): + roots = _roots(tmp_path) + workspace = roots[0].host_path + + before = scan_workspace_roots(roots) + (workspace / "guide.md").write_bytes(b"\x00\x01\x02binary\n") + after = scan_workspace_roots(roots) + + result = compare_snapshots(before, after) + + change = result.files[0] + assert change.path == "/mnt/user-data/workspace/guide.md" + assert change.binary is True + assert change.diff == "" + assert change.diff_unavailable_reason == "binary" + + def test_count_diff_lines_ignores_only_real_headers(): from deerflow.workspace_changes.diff import _count_diff_lines