From 4efba7161f848ab87be080f91bf28b06a7927c2c Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Thu, 13 Aug 2026 16:49:32 +0000 Subject: [PATCH] Unify non-Unicode file detection code (#62581) Closes https://github.com/zed-industries/zed/issues/62464 Closes https://github.com/zed-industries/zed/issues/62212 As a bonus, fixes the project search not working in BOM'd UTF-16 files. Release Notes: - Fixed project search not working in some non-Unicode files --- crates/project/src/project_search.rs | 69 ++++++++--- .../tests/integration/project_tests.rs | 111 ++++++++++++++++++ crates/worktree/src/worktree.rs | 4 +- 3 files changed, 163 insertions(+), 21 deletions(-) diff --git a/crates/project/src/project_search.rs b/crates/project/src/project_search.rs index 8b8bdac3010..438b2201880 100644 --- a/crates/project/src/project_search.rs +++ b/crates/project/src/project_search.rs @@ -1,7 +1,7 @@ use std::{ cell::LazyCell, collections::BTreeSet, - io::{BufRead, BufReader}, + io::{BufRead, BufReader, Cursor, ErrorKind, Read}, ops::Range, path::{Path, PathBuf}, pin::pin, @@ -21,8 +21,12 @@ use parking_lot::Mutex; use postage::oneshot; use rpc::{AnyProtoClient, proto}; +use language::ByteContent; use util::{ResultExt, maybe, paths::compare_rel_paths, rel_path::RelPath}; -use worktree::{Entry, ProjectEntryId, Snapshot, Worktree, WorktreeSettings}; +use worktree::{ + Entry, ProjectEntryId, Snapshot, Worktree, WorktreeSettings, decode_byte_header, + decode_file_text, +}; use crate::{ Project, ProjectItem, ProjectPath, RemotelyCreatedModels, @@ -779,32 +783,42 @@ impl RequestHandler<'_> { async fn handle_find_first_match(&self, mut entry: MatchingEntry) { async move { let abs_path = entry.worktree_root.join(entry.path.path.as_std_path()); - let Some(file) = self + let fs = self .fs - .context("Trying to query filesystem in remote project search")? - .open_sync(&abs_path) - .await - .log_err() - else { + .context("Trying to query filesystem in remote project search")?; + let Some(file) = fs.open_sync(&abs_path).await.log_err() else { return anyhow::Ok(()); }; let mut file = BufReader::new(file); let file_start = file.fill_buf()?; - - if let Err(Some(starting_position)) = - std::str::from_utf8(file_start).map_err(|e| e.error_len()) - { - // Before attempting to match the file content, throw away files that have invalid UTF-8 sequences early on; - // That way we can still match files in a streaming fashion without having look at "obviously binary" files. - log::debug!( - "Invalid UTF-8 sequence in file {abs_path:?} \ - at byte position {starting_position}" - ); + let (bom_encoding, byte_content) = decode_byte_header(file_start); + if byte_content == ByteContent::Binary { + log::debug!("Skipping binary file {abs_path:?}"); return Ok(()); } - if let Some(line_hint) = self.query.detect(file).await.ok().flatten() { + let is_plain_utf8 = bom_encoding.is_none() + && byte_content == ByteContent::Unknown + && is_utf8_prefix(file_start); + + let line_hint = if is_plain_utf8 { + match self.query.detect(file).await { + Ok(line_hint) => line_hint, + Err(error) + if error + .downcast_ref::() + .is_some_and(|error| error.kind() == ErrorKind::InvalidData) => + { + self.detect_in_decoded_file(fs, &abs_path).await? + } + Err(error) => return Err(error), + } + } else { + self.detect_in_decoded_file(fs, &abs_path).await? + }; + + if let Some(line_hint) = line_hint { // Yes, we should scan the whole file. entry.should_scan_tx.send((entry.path, line_hint)).await?; } @@ -814,6 +828,16 @@ impl RequestHandler<'_> { .ok(); } + async fn detect_in_decoded_file( + &self, + fs: &dyn Fs, + abs_path: &Path, + ) -> anyhow::Result> { + let (text, _encoding, _has_bom) = decode_file_text(fs, abs_path).await?; + let reader: Box = Box::new(Cursor::new(text.into_bytes())); + self.query.detect(BufReader::new(reader)).await + } + async fn handle_scan_path(&self, req: InputPath) { _ = maybe!(async move { let InputPath { @@ -870,6 +894,13 @@ impl RequestHandler<'_> { } } +fn is_utf8_prefix(bytes: &[u8]) -> bool { + match std::str::from_utf8(bytes) { + Ok(_) => true, + Err(error) => error.error_len().is_none(), + } +} + struct InputPath { entry: Entry, snapshot: Snapshot, diff --git a/crates/project/tests/integration/project_tests.rs b/crates/project/tests/integration/project_tests.rs index fa168b2fa4b..7734ed38166 100644 --- a/crates/project/tests/integration/project_tests.rs +++ b/crates/project/tests/integration/project_tests.rs @@ -9235,6 +9235,117 @@ async fn test_search_with_unicode(cx: &mut gpui::TestAppContext) { ); } +#[gpui::test] +async fn test_search_in_unopened_non_utf8_files(cx: &mut gpui::TestAppContext) { + init_test(cx); + + let text = "// 你好世界 hello\n// 这是一个中文注释，包含很多汉字，用来帮助编码检测器正确判断文件编码。\n// 编码检测需要足够多的中文内容才能可靠工作。\n"; + + let gb2312 = encoding_rs::Encoding::for_label(b"gb2312").unwrap(); + assert_eq!(gb2312, encoding_rs::GBK); + let (gbk_bytes, _, had_errors) = gb2312.encode(text); + assert!(!had_errors); + + let korean_text = "// 안녕하세요 세계 hello\n// 이것은 한국어 주석입니다. 인코딩 감지기가 파일 인코딩을 올바르게 판단할 수 있도록 충분히 많은 한글 내용을 담고 있습니다.\n// 인코딩 감지는 충분한 한국어 내용이 있어야 안정적으로 작동합니다.\n"; + let (euc_kr_bytes, _, had_errors) = encoding_rs::EUC_KR.encode(korean_text); + assert!(!had_errors); + + let mut utf16_bytes = vec![0xFF, 0xFE]; + utf16_bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes())); + + let mut binary_bytes = b"\x89PNG\r\n\x1a\n".to_vec(); + binary_bytes.extend_from_slice(text.as_bytes()); + + let fs = FakeFs::new(cx.executor()); + fs.insert_tree( + path!("/dir"), + json!({ + "utf8.rs": text, + }), + ) + .await; + fs.insert_file(path!("/dir/gbk.rs"), gbk_bytes.into_owned()) + .await; + fs.insert_file(path!("/dir/euc_kr.rs"), euc_kr_bytes.into_owned()) + .await; + fs.insert_file(path!("/dir/utf16.rs"), utf16_bytes).await; + fs.insert_file(path!("/dir/binary.dat"), binary_bytes).await; + + let project = Project::test(fs.clone(), [path!("/dir").as_ref()], cx).await; + + assert_eq!( + search( + &project, + SearchQuery::text( + "世界", + false, + true, + false, + PathMatcher::default(), + PathMatcher::default(), + false, + None, + ) + .unwrap(), + cx + ) + .await + .unwrap(), + HashMap::from_iter([ + (path!("dir/utf8.rs").to_string(), vec![9..15]), + (path!("dir/gbk.rs").to_string(), vec![9..15]), + (path!("dir/utf16.rs").to_string(), vec![9..15]), + ]) + ); + + assert_eq!( + search( + &project, + SearchQuery::text( + "HELLO", + false, + false, + false, + PathMatcher::default(), + PathMatcher::default(), + false, + None, + ) + .unwrap(), + cx + ) + .await + .unwrap(), + HashMap::from_iter([ + (path!("dir/utf8.rs").to_string(), vec![16..21]), + (path!("dir/gbk.rs").to_string(), vec![16..21]), + (path!("dir/euc_kr.rs").to_string(), vec![26..31]), + (path!("dir/utf16.rs").to_string(), vec![16..21]), + ]) + ); + + assert_eq!( + search( + &project, + SearchQuery::text( + "세계", + false, + true, + false, + PathMatcher::default(), + PathMatcher::default(), + false, + None, + ) + .unwrap(), + cx + ) + .await + .unwrap(), + HashMap::from_iter([(path!("dir/euc_kr.rs").to_string(), vec![19..25])]) + ); +} + #[gpui::test] async fn test_create_entry(cx: &mut gpui::TestAppContext) { init_test(cx); diff --git a/crates/worktree/src/worktree.rs b/crates/worktree/src/worktree.rs index 123debb3483..701f5e869bd 100644 --- a/crates/worktree/src/worktree.rs +++ b/crates/worktree/src/worktree.rs @@ -7143,7 +7143,7 @@ impl fs::Watcher for NullWatcher { } } -async fn decode_file_text( +pub async fn decode_file_text( fs: &dyn Fs, abs_path: &Path, ) -> Result<(String, &'static Encoding, bool)> { @@ -7193,7 +7193,7 @@ async fn decode_file_text( decode_byte_full(content, bom_encoding, byte_content) } -fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) { +pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) { if let Some((encoding, _bom_len)) = Encoding::for_bom(prefix) { return (Some(encoding), ByteContent::Unknown); }