mirror of
https://github.com/zed-industries/zed.git
synced 2026-08-15 20:14:37 +00:00
Unify non-Unicode file detection code (#62581)
Closes https://github.com/zed-industries/zed/issues/62464 Closes https://github.com/zed-industries/zed/issues/62212 As a bonus, fixes the project search not working in BOM'd UTF-16 files. Release Notes: - Fixed project search not working in some non-Unicode files
This commit is contained in:
Kirill Bulatov
GitHub
parent
b41505358f
commit
4efba7161f
3 changed files with 163 additions and 21 deletions
|
|
@ -1,7 +1,7 @@
|
|||
use std::{
|
||||
cell::LazyCell,
|
||||
collections::BTreeSet,
|
||||
io::{BufRead, BufReader},
|
||||
io::{BufRead, BufReader, Cursor, ErrorKind, Read},
|
||||
ops::Range,
|
||||
path::{Path, PathBuf},
|
||||
pin::pin,
|
||||
|
|
@ -21,8 +21,12 @@ use parking_lot::Mutex;
|
|||
use postage::oneshot;
|
||||
use rpc::{AnyProtoClient, proto};
|
||||
|
||||
use language::ByteContent;
|
||||
use util::{ResultExt, maybe, paths::compare_rel_paths, rel_path::RelPath};
|
||||
use worktree::{Entry, ProjectEntryId, Snapshot, Worktree, WorktreeSettings};
|
||||
use worktree::{
|
||||
Entry, ProjectEntryId, Snapshot, Worktree, WorktreeSettings, decode_byte_header,
|
||||
decode_file_text,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
Project, ProjectItem, ProjectPath, RemotelyCreatedModels,
|
||||
|
|
@ -779,32 +783,42 @@ impl RequestHandler<'_> {
|
|||
async fn handle_find_first_match(&self, mut entry: MatchingEntry) {
|
||||
async move {
|
||||
let abs_path = entry.worktree_root.join(entry.path.path.as_std_path());
|
||||
let Some(file) = self
|
||||
let fs = self
|
||||
.fs
|
||||
.context("Trying to query filesystem in remote project search")?
|
||||
.open_sync(&abs_path)
|
||||
.await
|
||||
.log_err()
|
||||
else {
|
||||
.context("Trying to query filesystem in remote project search")?;
|
||||
let Some(file) = fs.open_sync(&abs_path).await.log_err() else {
|
||||
return anyhow::Ok(());
|
||||
};
|
||||
|
||||
let mut file = BufReader::new(file);
|
||||
let file_start = file.fill_buf()?;
|
||||
|
||||
if let Err(Some(starting_position)) =
|
||||
std::str::from_utf8(file_start).map_err(|e| e.error_len())
|
||||
{
|
||||
// Before attempting to match the file content, throw away files that have invalid UTF-8 sequences early on;
|
||||
// That way we can still match files in a streaming fashion without having look at "obviously binary" files.
|
||||
log::debug!(
|
||||
"Invalid UTF-8 sequence in file {abs_path:?} \
|
||||
at byte position {starting_position}"
|
||||
);
|
||||
let (bom_encoding, byte_content) = decode_byte_header(file_start);
|
||||
if byte_content == ByteContent::Binary {
|
||||
log::debug!("Skipping binary file {abs_path:?}");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(line_hint) = self.query.detect(file).await.ok().flatten() {
|
||||
let is_plain_utf8 = bom_encoding.is_none()
|
||||
&& byte_content == ByteContent::Unknown
|
||||
&& is_utf8_prefix(file_start);
|
||||
|
||||
let line_hint = if is_plain_utf8 {
|
||||
match self.query.detect(file).await {
|
||||
Ok(line_hint) => line_hint,
|
||||
Err(error)
|
||||
if error
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.is_some_and(|error| error.kind() == ErrorKind::InvalidData) =>
|
||||
{
|
||||
self.detect_in_decoded_file(fs, &abs_path).await?
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
} else {
|
||||
self.detect_in_decoded_file(fs, &abs_path).await?
|
||||
};
|
||||
|
||||
if let Some(line_hint) = line_hint {
|
||||
// Yes, we should scan the whole file.
|
||||
entry.should_scan_tx.send((entry.path, line_hint)).await?;
|
||||
}
|
||||
|
|
@ -814,6 +828,16 @@ impl RequestHandler<'_> {
|
|||
.ok();
|
||||
}
|
||||
|
||||
async fn detect_in_decoded_file(
|
||||
&self,
|
||||
fs: &dyn Fs,
|
||||
abs_path: &Path,
|
||||
) -> anyhow::Result<Option<MatchPositionHint>> {
|
||||
let (text, _encoding, _has_bom) = decode_file_text(fs, abs_path).await?;
|
||||
let reader: Box<dyn Read + Send + Sync> = Box::new(Cursor::new(text.into_bytes()));
|
||||
self.query.detect(BufReader::new(reader)).await
|
||||
}
|
||||
|
||||
async fn handle_scan_path(&self, req: InputPath) {
|
||||
_ = maybe!(async move {
|
||||
let InputPath {
|
||||
|
|
@ -870,6 +894,13 @@ impl RequestHandler<'_> {
|
|||
}
|
||||
}
|
||||
|
||||
fn is_utf8_prefix(bytes: &[u8]) -> bool {
|
||||
match std::str::from_utf8(bytes) {
|
||||
Ok(_) => true,
|
||||
Err(error) => error.error_len().is_none(),
|
||||
}
|
||||
}
|
||||
|
||||
struct InputPath {
|
||||
entry: Entry,
|
||||
snapshot: Snapshot,
|
||||
|
|
|
|||
|
|
@ -9235,6 +9235,117 @@ async fn test_search_with_unicode(cx: &mut gpui::TestAppContext) {
|
|||
);
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
async fn test_search_in_unopened_non_utf8_files(cx: &mut gpui::TestAppContext) {
|
||||
init_test(cx);
|
||||
|
||||
let text = "// 你好世界 hello\n// 这是一个中文注释，包含很多汉字，用来帮助编码检测器正确判断文件编码。\n// 编码检测需要足够多的中文内容才能可靠工作。\n";
|
||||
|
||||
let gb2312 = encoding_rs::Encoding::for_label(b"gb2312").unwrap();
|
||||
assert_eq!(gb2312, encoding_rs::GBK);
|
||||
let (gbk_bytes, _, had_errors) = gb2312.encode(text);
|
||||
assert!(!had_errors);
|
||||
|
||||
let korean_text = "// 안녕하세요 세계 hello\n// 이것은 한국어 주석입니다. 인코딩 감지기가 파일 인코딩을 올바르게 판단할 수 있도록 충분히 많은 한글 내용을 담고 있습니다.\n// 인코딩 감지는 충분한 한국어 내용이 있어야 안정적으로 작동합니다.\n";
|
||||
let (euc_kr_bytes, _, had_errors) = encoding_rs::EUC_KR.encode(korean_text);
|
||||
assert!(!had_errors);
|
||||
|
||||
let mut utf16_bytes = vec![0xFF, 0xFE];
|
||||
utf16_bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));
|
||||
|
||||
let mut binary_bytes = b"\x89PNG\r\n\x1a\n".to_vec();
|
||||
binary_bytes.extend_from_slice(text.as_bytes());
|
||||
|
||||
let fs = FakeFs::new(cx.executor());
|
||||
fs.insert_tree(
|
||||
path!("/dir"),
|
||||
json!({
|
||||
"utf8.rs": text,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
fs.insert_file(path!("/dir/gbk.rs"), gbk_bytes.into_owned())
|
||||
.await;
|
||||
fs.insert_file(path!("/dir/euc_kr.rs"), euc_kr_bytes.into_owned())
|
||||
.await;
|
||||
fs.insert_file(path!("/dir/utf16.rs"), utf16_bytes).await;
|
||||
fs.insert_file(path!("/dir/binary.dat"), binary_bytes).await;
|
||||
|
||||
let project = Project::test(fs.clone(), [path!("/dir").as_ref()], cx).await;
|
||||
|
||||
assert_eq!(
|
||||
search(
|
||||
&project,
|
||||
SearchQuery::text(
|
||||
"世界",
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
PathMatcher::default(),
|
||||
PathMatcher::default(),
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
cx
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
HashMap::from_iter([
|
||||
(path!("dir/utf8.rs").to_string(), vec![9..15]),
|
||||
(path!("dir/gbk.rs").to_string(), vec![9..15]),
|
||||
(path!("dir/utf16.rs").to_string(), vec![9..15]),
|
||||
])
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
search(
|
||||
&project,
|
||||
SearchQuery::text(
|
||||
"HELLO",
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
PathMatcher::default(),
|
||||
PathMatcher::default(),
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
cx
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
HashMap::from_iter([
|
||||
(path!("dir/utf8.rs").to_string(), vec![16..21]),
|
||||
(path!("dir/gbk.rs").to_string(), vec![16..21]),
|
||||
(path!("dir/euc_kr.rs").to_string(), vec![26..31]),
|
||||
(path!("dir/utf16.rs").to_string(), vec![16..21]),
|
||||
])
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
search(
|
||||
&project,
|
||||
SearchQuery::text(
|
||||
"세계",
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
PathMatcher::default(),
|
||||
PathMatcher::default(),
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
cx
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
HashMap::from_iter([(path!("dir/euc_kr.rs").to_string(), vec![19..25])])
|
||||
);
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
async fn test_create_entry(cx: &mut gpui::TestAppContext) {
|
||||
init_test(cx);
|
||||
|
|
|
|||
|
|
@ -7143,7 +7143,7 @@ impl fs::Watcher for NullWatcher {
|
|||
}
|
||||
}
|
||||
|
||||
async fn decode_file_text(
|
||||
pub async fn decode_file_text(
|
||||
fs: &dyn Fs,
|
||||
abs_path: &Path,
|
||||
) -> Result<(String, &'static Encoding, bool)> {
|
||||
|
|
@ -7193,7 +7193,7 @@ async fn decode_file_text(
|
|||
decode_byte_full(content, bom_encoding, byte_content)
|
||||
}
|
||||
|
||||
fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {
|
||||
pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {
|
||||
if let Some((encoding, _bom_len)) = Encoding::for_bom(prefix) {
|
||||
return (Some(encoding), ByteContent::Unknown);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue