diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs index 95a58b0cf95..e633ecc21b3 100644 --- a/crates/editor/src/display_map.rs +++ b/crates/editor/src/display_map.rs @@ -2110,7 +2110,7 @@ impl DisplaySnapshot { }); chars.collect::().graphemes(true).next().map(|s| { if let Some(invisible) = s.chars().next().filter(|&c| is_invisible(c)) { - replacement(invisible).unwrap_or(s).to_owned().into() + replacement(invisible).map_or_else(|| s.to_owned().into(), SharedString::from) } else if s == "\n" { " ".into() } else { diff --git a/crates/editor/src/display_map/invisibles.rs b/crates/editor/src/display_map/invisibles.rs index 90bd54ab280..20f112512f0 100644 --- a/crates/editor/src/display_map/invisibles.rs +++ b/crates/editor/src/display_map/invisibles.rs @@ -46,9 +46,9 @@ pub fn is_invisible(c: char) -> bool { // ASCII control characters have fancy unicode glyphs, everything else // is replaced by a space - unless it is used in combining characters in // which case we need to leave it in the string. -pub fn replacement(c: char) -> Option<&'static str> { +pub fn replacement(c: char) -> Option { if c <= '\x1f' { - Some(C0_SYMBOLS[c as usize]) + C0_SYMBOLS.get(c as usize).copied() } else if c == '\x7f' { Some(DEL) } else if contains(c, PRESERVE) { @@ -58,18 +58,18 @@ pub fn replacement(c: char) -> Option<&'static str> { } } -const FIXED_WIDTH_SPACE: &str = "\u{2007}"; +const FIXED_WIDTH_SPACE: char = '\u{2007}'; // IDEOGRAPHIC SPACE is common alongside Chinese and other wide character sets. // We don't highlight this for now (as it already shows up wide in the editor), // but could if we tracked state in the classifier. const IDEOGRAPHIC_SPACE: char = '\u{3000}'; -const C0_SYMBOLS: &[&str] = &[ - "␀", "␁", "␂", "␃", "␄", "␅", "␆", "␇", "␈", "␉", "␊", "␋", "␌", "␍", "␎", "␏", "␐", "␑", "␒", - "␓", "␔", "␕", "␖", "␗", "␘", "␙", "␚", "␛", "␜", "␝", "␞", "␟", +const C0_SYMBOLS: &[char] = &[ + '␀', '␁', '␂', '␃', '␄', '␅', '␆', '␇', '␈', '␉', '␊', '␋', '␌', '␍', '␎', '␏', '␐', '␑', '␒', + '␓', '␔', '␕', '␖', '␗', '␘', '␙', '␚', '␛', '␜', '␝', '␞', '␟', ]; -const DEL: &str = "␡"; +const DEL: char = '␡'; // generated using ucd-generate: ucd-generate general-category --include Format --chars ucd-16.0.0 pub const FORMAT: &[(char, char)] = &[ diff --git a/crates/editor/src/display_map/wrap_map.rs b/crates/editor/src/display_map/wrap_map.rs index 8e01e3ffaeb..7eeb1be3946 100644 --- a/crates/editor/src/display_map/wrap_map.rs +++ b/crates/editor/src/display_map/wrap_map.rs @@ -2,16 +2,28 @@ use super::{ Highlights, dimensions::RowDelta, fold_map::{Chunk, FoldRows}, + invisibles::{is_invisible, replacement}, tab_map::{self, TabEdit, TabPoint, TabSnapshot}, }; +use collections::HashMap; use futures_lite::future::yield_now; -use gpui::{App, AppContext as _, Context, Entity, Font, LineWrapper, Pixels, Task}; +use gpui::{ + App, AppContext as _, Context, Entity, Font, FontId, LineWrapper, Pixels, Task, TextSystem, +}; use language::{LanguageAwareStyling, Point}; use multi_buffer::RowInfo; -use std::{cmp, collections::VecDeque, mem, ops::Range, sync::LazyLock, time::Duration}; +use std::{ + cmp, + collections::VecDeque, + mem, + ops::Range, + sync::{Arc, LazyLock}, + time::Duration, +}; use sum_tree::{Bias, Cursor, Dimensions, SumTree}; use text::Patch; +use unicode_segmentation::GraphemeCursor; pub use super::tab_map::TextSummary; pub type WrapEdit = text::Edit; @@ -75,6 +87,70 @@ impl TransformSummary { #[derive(Copy, Clone, Debug, Default, Eq, Ord, PartialOrd, PartialEq)] pub struct WrapPoint(pub Point); +struct LineFragmentBuilder { + text_system: Arc, + font_id: FontId, + font_size: Pixels, + cached_replacement_widths: HashMap, +} + +impl LineFragmentBuilder { + fn new(text_system: Arc, font: &Font, font_size: Pixels) -> Self { + let font_id = text_system.resolve_font(font); + Self { + text_system, + font_id, + font_size, + cached_replacement_widths: HashMap::default(), + } + } + + fn push_fragments<'a>(&mut self, fragments: &mut Vec>, text: &'a str) { + let mut prefix_start = 0; + for (offset, ch) in text.char_indices() { + if !is_invisible(ch) { + continue; + } + let ch_end = offset + ch.len_utf8(); + if !is_standalone_grapheme(text, offset, ch_end) { + continue; + } + let Some(width) = self.replacement_width(ch) else { + continue; + }; + if prefix_start < offset { + fragments.push(gpui::LineFragment::text(&text[prefix_start..offset])); + } + fragments.push(gpui::LineFragment::element(width, ch_end - offset)); + prefix_start = ch_end; + } + if prefix_start < text.len() || text.is_empty() { + fragments.push(gpui::LineFragment::text(&text[prefix_start..])); + } + } + + fn replacement_width(&mut self, ch: char) -> Option { + let replacement_char = replacement(ch)?; + let width = *self + .cached_replacement_widths + .entry(replacement_char) + .or_insert_with(|| { + self.text_system + .layout_width(self.font_id, self.font_size, replacement_char) + }); + Some(width) + } +} + +fn is_standalone_grapheme(text: &str, start: usize, end: usize) -> bool { + let mut cursor = GraphemeCursor::new(start, text.len(), true); + if cursor.is_boundary(text, 0) != Ok(true) { + return false; + } + cursor.set_cursor(end); + cursor.is_boundary(text, 0) == Ok(true) +} + pub struct WrapChunks<'a> { input_chunks: tab_map::TabChunks<'a>, input_chunk: Chunk<'a>, @@ -202,6 +278,8 @@ impl WrapMap { let text_system = cx.text_system(); let (font, font_size) = self.font_with_size.clone(); + let mut fragment_builder = + LineFragmentBuilder::new(text_system.clone(), &font, font_size); let mut line_wrapper = text_system.line_wrapper(font, font_size); let tab_snapshot = new_snapshot.tab_snapshot.clone(); let total_rows = tab_snapshot.max_point().row() as usize + 1; @@ -217,13 +295,20 @@ impl WrapMap { &tab_edits, wrap_width, &mut line_wrapper, + &mut fragment_builder, )); self.snapshot = new_snapshot; self.edits_since_sync = self.edits_since_sync.compose(&edits); } else { let task = cx.background_spawn(async move { let edits = new_snapshot - .update(tab_snapshot, &tab_edits, wrap_width, &mut line_wrapper) + .update( + tab_snapshot, + &tab_edits, + wrap_width, + &mut line_wrapper, + &mut fragment_builder, + ) .await; (new_snapshot, edits) }); @@ -297,6 +382,8 @@ impl WrapMap { let mut snapshot = self.snapshot.clone(); let text_system = cx.text_system().clone(); let (font, font_size) = self.font_with_size.clone(); + let mut fragment_builder = + LineFragmentBuilder::new(text_system.clone(), &font, font_size); let mut line_wrapper = text_system.line_wrapper(font, font_size); if pending_edits.len() == 1 @@ -311,6 +398,7 @@ impl WrapMap { &tab_edits, wrap_width, &mut line_wrapper, + &mut fragment_builder, )); self.snapshot = snapshot; self.edits_since_sync = self.edits_since_sync.compose(&wrap_edits); @@ -319,7 +407,13 @@ impl WrapMap { let mut edits = Patch::default(); for (tab_snapshot, tab_edits) in pending_edits { let wrap_edits = snapshot - .update(tab_snapshot, &tab_edits, wrap_width, &mut line_wrapper) + .update( + tab_snapshot, + &tab_edits, + wrap_width, + &mut line_wrapper, + &mut fragment_builder, + ) .await; edits = edits.compose(&wrap_edits); } @@ -465,6 +559,7 @@ impl WrapSnapshot { tab_edits: &[TabEdit], wrap_width: Pixels, line_wrapper: &mut LineWrapper, + fragment_builder: &mut LineFragmentBuilder, ) -> WrapPatch { #[derive(Debug)] struct RowEdit { @@ -532,7 +627,7 @@ impl WrapSnapshot { while let Some(chunk) = remaining.take().or_else(|| chunks.next()) { if let Some(ix) = chunk.text.find('\n') { let (prefix, suffix) = chunk.text.split_at(ix + 1); - line_fragments.push(gpui::LineFragment::text(prefix)); + fragment_builder.push_fragments(&mut line_fragments, prefix); line.push_str(prefix); remaining = Some(Chunk { text: suffix, @@ -546,7 +641,7 @@ impl WrapSnapshot { line_fragments .push(gpui::LineFragment::element(width, chunk.text.len())); } else { - line_fragments.push(gpui::LineFragment::text(chunk.text)); + fragment_builder.push_fragments(&mut line_fragments, chunk.text); } line.push_str(chunk.text); } @@ -1443,6 +1538,115 @@ mod tests { assert_eq!(row.0, 3); } + #[gpui::test] + async fn test_invisibles_become_width_measured_elements(cx: &mut gpui::TestAppContext) { + init_test(cx); + + let text_system = cx.read(|cx| cx.text_system().clone()); + let font = test_font(); + let font_size = px(14.0); + let mut fragment_builder = LineFragmentBuilder::new(text_system, &font, font_size); + + let mut fragments = Vec::new(); + fragment_builder.push_fragments(&mut fragments, "ab\u{7f}cd\u{80}\u{81}e"); + let shapes = fragments + .iter() + .map(|fragment| match fragment { + LineFragment::Text { text } => (false, text.len()), + LineFragment::Element { len_utf8, width } => { + assert!( + *width > px(0.), + "invisible characters must carry the measured width of their \ + replacement glyph so the wrapper accounts for what the renderer \ + actually draws" + ); + (true, *len_utf8) + } + }) + .collect::>(); + assert_eq!( + shapes, + vec![ + (false, 2), + (true, 1), + (false, 2), + (true, 2), + (true, 2), + (false, 1), + ], + "invisible chars must become single-char elements with exact utf8 lengths, \ + keeping the wrapper's byte offsets aligned with the source text" + ); + + let mut plain_fragments = Vec::new(); + fragment_builder.push_fragments(&mut plain_fragments, "plain text"); + assert_eq!( + plain_fragments.len(), + 1, + "text without invisibles must stay one fragment" + ); + assert_eq!( + plain_fragments.first().map(|fragment| match fragment { + LineFragment::Text { text } => text.len(), + LineFragment::Element { len_utf8, .. } => *len_utf8, + }), + Some(10), + ); + + let mut preserved_fragments = Vec::new(); + fragment_builder.push_fragments(&mut preserved_fragments, "a\u{200d}b"); + assert_eq!( + preserved_fragments.len(), + 1, + "combining-class invisibles like ZWJ have no replacement and must stay \ + inside their text fragment" + ); + } + + #[gpui::test] + async fn test_invisibles_wrap_at_replacement_glyph_width(cx: &mut gpui::TestAppContext) { + init_test(cx); + + let text_system = cx.read(|cx| cx.text_system().clone()); + let font = test_font(); + let font_id = text_system.resolve_font(&font); + let font_size = px(14.0); + let wrap_width = px(140.0); + + let text = "\u{7f}\u{80}\u{81}\u{82}\u{83}\u{84}".repeat(20); + let buffer = cx.new(|cx| language::Buffer::local(text, cx)); + let buffer = cx.new(|cx| MultiBuffer::singleton(buffer, cx)); + let buffer_snapshot = buffer.read_with(cx, |buffer, cx| buffer.snapshot(cx)); + let (_inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot); + let (_fold_map, fold_snapshot) = FoldMap::new(inlay_snapshot); + let (mut tab_map, _) = TabMap::new(fold_snapshot, 4.try_into().unwrap()); + let tabs_snapshot = tab_map.set_max_expansion_column(32); + let (_wrap_map, wrap_snapshot) = + cx.update(|cx| WrapMap::new(tabs_snapshot, font, font_size, Some(wrap_width), cx)); + + let wrapped_text = wrap_snapshot.text(); + for row in wrapped_text.split('\n') { + let rendered_width = row + .chars() + .map(|ch| { + let rendered_char = if is_invisible(ch) { + replacement(ch).unwrap_or(ch) + } else { + ch + }; + text_system.layout_width(font_id, font_size, rendered_char) + }) + .fold(px(0.), |width, char_width| width + char_width); + assert!( + rendered_width <= wrap_width, + "a soft-wrapped row must fit the wrap width when drawn with the \ + replacement glyphs the renderer substitutes for invisible characters, \ + but this row renders at {rendered_width:?} against a wrap width of \ + {wrap_width:?}: {row:?}" + ); + } + } + #[gpui::test(iterations = 100)] async fn test_random_wraps(cx: &mut gpui::TestAppContext, mut rng: StdRng) { // todo this test is flaky diff --git a/crates/gpui_shared_string/gpui_shared_string.rs b/crates/gpui_shared_string/gpui_shared_string.rs index 4395065b9fd..8449c065ad2 100644 --- a/crates/gpui_shared_string/gpui_shared_string.rs +++ b/crates/gpui_shared_string/gpui_shared_string.rs @@ -1,5 +1,6 @@ use std::{ borrow::{Borrow, Cow}, + iter, sync::Arc, }; @@ -120,6 +121,13 @@ impl From<&str> for SharedString { } } +impl From for SharedString { + #[inline] + fn from(c: char) -> SharedString { + SharedString(SmolStr::from_iter(iter::once(c))) + } +} + impl From<&mut str> for SharedString { #[inline] fn from(s: &mut str) -> SharedString {