diff --git a/crates/benchmarks/benches/editor_render.rs b/crates/benchmarks/benches/editor_render.rs index 67433bdb225..813069adaa0 100644 --- a/crates/benchmarks/benches/editor_render.rs +++ b/crates/benchmarks/benches/editor_render.rs @@ -1,10 +1,13 @@ +use benchmarks::bench_utils::random_rust_file; use editor::{ Editor, EditorMode, MultiBuffer, actions::{DeleteToPreviousWordStart, SelectAll, SplitSelectionIntoLines}, }; -use gpui::{AppContext as _, BenchAppContext, Focusable as _}; +use gpui::{AppContext as _, BenchAppContext, Focusable as _, UpdateGlobal as _}; +use language::{Buffer, Rope}; use rand::{Rng as _, SeedableRng as _, rngs::StdRng}; -use settings::SettingsStore; +use settings::{DisplayIn, SettingsStore, ShowMinimap}; +use theme::ActiveTheme as _; use util::RandomCharIter; use zed_actions::editor::{MoveDown, MoveUp}; @@ -124,6 +127,78 @@ fn editor_render(cx: &mut BenchAppContext) { }); } +#[gpui::bench] +fn editor_render_highlighted(cx: &mut BenchAppContext) { + init_context(cx); + render_highlighted_editor(cx); +} + +#[gpui::bench] +fn editor_render_highlighted_minimap(cx: &mut BenchAppContext) { + init_context(cx); + cx.update(|cx| { + SettingsStore::update_global(cx, |store, cx| { + store.update_user_settings(cx, |settings| { + let minimap = settings.editor.minimap.get_or_insert_default(); + minimap.show = Some(ShowMinimap::Always); + minimap.display_in = Some(DisplayIn::AllEditors); + }); + }); + }); + render_highlighted_editor(cx); +} + +fn render_highlighted_editor(cx: &mut BenchAppContext) { + let mut rng = StdRng::seed_from_u64(1); + let text = random_rust_file(&mut rng, 10_000).join("\n"); + let language = language::rust_lang(); + let syntax_theme = cx.update(|cx| { + let syntax_theme = cx.theme().syntax().clone(); + language.set_theme(&syntax_theme); + syntax_theme + }); + let probe = "fn main() {}"; + assert!( + !language + .highlight_text(&Rope::from(probe), 0..probe.len()) + .is_empty(), + "the benchmark language must resolve syntax highlights against the theme" + ); + let buffer = cx.update(|cx| { + let buffer = cx.new(|cx| Buffer::local(text, cx).with_language(language, cx)); + cx.new(|cx| MultiBuffer::singleton(buffer, cx)) + }); + cx.run_until_idle(); + + let mut window = cx.add_empty_window(); + let editor = window.update(|window, cx| { + let editor = window.replace_root(cx, |window, cx| { + let mut editor = Editor::new(EditorMode::full(), buffer, None, window, cx); + editor.set_style( + editor::EditorStyle { + syntax: syntax_theme.clone(), + ..editor::EditorStyle::default() + }, + window, + cx, + ); + editor + }); + window.focus(&editor.focus_handle(cx), cx); + editor + }); + + let mut move_down = true; + cx.bench_renderer(editor, move |editor, window, cx| { + if move_down { + editor.move_down(&MoveDown, window, cx); + } else { + editor.move_up(&MoveUp, window, cx); + } + move_down = !move_down; + }); +} + fn init_context(cx: &mut BenchAppContext) { cx.update(|cx| { let store = SettingsStore::test(cx); @@ -146,6 +221,8 @@ gpui::bench_group!( benches, editor_multi_cursor_input, open_editor_with_one_long_line, - editor_render + editor_render, + editor_render_highlighted, + editor_render_highlighted_minimap ); gpui::bench_main!(benches); diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs index 6928eb91ff5..63a98037dae 100644 --- a/crates/language/src/buffer.rs +++ b/crates/language/src/buffer.rs @@ -9,11 +9,12 @@ use crate::{ diagnostic_set::{DiagnosticEntry, DiagnosticEntryRef, DiagnosticGroup}, language_settings::{AutoIndentMode, LanguageSettings}, outline::OutlineItem, - row_chunk::RowChunks, + row_chunk::{RowChunkId, RowChunks}, runnable::{self, RunnableRange}, syntax_map::{ MAX_BYTES_TO_QUERY, SyntaxLayer, SyntaxMap, SyntaxMapCapture, SyntaxMapCaptures, SyntaxMapMatch, SyntaxMapMatches, SyntaxSnapshot, ToTreeSitterPoint, + flattened_highlight_regions, }, text_diff::text_diff, unified_diff_with_offsets, @@ -35,6 +36,7 @@ use gpui::{ App, AppContext as _, Context, Entity, EventEmitter, HighlightStyle, SharedString, StyledText, Task, TextStyle, }; +use language_core::highlight_cache::{ChunkCaptureRun, ChunkCaptures, ChunkHighlightCache}; use lsp::LanguageServerId; use parking_lot::Mutex; @@ -147,21 +149,25 @@ pub struct Buffer { #[derive(Debug)] pub struct TreeSitterData { chunks: RowChunks, - brackets_by_chunks: Mutex>>, + brackets_by_chunks: Mutex>>, + highlights_by_chunks: ChunkHighlightCache, } -const MAX_ROWS_IN_A_CHUNK: u32 = 50; +pub(crate) const MAX_ROWS_IN_A_CHUNK: u32 = 50; +pub(crate) const MAX_BYTES_TO_HIGHLIGHT_IN_A_CHUNK: usize = 4 * MAX_BYTES_TO_QUERY; impl TreeSitterData { fn clear(&mut self, snapshot: &text::BufferSnapshot) { self.chunks = RowChunks::new(snapshot, MAX_ROWS_IN_A_CHUNK); self.brackets_by_chunks.get_mut().clear(); + self.highlights_by_chunks.clear(); } fn new(snapshot: &text::BufferSnapshot) -> Self { Self { chunks: RowChunks::new(snapshot, MAX_ROWS_IN_A_CHUNK), brackets_by_chunks: Mutex::new(HashMap::default()), + highlights_by_chunks: ChunkHighlightCache::default(), } } @@ -514,6 +520,13 @@ struct BufferChunkHighlights<'a> { highlight_maps: Vec, } +type HighlightRun = (Range, HighlightId); + +struct CachedChunkHighlightsIter { + runs: Vec, + ix: usize, +} + /// An iterator that yields chunks of a buffer's text, along with their /// syntax highlights and diagnostic status. pub struct BufferChunks<'a> { @@ -528,6 +541,7 @@ pub struct BufferChunks<'a> { unnecessary_depth: usize, underline: bool, highlights: Option>, + cached_highlights: Option, } /// A chunk of a buffer's text, along with its syntax highlight and @@ -1496,6 +1510,7 @@ impl Buffer { } self.non_text_state_update_count += 1; self.syntax_map.lock().clear(&self.text); + Self::invalidate_tree_sitter_data(&mut self.tree_sitter_data, self.text.snapshot()); let old_language = std::mem::replace(&mut self.language, language); self.was_changed(); self.reparse(cx, may_block); @@ -3980,7 +3995,18 @@ impl BufferSnapshot { let mut syntax = None; if language_aware.tree_sitter { - syntax = Some(self.get_highlights(range.clone())); + match self.cached_highlight_runs(range.clone()) { + Some(runs) => { + return BufferChunks::with_cached_highlights( + self.text.as_rope(), + range, + runs, + language_aware.diagnostics, + self, + ); + } + None => syntax = Some(self.get_highlights(range.clone())), + } } BufferChunks::new( self.text.as_rope(), @@ -3991,6 +4017,100 @@ impl BufferSnapshot { ) } + pub(crate) fn cached_highlight_runs(&self, range: Range) -> Option> { + #[cfg(any(test, feature = "test-support"))] + { + static DISABLE_HIGHLIGHT_CACHE: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + std::env::var_os("ZED_DISABLE_HIGHLIGHT_CACHE").is_some() + }); + if *DISABLE_HIGHLIGHT_CACHE { + return None; + } + } + self.language.as_ref()?.grammar()?; + if range.is_empty() { + return Some(Vec::new()); + } + let mut runs = Vec::::new(); + for chunk in self + .tree_sitter_data + .chunks + .applicable_chunks(&[range.to_point(self)]) + { + let chunk_range = chunk.anchor_range().to_offset(self); + if chunk_range.end <= range.start || chunk_range.start >= range.end { + continue; + } + if chunk_range.len() > MAX_BYTES_TO_HIGHLIGHT_IN_A_CHUNK { + return None; + } + let chunk_captures = match self.tree_sitter_data.highlights_by_chunks.get(chunk.id) { + Some(chunk_captures) => chunk_captures, + None => { + let chunk_captures = self.compute_chunk_captures(chunk_range); + self.tree_sitter_data + .highlights_by_chunks + .insert(chunk.id, chunk_captures.clone()); + chunk_captures + } + }; + let highlight_maps = chunk_captures + .grammars + .iter() + .map(|grammar| grammar.highlight_map()) + .collect::>(); + for run in chunk_captures.runs.iter() { + if run.range.end <= range.start { + continue; + } + if run.range.start >= range.end { + break; + } + let highlight_id = run.stack.iter().rev().find_map(|capture| { + highlight_maps + .get(capture.grammar_index)? + .get(capture.capture_id) + }); + let Some(highlight_id) = highlight_id else { + continue; + }; + match runs.last_mut() { + Some((last_range, last_highlight_id)) + if *last_highlight_id == highlight_id + && last_range.end == run.range.start => + { + last_range.end = run.range.end; + } + _ => runs.push((run.range.clone(), highlight_id)), + } + } + } + Some(runs) + } + + fn compute_chunk_captures(&self, range: Range) -> ChunkCaptures { + let captures = self.syntax.captures(range.clone(), &self.text, |grammar| { + grammar + .highlights_config + .as_ref() + .map(|config| &config.query) + }); + let grammars = captures + .grammars() + .iter() + .map(|&grammar| grammar.clone()) + .collect(); + let runs = flattened_highlight_regions(captures, range) + .into_iter() + .map(|region| ChunkCaptureRun { + range: region.range, + stack: region.stack, + }) + .collect(); + ChunkCaptures { grammars, runs } + } + pub fn highlighted_text_for_range( &self, range: Range, @@ -5343,16 +5463,40 @@ impl<'a> BufferChunks<'a> { diagnostics: bool, buffer_snapshot: Option<&'a BufferSnapshot>, ) -> Self { - let mut highlights = None; - if let Some((captures, highlight_maps)) = syntax { - highlights = Some(BufferChunkHighlights { - captures, - next_capture: None, - stack: Default::default(), - highlight_maps, - }) - } + let highlights = syntax.map(|(captures, highlight_maps)| BufferChunkHighlights { + captures, + next_capture: None, + stack: Vec::new(), + highlight_maps, + }); + Self::init(text, range, highlights, None, diagnostics, buffer_snapshot) + } + fn with_cached_highlights( + text: &'a Rope, + range: Range, + runs: Vec, + diagnostics: bool, + buffer_snapshot: &'a BufferSnapshot, + ) -> Self { + Self::init( + text, + range, + None, + Some(CachedChunkHighlightsIter { runs, ix: 0 }), + diagnostics, + Some(buffer_snapshot), + ) + } + + fn init( + text: &'a Rope, + range: Range, + highlights: Option>, + cached_highlights: Option, + diagnostics: bool, + buffer_snapshot: Option<&'a BufferSnapshot>, + ) -> Self { let diagnostic_endpoints = diagnostics.then(|| Vec::new().into_iter().peekable()); let chunks = text.chunks_in_range(range.clone()); @@ -5368,6 +5512,7 @@ impl<'a> BufferChunks<'a> { unnecessary_depth: 0, underline: true, highlights, + cached_highlights, }; this.initialize_diagnostic_endpoints(); this @@ -5377,7 +5522,33 @@ impl<'a> BufferChunks<'a> { pub fn seek(&mut self, range: Range) { let old_range = std::mem::replace(&mut self.range, range.clone()); self.chunks.set_range(self.range.clone()); - if let Some(highlights) = self.highlights.as_mut() { + if let Some(cached) = self.cached_highlights.as_mut() { + if old_range.start <= self.range.start && old_range.end >= self.range.end { + cached.ix = cached + .runs + .partition_point(|(run_range, _)| run_range.end <= range.start); + } else if let Some(snapshot) = self.buffer_snapshot { + if let Some(runs) = snapshot.cached_highlight_runs(self.range.clone()) { + cached.runs = runs; + cached.ix = 0; + } else { + let (captures, highlight_maps) = snapshot.get_highlights(self.range.clone()); + self.cached_highlights = None; + self.highlights = Some(BufferChunkHighlights { + captures, + next_capture: None, + stack: Vec::new(), + highlight_maps, + }); + } + } else { + debug_assert!( + false, + "Attempted to seek on a language-aware buffer iterator without associated buffer snapshot" + ); + } + self.initialize_diagnostic_endpoints(); + } else if let Some(highlights) = self.highlights.as_mut() { if old_range.start <= self.range.start && old_range.end >= self.range.end { // Reuse existing highlights stack, as the new range is a subrange of the old one. highlights @@ -5400,7 +5571,7 @@ impl<'a> BufferChunks<'a> { *highlights = BufferChunkHighlights { captures, next_capture: None, - stack: Default::default(), + stack: Vec::new(), highlight_maps, }; } else { @@ -5536,6 +5707,21 @@ impl<'a> Iterator for BufferChunks<'a> { } } + if let Some(cached) = self.cached_highlights.as_mut() { + while cached + .runs + .get(cached.ix) + .is_some_and(|(run_range, _)| run_range.end <= self.range.start) + { + cached.ix += 1; + } + if let Some((run_range, _)) = cached.runs.get(cached.ix) + && self.range.start < run_range.start + { + next_capture_start = run_range.start; + } + } + let mut diagnostic_endpoints = std::mem::take(&mut self.diagnostic_endpoints); if let Some(diagnostic_endpoints) = diagnostic_endpoints.as_mut() { while let Some(endpoint) = diagnostic_endpoints.peek().copied() { @@ -5569,6 +5755,13 @@ impl<'a> Iterator for BufferChunks<'a> { chunk_end = chunk_end.min(*parent_capture_end); highlight_id = Some(*parent_highlight_id); } + if let Some(cached) = self.cached_highlights.as_ref() + && let Some((run_range, run_highlight_id)) = cached.runs.get(cached.ix) + && run_range.start <= chunk_start + { + chunk_end = chunk_end.min(run_range.end); + highlight_id = Some(*run_highlight_id); + } let bit_start = chunk_start - self.chunks.offset(); let bit_end = chunk_end - self.chunks.offset(); diff --git a/crates/language/src/buffer/row_chunk.rs b/crates/language/src/buffer/row_chunk.rs index 605299f67f0..7764067f922 100644 --- a/crates/language/src/buffer/row_chunk.rs +++ b/crates/language/src/buffer/row_chunk.rs @@ -10,6 +10,8 @@ use util::RangeExt; use crate::BufferRow; +pub type RowChunkId = usize; + /// An range of rows, exclusive as [`lsp::Range`] and /// /// denote. @@ -27,7 +29,7 @@ pub struct RowChunks { buffer_snapshot: text::BufferSnapshot, last_row: BufferRow, max_rows_per_chunk: u32, - computed_chunks: Mutex>, + computed_chunks: Mutex>, } impl std::fmt::Debug for RowChunks { @@ -86,12 +88,18 @@ impl RowChunks { } } - fn chunk_row_range(&self, id: usize) -> Range { - let start = id as u32 * self.max_rows_per_chunk; - start..(start + self.max_rows_per_chunk).min(self.last_row) + fn chunk_row_range(&self, id: RowChunkId) -> Range { + let start = u32::try_from(id) + .unwrap_or(u32::MAX) + .saturating_mul(self.max_rows_per_chunk) + .min(self.last_row); + start + ..start + .saturating_add(self.max_rows_per_chunk) + .min(self.last_row) } - fn chunk(&self, id: usize) -> RowChunk { + fn chunk(&self, id: RowChunkId) -> RowChunk { *self.computed_chunks.lock().entry(id).or_insert_with(|| { let row_range = self.chunk_row_range(id); let start = Point::new(row_range.start, 0); @@ -113,7 +121,7 @@ impl RowChunks { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct RowChunk { - pub id: usize, + pub id: RowChunkId, pub start: BufferRow, pub end_exclusive: BufferRow, pub start_anchor: Anchor, diff --git a/crates/language/src/buffer_tests.rs b/crates/language/src/buffer_tests.rs index f5b08929066..0eea4a065c4 100644 --- a/crates/language/src/buffer_tests.rs +++ b/crates/language/src/buffer_tests.rs @@ -5161,6 +5161,269 @@ fn init_settings(cx: &mut App, f: fn(&mut AllLanguageSettingsContent)) { }); } +#[gpui::test] +fn test_chunk_highlights_follow_edits_and_theme_changes(cx: &mut TestAppContext) { + cx.update(|cx| init_settings(cx, |_| {})); + + let language = keyword_and_function_lang(); + let theme_with_keyword = keyword_and_function_theme(); + let theme_without_keyword = + SyntaxTheme::new([("function".to_string(), gpui::rgba(0x0000ffff).into())]); + language.set_theme(&theme_with_keyword); + + let buffer = cx.new(|cx| { + let mut buffer = Buffer::local("fn main() {}", cx); + buffer.set_language(Some(language.clone()), cx); + buffer + }); + cx.run_until_parked(); + + let highlighted_chunks = |snapshot: &BufferSnapshot| { + snapshot + .chunks( + 0..snapshot.len(), + LanguageAwareStyling { + tree_sitter: true, + diagnostics: false, + }, + ) + .filter_map(|chunk| Some((chunk.text.to_string(), chunk.syntax_highlight_id?))) + .collect::>() + }; + + let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot()); + let expected_with_keyword = vec![ + ( + "fn".to_string(), + theme_highlight_id(&theme_with_keyword, "keyword"), + ), + ( + "main".to_string(), + theme_highlight_id(&theme_with_keyword, "function"), + ), + ]; + assert_eq!(highlighted_chunks(&snapshot), expected_with_keyword); + assert_eq!( + highlighted_chunks(&snapshot), + expected_with_keyword, + "repeated chunking of the same snapshot must return the same highlights" + ); + + language.set_theme(&theme_without_keyword); + assert_eq!( + highlighted_chunks(&snapshot), + vec![( + "main".to_string(), + theme_highlight_id(&theme_without_keyword, "function") + )], + "a theme change must invalidate highlights of an existing snapshot" + ); + + buffer.update(cx, |buffer, cx| { + buffer.edit([(3..7, "launch")], None, cx); + }); + cx.run_until_parked(); + let edited_snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot()); + assert_eq!( + highlighted_chunks(&edited_snapshot), + vec![( + "launch".to_string(), + theme_highlight_id(&theme_without_keyword, "function") + )], + "an edit must invalidate the cached highlights" + ); +} + +#[gpui::test] +fn test_chunk_highlights_across_row_chunk_seeks(cx: &mut TestAppContext) { + cx.update(|cx| init_settings(cx, |_| {})); + + let language = keyword_and_function_lang(); + let theme = keyword_and_function_theme(); + language.set_theme(&theme); + + let short_row = "fn short() {}\n"; + let last_row = "fn omega() {}"; + let text = format!( + "{}{last_row}", + short_row.repeat(MAX_ROWS_IN_A_CHUNK as usize) + ); + + let buffer = cx.new(|cx| { + let mut buffer = Buffer::local(text, cx); + buffer.set_language(Some(language.clone()), cx); + buffer + }); + cx.run_until_parked(); + let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot()); + + let keyword = theme_highlight_id(&theme, "keyword"); + let function = theme_highlight_id(&theme, "function"); + + assert_eq!( + merged_highlight_runs(&snapshot, 0..short_row.len()), + vec![("fn".to_string(), keyword), ("short".to_string(), function)], + ); + + let last_row_start = short_row.len() * MAX_ROWS_IN_A_CHUNK as usize; + let expected_last_row_runs = vec![("fn".to_string(), keyword), ("omega".to_string(), function)]; + assert_eq!( + merged_highlight_runs(&snapshot, last_row_start..snapshot.len()), + expected_last_row_runs, + ); + + let mut chunks = snapshot.chunks( + 0..short_row.len(), + LanguageAwareStyling { + tree_sitter: true, + diagnostics: false, + }, + ); + chunks.seek(last_row_start..snapshot.len()); + let mut runs_after_seek = Vec::new(); + for chunk in chunks { + merge_highlighted_chunk(&mut runs_after_seek, &chunk); + } + assert_eq!( + runs_after_seek, expected_last_row_runs, + "seeking into another row chunk must refetch that chunk's highlights" + ); +} + +#[gpui::test] +fn test_oversized_chunks_bypass_the_highlight_cache(cx: &mut TestAppContext) { + if std::env::var_os("ZED_DISABLE_HIGHLIGHT_CACHE").is_some() { + return; + } + cx.update(|cx| init_settings(cx, |_| {})); + + let language = keyword_and_function_lang(); + let theme = keyword_and_function_theme(); + language.set_theme(&theme); + + let short_row = "fn short() {}\n"; + let giant_row = format!( + "fn omega() {{}}{}", + " ".repeat(MAX_BYTES_TO_HIGHLIGHT_IN_A_CHUNK) + ); + let text = format!( + "{}{giant_row}", + short_row.repeat(MAX_ROWS_IN_A_CHUNK as usize) + ); + + let buffer = cx.new(|cx| { + let mut buffer = Buffer::local(text, cx); + buffer.set_language(Some(language.clone()), cx); + buffer + }); + cx.run_until_parked(); + let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot()); + + let keyword = theme_highlight_id(&theme, "keyword"); + let function = theme_highlight_id(&theme, "function"); + let expected_giant_row_runs = + vec![("fn".to_string(), keyword), ("omega".to_string(), function)]; + + let last_row_start = short_row.len() * MAX_ROWS_IN_A_CHUNK as usize; + assert_eq!( + snapshot.cached_highlight_runs(last_row_start..snapshot.len()), + None, + "chunks larger than the byte cap must not be cached" + ); + assert!( + snapshot.cached_highlight_runs(0..short_row.len()).is_some(), + "chunks within the byte cap must still be cached" + ); + assert_eq!( + snapshot.cached_highlight_runs(last_row_start + 1..last_row_start + 1), + Some(Vec::new()), + "an empty range must not compute captures for its oversized chunk" + ); + + assert_eq!( + merged_highlight_runs(&snapshot, last_row_start..snapshot.len()), + expected_giant_row_runs, + "oversized chunks must fall back to direct highlighting" + ); + + let mut chunks = snapshot.chunks( + 0..short_row.len(), + LanguageAwareStyling { + tree_sitter: true, + diagnostics: false, + }, + ); + chunks.seek(last_row_start..snapshot.len()); + let mut runs_after_seek = Vec::new(); + for chunk in chunks { + merge_highlighted_chunk(&mut runs_after_seek, &chunk); + } + assert_eq!( + runs_after_seek, expected_giant_row_runs, + "seeking into an oversized chunk must fall back to direct highlighting" + ); +} + +#[gpui::test] +fn test_language_change_invalidates_cached_chunk_highlights(cx: &mut TestAppContext) { + cx.update(|cx| init_settings(cx, |_| {})); + + let rust = keyword_and_function_lang(); + let identifiers_only = Arc::new( + Language::new( + LanguageConfig { + name: "Identifiers".into(), + ..LanguageConfig::default() + }, + Some(tree_sitter_rust::LANGUAGE.into()), + ) + .with_highlights_query("(identifier) @variable") + .unwrap(), + ); + let theme = SyntaxTheme::new([ + ("keyword".to_string(), gpui::rgba(0xff0000ff).into()), + ("function".to_string(), gpui::rgba(0x00ff00ff).into()), + ("variable".to_string(), gpui::rgba(0x0000ffff).into()), + ]); + rust.set_theme(&theme); + identifiers_only.set_theme(&theme); + + let buffer = cx.new(|cx| { + let mut buffer = Buffer::local("fn main() {}", cx); + buffer.set_language(Some(rust), cx); + buffer + }); + cx.run_until_parked(); + + let keyword = theme_highlight_id(&theme, "keyword"); + let function = theme_highlight_id(&theme, "function"); + let variable = theme_highlight_id(&theme, "variable"); + + let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot()); + assert_eq!( + merged_highlight_runs(&snapshot, 0..snapshot.len()), + vec![("fn".to_string(), keyword), ("main".to_string(), function)], + ); + + buffer.update(cx, |buffer, cx| { + buffer.set_sync_parse_timeout(None); + buffer.set_language(Some(identifiers_only), cx); + }); + let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot()); + assert_eq!( + merged_highlight_runs(&snapshot, 0..snapshot.len()), + Vec::new(), + "a language change must not serve the old language's cached highlights while the new parse is pending" + ); + + cx.run_until_parked(); + let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot()); + assert_eq!( + merged_highlight_runs(&snapshot, 0..snapshot.len()), + vec![("main".to_string(), variable)], + ); +} + #[gpui::test(iterations = 100)] fn test_random_chunk_bitmaps(cx: &mut App, mut rng: StdRng) { use util::RandomCharIter; @@ -5288,3 +5551,63 @@ fn test_formatted_chunks(cx: &mut gpui::App) { } } } + +fn merged_highlight_runs( + snapshot: &BufferSnapshot, + range: Range, +) -> Vec<(String, HighlightId)> { + let chunks = snapshot.chunks( + range, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: false, + }, + ); + let mut runs = Vec::new(); + for chunk in chunks { + merge_highlighted_chunk(&mut runs, &chunk); + } + runs +} + +fn merge_highlighted_chunk(runs: &mut Vec<(String, HighlightId)>, chunk: &Chunk<'_>) { + let Some(highlight_id) = chunk.syntax_highlight_id else { + return; + }; + match runs.last_mut() { + Some((last_text, last_highlight_id)) if *last_highlight_id == highlight_id => { + last_text.push_str(chunk.text); + } + _ => runs.push((chunk.text.to_string(), highlight_id)), + } +} + +fn keyword_and_function_lang() -> Arc { + Arc::new( + Language::new( + LanguageConfig { + name: "Rust".into(), + ..LanguageConfig::default() + }, + Some(tree_sitter_rust::LANGUAGE.into()), + ) + .with_highlights_query( + r#" + "fn" @keyword + (identifier) @function + "#, + ) + .unwrap(), + ) +} + +fn keyword_and_function_theme() -> SyntaxTheme { + SyntaxTheme::new([ + ("keyword".to_string(), gpui::rgba(0xff0000ff).into()), + ("function".to_string(), gpui::rgba(0x00ff00ff).into()), + ]) +} + +fn theme_highlight_id(theme: &SyntaxTheme, capture_name: &str) -> HighlightId { + HighlightId::new(theme.highlight_id(capture_name).unwrap()) +} diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs index 493f782ecbb..ff168f5737b 100644 --- a/crates/language/src/language.rs +++ b/crates/language/src/language.rs @@ -1183,7 +1183,17 @@ impl Language { .as_ref() .map(|config| &config.query) }); - Arc::from(flattened_highlight_regions(captures, 0..text.len())) + flattened_highlight_regions(captures, 0..text.len()) + .into_iter() + .map(|region| CapturedRange { + range: region.range, + capture_ids: region + .stack + .iter() + .map(|capture| capture.capture_id) + .collect(), + }) + .collect() } pub fn path_suffixes(&self) -> &[String] { diff --git a/crates/language/src/syntax_map.rs b/crates/language/src/syntax_map.rs index 4333ef46aef..398c76be4ee 100644 --- a/crates/language/src/syntax_map.rs +++ b/crates/language/src/syntax_map.rs @@ -2,12 +2,14 @@ mod syntax_map_tests; use crate::{ - CaptureId, CapturedRange, Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry, - QUERY_CURSORS, with_parser, + CaptureId, Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry, QUERY_CURSORS, + with_parser, }; use collections::HashMap; use futures::FutureExt; use gpui::SharedString; +use language_core::highlight_cache::HighlightCaptureRef; +use smallvec::SmallVec; use std::{ borrow::Cow, cmp::{self, Ordering, Reverse}, @@ -74,7 +76,7 @@ impl Drop for SyntaxSnapshot { pub struct SyntaxMapCaptures<'a> { layers: Vec>, active_layer_count: usize, - grammars: Vec<&'a Grammar>, + grammars: Vec<&'a Arc>, } #[derive(Default)] @@ -91,34 +93,45 @@ pub struct SyntaxMapCapture<'a> { pub grammar_index: usize, } +pub(crate) struct HighlightCaptureRegion { + pub range: Range, + pub stack: SmallVec<[HighlightCaptureRef; 4]>, +} + pub(crate) fn flattened_highlight_regions( mut captures: SyntaxMapCaptures<'_>, range: Range, -) -> Vec { +) -> Vec { let capture_refs = iter::from_fn(move || { let capture = captures.next()?; - Some((capture.node.byte_range(), CaptureId(capture.index))) + Some(( + capture.node.byte_range(), + HighlightCaptureRef { + grammar_index: capture.grammar_index, + capture_id: CaptureId(capture.index), + }, + )) }); flatten_capture_regions(range, capture_refs) } fn flatten_capture_regions( range: Range, - mut captures: impl Iterator, CaptureId)>, -) -> Vec { + mut captures: impl Iterator, HighlightCaptureRef)>, +) -> Vec { let mut result = Vec::new(); - let mut stack = Vec::<(Range, CaptureId)>::new(); + let mut stack = Vec::<(Range, HighlightCaptureRef)>::new(); let mut offset = range.start; let mut next_capture = captures.next(); loop { stack.retain(|(capture_range, _)| capture_range.end > offset); - while let Some((capture_range, capture_id)) = next_capture.take() { + while let Some((capture_range, capture_ref)) = next_capture.take() { if capture_range.start > offset { - next_capture = Some((capture_range, capture_id)); + next_capture = Some((capture_range, capture_ref)); break; } if capture_range.end > offset { - stack.push((capture_range, capture_id)); + stack.push((capture_range, capture_ref)); } next_capture = captures.next(); } @@ -134,9 +147,9 @@ fn flatten_capture_regions( next_boundary = next_boundary.min(capture_range.start); } if !stack.is_empty() && next_boundary > offset { - result.push(CapturedRange { + result.push(HighlightCaptureRegion { range: offset..next_boundary, - capture_ids: stack.iter().map(|(_, capture_id)| *capture_id).collect(), + stack: stack.iter().map(|(_, capture_ref)| *capture_ref).collect(), }); } if next_boundary >= range.end { @@ -1239,7 +1252,7 @@ impl<'a> SyntaxMapCaptures<'a> { result } - pub fn grammars(&self) -> &[&'a Grammar] { + pub fn grammars(&self) -> &[&'a Arc] { &self.grammars } diff --git a/crates/language/src/syntax_map/syntax_map_tests.rs b/crates/language/src/syntax_map/syntax_map_tests.rs index 414bd1c6637..eb8b7cf49c1 100644 --- a/crates/language/src/syntax_map/syntax_map_tests.rs +++ b/crates/language/src/syntax_map/syntax_map_tests.rs @@ -1223,8 +1223,8 @@ fn test_random_syntax_map_edits_with_heex(rng: StdRng, cx: &mut App) { #[test] fn test_flatten_capture_regions_with_nested_captures() { - let outer = CaptureId(1); - let inner = CaptureId(2); + let outer = capture_ref(1); + let inner = capture_ref(2); assert_eq!( flattened(0..12, &[(0..10, outer), (2..5, inner)]), vec![ @@ -1237,8 +1237,8 @@ fn test_flatten_capture_regions_with_nested_captures() { #[test] fn test_flatten_capture_regions_with_overlapping_captures() { - let first = CaptureId(1); - let second = CaptureId(2); + let first = capture_ref(1); + let second = capture_ref(2); assert_eq!( flattened(0..25, &[(0..10, first), (2..20, second)]), vec![ @@ -1252,7 +1252,7 @@ fn test_flatten_capture_regions_with_overlapping_captures() { #[test] fn test_flatten_capture_regions_clips_to_the_requested_range() { - let capture = CaptureId(1); + let capture = capture_ref(1); assert_eq!( flattened(5..8, &[(0..10, capture)]), vec![(5..8, vec![capture])], @@ -1672,13 +1672,20 @@ fn comment_lang() -> Language { ) } +fn capture_ref(capture_id: u32) -> HighlightCaptureRef { + HighlightCaptureRef { + grammar_index: 0, + capture_id: CaptureId(capture_id), + } +} + fn flattened( range: Range, - captures: &[(Range, CaptureId)], -) -> Vec<(Range, Vec)> { + captures: &[(Range, HighlightCaptureRef)], +) -> Vec<(Range, Vec)> { flatten_capture_regions(range, captures.iter().cloned()) .into_iter() - .map(|region| (region.range, region.capture_ids.to_vec())) + .map(|region| (region.range, region.stack.to_vec())) .collect() } diff --git a/crates/language_core/src/highlight_cache.rs b/crates/language_core/src/highlight_cache.rs index 9fa7efe185e..77f405a23f0 100644 --- a/crates/language_core/src/highlight_cache.rs +++ b/crates/language_core/src/highlight_cache.rs @@ -1,16 +1,20 @@ -use crate::highlight_map::CapturedRange; +use crate::grammar::Grammar; +use crate::highlight_map::{CaptureId, CapturedRange}; use collections::FxHasher; use lru::LruCache; use parking_lot::Mutex; use smallvec::{Array, SmallVec}; use std::{ + fmt, hash::{Hash, Hasher as _}, + ops::Range, sync::Arc, }; pub const MAX_TEXT_CAPTURES_ENTRY_BYTES: usize = MAX_TEXT_CAPTURES_CACHE_BYTES / 8; const MAX_TEXT_CAPTURES_CACHE_BYTES: usize = 4 * 1024 * 1024; +const MAX_CHUNK_HIGHLIGHT_CACHE_BYTES: usize = 10 * 1024 * 1024; const APPROXIMATE_LRU_NODE_BYTES: usize = 4 * size_of::(); #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -82,6 +86,62 @@ impl TextHighlightCache { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HighlightCaptureRef { + pub grammar_index: usize, + pub capture_id: CaptureId, +} + +#[derive(Clone, Debug)] +pub struct ChunkCaptureRun { + pub range: Range, + pub stack: SmallVec<[HighlightCaptureRef; 4]>, +} + +#[derive(Clone)] +pub struct ChunkCaptures { + pub grammars: SmallVec<[Arc; 2]>, + pub runs: Arc<[ChunkCaptureRun]>, +} + +pub struct ChunkHighlightCache(Mutex>); + +impl Default for ChunkHighlightCache { + fn default() -> Self { + Self(Mutex::new(CostBudgetedLru::new( + MAX_CHUNK_HIGHLIGHT_CACHE_BYTES, + MAX_CHUNK_HIGHLIGHT_CACHE_BYTES, + ))) + } +} + +impl fmt::Debug for ChunkHighlightCache { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ChunkHighlightCache") + .finish_non_exhaustive() + } +} + +impl ChunkHighlightCache { + pub fn get(&self, chunk_id: usize) -> Option { + self.0.lock().get(&chunk_id).cloned() + } + + pub fn insert(&self, chunk_id: usize, captures: ChunkCaptures) { + let cost = captures.grammars.len() * size_of::>() + + captures + .runs + .iter() + .map(|run| size_of::() + small_vec_heap_bytes(&run.stack)) + .sum::(); + self.0.lock().insert(chunk_id, captures, cost); + } + + pub fn clear(&mut self) { + self.0.get_mut().clear(); + } +} + struct CostBudgetedLru { entries: LruCache, total_cost: usize, @@ -122,6 +182,11 @@ impl CostBudgetedLru { self.total_cost -= evicted_cost; } } + + fn clear(&mut self) { + self.entries.clear(); + self.total_cost = 0; + } } struct TextCapturesEntry { @@ -151,8 +216,6 @@ fn chunks_match_text<'a>(text: &str, text_chunks: impl Iterator) #[cfg(test)] mod tests { use super::*; - use crate::highlight_map::CaptureId; - use std::ops::Range; #[test] fn test_budget_evicts_least_recently_used() { @@ -192,6 +255,17 @@ mod tests { assert_eq!(cache.total_cost, 40 + overhead); } + #[test] + fn test_clear_resets_the_budget() { + let overhead = CostBudgetedLru::<&str, u32>::ENTRY_OVERHEAD_BYTES; + let mut cache = CostBudgetedLru::<&str, u32>::new(90 + overhead, 100); + cache.insert("a", 1, 90); + cache.clear(); + assert_eq!(cache.total_cost, 0); + cache.insert("b", 2, 90); + assert_eq!(cache.get(&"b"), Some(&2)); + } + #[test] fn test_zero_cost_entries_still_consume_budget() { let overhead = CostBudgetedLru::::ENTRY_OVERHEAD_BYTES;