vrr/zed
2
0
Fork 0
mirror of https://github.com/zed-industries/zed.git synced 2026-08-25 08:53:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 20

language: Cache buffer syntax highlighting per row chunk

Browse source 
BufferSnapshot::chunks re-ran the tree-sitter highlight query over the
requested range on every call, so scrolling and cursor movement paid the full
query cost per frame.

Highlight captures are now computed once per 50-row chunk (the same RowChunks
that back bracket colorization), flattened into runs of capture-id stacks, and
kept in a cost-budgeted per-buffer LRU (10MB). Runs resolve through the
current HighlightMap at read time, so theme changes need no invalidation;
edits and reparses drop the cache through the existing TreeSitterData swap.
Chunks spanning more than 64KB bypass the cache and fall back to direct
querying, so files with giant lines behave as before.

New editor_render_highlighted and editor_render_highlighted_minimap benches
move the cursor through a generated 10K-line Rust file with real highlighting,
with and without the minimap; ZED_DISABLE_HIGHLIGHT_CACHE=1 routes rendering
through the pre-existing direct-query path for comparison.
This commit is contained in:
Kirill Bulatov 2026-08-21 12:57:37 +02:00
parent 5706e6ccd6
commit 9258c00626
8 changed files with 755 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

83
crates/benchmarks/benches/editor_render.rs
View file

@ -1,10 +1,13 @@
use benchmarks::bench_utils::random_rust_file;
use editor::{
Editor, EditorMode, MultiBuffer,
actions::{DeleteToPreviousWordStart, SelectAll, SplitSelectionIntoLines},
};
use gpui::{AppContext as _, BenchAppContext, Focusable as _};
use gpui::{AppContext as _, BenchAppContext, Focusable as _, UpdateGlobal as _};
use language::{Buffer, Rope};
use rand::{Rng as _, SeedableRng as _, rngs::StdRng};
use settings::SettingsStore;
use settings::{DisplayIn, SettingsStore, ShowMinimap};
use theme::ActiveTheme as _;
use util::RandomCharIter;
use zed_actions::editor::{MoveDown, MoveUp};
@ -124,6 +127,78 @@ fn editor_render(cx: &mut BenchAppContext) {
});
}
#[gpui::bench]
fn editor_render_highlighted(cx: &mut BenchAppContext) {
init_context(cx);
render_highlighted_editor(cx);
}
#[gpui::bench]
fn editor_render_highlighted_minimap(cx: &mut BenchAppContext) {
init_context(cx);
cx.update(|cx| {
SettingsStore::update_global(cx, |store, cx| {
store.update_user_settings(cx, |settings| {
let minimap = settings.editor.minimap.get_or_insert_default();
minimap.show = Some(ShowMinimap::Always);
minimap.display_in = Some(DisplayIn::AllEditors);
});
});
});
render_highlighted_editor(cx);
}
fn render_highlighted_editor(cx: &mut BenchAppContext) {
let mut rng = StdRng::seed_from_u64(1);
let text = random_rust_file(&mut rng, 10_000).join("\n");
let language = language::rust_lang();
let syntax_theme = cx.update(|cx| {
let syntax_theme = cx.theme().syntax().clone();
language.set_theme(&syntax_theme);
syntax_theme
});
let probe = "fn main() {}";
assert!(
!language
.highlight_text(&Rope::from(probe), 0..probe.len())
.is_empty(),
"the benchmark language must resolve syntax highlights against the theme"
);
let buffer = cx.update(|cx| {
let buffer = cx.new(|cx| Buffer::local(text, cx).with_language(language, cx));
cx.new(|cx| MultiBuffer::singleton(buffer, cx))
});
cx.run_until_idle();
let mut window = cx.add_empty_window();
let editor = window.update(|window, cx| {
let editor = window.replace_root(cx, |window, cx| {
let mut editor = Editor::new(EditorMode::full(), buffer, None, window, cx);
editor.set_style(
editor::EditorStyle {
syntax: syntax_theme.clone(),
..editor::EditorStyle::default()
},
window,
cx,
);
editor
});
window.focus(&editor.focus_handle(cx), cx);
editor
});
let mut move_down = true;
cx.bench_renderer(editor, move |editor, window, cx| {
if move_down {
editor.move_down(&MoveDown, window, cx);
} else {
editor.move_up(&MoveUp, window, cx);
}
move_down = !move_down;
});
}
fn init_context(cx: &mut BenchAppContext) {
cx.update(|cx| {
let store = SettingsStore::test(cx);
@ -146,6 +221,8 @@ gpui::bench_group!(
benches,
editor_multi_cursor_input,
open_editor_with_one_long_line,
editor_render
editor_render,
editor_render_highlighted,
editor_render_highlighted_minimap
);
gpui::bench_main!(benches);

223
crates/language/src/buffer.rs
View file

@ -9,11 +9,12 @@ use crate::{
diagnostic_set::{DiagnosticEntry, DiagnosticEntryRef, DiagnosticGroup},
language_settings::{AutoIndentMode, LanguageSettings},
outline::OutlineItem,
row_chunk::RowChunks,
row_chunk::{RowChunkId, RowChunks},
runnable::{self, RunnableRange},
syntax_map::{
MAX_BYTES_TO_QUERY, SyntaxLayer, SyntaxMap, SyntaxMapCapture, SyntaxMapCaptures,
SyntaxMapMatch, SyntaxMapMatches, SyntaxSnapshot, ToTreeSitterPoint,
flattened_highlight_regions,
},
text_diff::text_diff,
unified_diff_with_offsets,
@ -35,6 +36,7 @@ use gpui::{
App, AppContext as _, Context, Entity, EventEmitter, HighlightStyle, SharedString, StyledText,
Task, TextStyle,
};
use language_core::highlight_cache::{ChunkCaptureRun, ChunkCaptures, ChunkHighlightCache};
use lsp::LanguageServerId;
use parking_lot::Mutex;
@ -147,21 +149,25 @@ pub struct Buffer {
#[derive(Debug)]
pub struct TreeSitterData {
chunks: RowChunks,
brackets_by_chunks: Mutex<HashMap<usize, Vec<BracketMatch>>>,
brackets_by_chunks: Mutex<HashMap<RowChunkId, Vec<BracketMatch>>>,
highlights_by_chunks: ChunkHighlightCache,
}
const MAX_ROWS_IN_A_CHUNK: u32 = 50;
pub(crate) const MAX_ROWS_IN_A_CHUNK: u32 = 50;
pub(crate) const MAX_BYTES_TO_HIGHLIGHT_IN_A_CHUNK: usize = 4 * MAX_BYTES_TO_QUERY;
impl TreeSitterData {
fn clear(&mut self, snapshot: &text::BufferSnapshot) {
self.chunks = RowChunks::new(snapshot, MAX_ROWS_IN_A_CHUNK);
self.brackets_by_chunks.get_mut().clear();
self.highlights_by_chunks.clear();
}
fn new(snapshot: &text::BufferSnapshot) -> Self {
Self {
chunks: RowChunks::new(snapshot, MAX_ROWS_IN_A_CHUNK),
brackets_by_chunks: Mutex::new(HashMap::default()),
highlights_by_chunks: ChunkHighlightCache::default(),
}
}
@ -514,6 +520,13 @@ struct BufferChunkHighlights<'a> {
highlight_maps: Vec<HighlightMap>,
}
type HighlightRun = (Range<usize>, HighlightId);
struct CachedChunkHighlightsIter {
runs: Vec<HighlightRun>,
ix: usize,
}
/// An iterator that yields chunks of a buffer's text, along with their
/// syntax highlights and diagnostic status.
pub struct BufferChunks<'a> {
@ -528,6 +541,7 @@ pub struct BufferChunks<'a> {
unnecessary_depth: usize,
underline: bool,
highlights: Option<BufferChunkHighlights<'a>>,
cached_highlights: Option<CachedChunkHighlightsIter>,
}
/// A chunk of a buffer's text, along with its syntax highlight and
@ -1496,6 +1510,7 @@ impl Buffer {
}
self.non_text_state_update_count += 1;
self.syntax_map.lock().clear(&self.text);
Self::invalidate_tree_sitter_data(&mut self.tree_sitter_data, self.text.snapshot());
let old_language = std::mem::replace(&mut self.language, language);
self.was_changed();
self.reparse(cx, may_block);
@ -3980,7 +3995,18 @@ impl BufferSnapshot {
let mut syntax = None;
if language_aware.tree_sitter {
syntax = Some(self.get_highlights(range.clone()));
match self.cached_highlight_runs(range.clone()) {
Some(runs) => {
return BufferChunks::with_cached_highlights(
self.text.as_rope(),
range,
runs,
language_aware.diagnostics,
self,
);
}
None => syntax = Some(self.get_highlights(range.clone())),
}
}
BufferChunks::new(
self.text.as_rope(),
@ -3991,6 +4017,100 @@ impl BufferSnapshot {
)
}
pub(crate) fn cached_highlight_runs(&self, range: Range<usize>) -> Option<Vec<HighlightRun>> {
#[cfg(any(test, feature = "test-support"))]
{
static DISABLE_HIGHLIGHT_CACHE: std::sync::LazyLock<bool> =
std::sync::LazyLock::new(|| {
std::env::var_os("ZED_DISABLE_HIGHLIGHT_CACHE").is_some()
});
if *DISABLE_HIGHLIGHT_CACHE {
return None;
}
}
self.language.as_ref()?.grammar()?;
if range.is_empty() {
return Some(Vec::new());
}
let mut runs = Vec::<HighlightRun>::new();
for chunk in self
.tree_sitter_data
.chunks
.applicable_chunks(&[range.to_point(self)])
{
let chunk_range = chunk.anchor_range().to_offset(self);
if chunk_range.end <= range.start || chunk_range.start >= range.end {
continue;
}
if chunk_range.len() > MAX_BYTES_TO_HIGHLIGHT_IN_A_CHUNK {
return None;
}
let chunk_captures = match self.tree_sitter_data.highlights_by_chunks.get(chunk.id) {
Some(chunk_captures) => chunk_captures,
None => {
let chunk_captures = self.compute_chunk_captures(chunk_range);
self.tree_sitter_data
.highlights_by_chunks
.insert(chunk.id, chunk_captures.clone());
chunk_captures
}
};
let highlight_maps = chunk_captures
.grammars
.iter()
.map(|grammar| grammar.highlight_map())
.collect::<SmallVec<[HighlightMap; 2]>>();
for run in chunk_captures.runs.iter() {
if run.range.end <= range.start {
continue;
}
if run.range.start >= range.end {
break;
}
let highlight_id = run.stack.iter().rev().find_map(|capture| {
highlight_maps
.get(capture.grammar_index)?
.get(capture.capture_id)
});
let Some(highlight_id) = highlight_id else {
continue;
};
match runs.last_mut() {
Some((last_range, last_highlight_id))
if *last_highlight_id == highlight_id
&& last_range.end == run.range.start =>
{
last_range.end = run.range.end;
}
_ => runs.push((run.range.clone(), highlight_id)),
}
}
}
Some(runs)
}
fn compute_chunk_captures(&self, range: Range<usize>) -> ChunkCaptures {
let captures = self.syntax.captures(range.clone(), &self.text, |grammar| {
grammar
.highlights_config
.as_ref()
.map(|config| &config.query)
});
let grammars = captures
.grammars()
.iter()
.map(|&grammar| grammar.clone())
.collect();
let runs = flattened_highlight_regions(captures, range)
.into_iter()
.map(|region| ChunkCaptureRun {
range: region.range,
stack: region.stack,
})
.collect();
ChunkCaptures { grammars, runs }
}
pub fn highlighted_text_for_range<T: ToOffset>(
&self,
range: Range<T>,
@ -5343,16 +5463,40 @@ impl<'a> BufferChunks<'a> {
diagnostics: bool,
buffer_snapshot: Option<&'a BufferSnapshot>,
) -> Self {
let mut highlights = None;
if let Some((captures, highlight_maps)) = syntax {
highlights = Some(BufferChunkHighlights {
captures,
next_capture: None,
stack: Default::default(),
highlight_maps,
})
}
let highlights = syntax.map(|(captures, highlight_maps)| BufferChunkHighlights {
captures,
next_capture: None,
stack: Vec::new(),
highlight_maps,
});
Self::init(text, range, highlights, None, diagnostics, buffer_snapshot)
}
fn with_cached_highlights(
text: &'a Rope,
range: Range<usize>,
runs: Vec<HighlightRun>,
diagnostics: bool,
buffer_snapshot: &'a BufferSnapshot,
) -> Self {
Self::init(
text,
range,
None,
Some(CachedChunkHighlightsIter { runs, ix: 0 }),
diagnostics,
Some(buffer_snapshot),
)
}
fn init(
text: &'a Rope,
range: Range<usize>,
highlights: Option<BufferChunkHighlights<'a>>,
cached_highlights: Option<CachedChunkHighlightsIter>,
diagnostics: bool,
buffer_snapshot: Option<&'a BufferSnapshot>,
) -> Self {
let diagnostic_endpoints = diagnostics.then(|| Vec::new().into_iter().peekable());
let chunks = text.chunks_in_range(range.clone());
@ -5368,6 +5512,7 @@ impl<'a> BufferChunks<'a> {
unnecessary_depth: 0,
underline: true,
highlights,
cached_highlights,
};
this.initialize_diagnostic_endpoints();
this
@ -5377,7 +5522,33 @@ impl<'a> BufferChunks<'a> {
pub fn seek(&mut self, range: Range<usize>) {
let old_range = std::mem::replace(&mut self.range, range.clone());
self.chunks.set_range(self.range.clone());
if let Some(highlights) = self.highlights.as_mut() {
if let Some(cached) = self.cached_highlights.as_mut() {
if old_range.start <= self.range.start && old_range.end >= self.range.end {
cached.ix = cached
.runs
.partition_point(|(run_range, _)| run_range.end <= range.start);
} else if let Some(snapshot) = self.buffer_snapshot {
if let Some(runs) = snapshot.cached_highlight_runs(self.range.clone()) {
cached.runs = runs;
cached.ix = 0;
} else {
let (captures, highlight_maps) = snapshot.get_highlights(self.range.clone());
self.cached_highlights = None;
self.highlights = Some(BufferChunkHighlights {
captures,
next_capture: None,
stack: Vec::new(),
highlight_maps,
});
}
} else {
debug_assert!(
false,
"Attempted to seek on a language-aware buffer iterator without associated buffer snapshot"
);
}
self.initialize_diagnostic_endpoints();
} else if let Some(highlights) = self.highlights.as_mut() {
if old_range.start <= self.range.start && old_range.end >= self.range.end {
// Reuse existing highlights stack, as the new range is a subrange of the old one.
highlights
@ -5400,7 +5571,7 @@ impl<'a> BufferChunks<'a> {
*highlights = BufferChunkHighlights {
captures,
next_capture: None,
stack: Default::default(),
stack: Vec::new(),
highlight_maps,
};
} else {
@ -5536,6 +5707,21 @@ impl<'a> Iterator for BufferChunks<'a> {
}
}
if let Some(cached) = self.cached_highlights.as_mut() {
while cached
.runs
.get(cached.ix)
.is_some_and(|(run_range, _)| run_range.end <= self.range.start)
{
cached.ix += 1;
}
if let Some((run_range, _)) = cached.runs.get(cached.ix)
&& self.range.start < run_range.start
{
next_capture_start = run_range.start;
}
}
let mut diagnostic_endpoints = std::mem::take(&mut self.diagnostic_endpoints);
if let Some(diagnostic_endpoints) = diagnostic_endpoints.as_mut() {
while let Some(endpoint) = diagnostic_endpoints.peek().copied() {
@ -5569,6 +5755,13 @@ impl<'a> Iterator for BufferChunks<'a> {
chunk_end = chunk_end.min(*parent_capture_end);
highlight_id = Some(*parent_highlight_id);
}
if let Some(cached) = self.cached_highlights.as_ref()
&& let Some((run_range, run_highlight_id)) = cached.runs.get(cached.ix)
&& run_range.start <= chunk_start
{
chunk_end = chunk_end.min(run_range.end);
highlight_id = Some(*run_highlight_id);
}
let bit_start = chunk_start - self.chunks.offset();
let bit_end = chunk_end - self.chunks.offset();

20
crates/language/src/buffer/row_chunk.rs
View file

@ -10,6 +10,8 @@ use util::RangeExt;
use crate::BufferRow;
pub type RowChunkId = usize;
/// An range of rows, exclusive as [`lsp::Range`] and
/// <https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#range>
/// denote.
@ -27,7 +29,7 @@ pub struct RowChunks {
buffer_snapshot: text::BufferSnapshot,
last_row: BufferRow,
max_rows_per_chunk: u32,
computed_chunks: Mutex<HashMap<usize, RowChunk>>,
computed_chunks: Mutex<HashMap<RowChunkId, RowChunk>>,
}
impl std::fmt::Debug for RowChunks {
@ -86,12 +88,18 @@ impl RowChunks {
}
}
fn chunk_row_range(&self, id: usize) -> Range<BufferRow> {
let start = id as u32 * self.max_rows_per_chunk;
start..(start + self.max_rows_per_chunk).min(self.last_row)
fn chunk_row_range(&self, id: RowChunkId) -> Range<BufferRow> {
let start = u32::try_from(id)
.unwrap_or(u32::MAX)
.saturating_mul(self.max_rows_per_chunk)
.min(self.last_row);
start
..start
.saturating_add(self.max_rows_per_chunk)
.min(self.last_row)
}
fn chunk(&self, id: usize) -> RowChunk {
fn chunk(&self, id: RowChunkId) -> RowChunk {
*self.computed_chunks.lock().entry(id).or_insert_with(|| {
let row_range = self.chunk_row_range(id);
let start = Point::new(row_range.start, 0);
@ -113,7 +121,7 @@ impl RowChunks {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct RowChunk {
pub id: usize,
pub id: RowChunkId,
pub start: BufferRow,
pub end_exclusive: BufferRow,
pub start_anchor: Anchor,

323
crates/language/src/buffer_tests.rs
View file

@ -5161,6 +5161,269 @@ fn init_settings(cx: &mut App, f: fn(&mut AllLanguageSettingsContent)) {
});
}
#[gpui::test]
fn test_chunk_highlights_follow_edits_and_theme_changes(cx: &mut TestAppContext) {
cx.update(|cx| init_settings(cx, |_| {}));
let language = keyword_and_function_lang();
let theme_with_keyword = keyword_and_function_theme();
let theme_without_keyword =
SyntaxTheme::new([("function".to_string(), gpui::rgba(0x0000ffff).into())]);
language.set_theme(&theme_with_keyword);
let buffer = cx.new(|cx| {
let mut buffer = Buffer::local("fn main() {}", cx);
buffer.set_language(Some(language.clone()), cx);
buffer
});
cx.run_until_parked();
let highlighted_chunks = |snapshot: &BufferSnapshot| {
snapshot
.chunks(
0..snapshot.len(),
LanguageAwareStyling {
tree_sitter: true,
diagnostics: false,
},
)
.filter_map(|chunk| Some((chunk.text.to_string(), chunk.syntax_highlight_id?)))
.collect::<Vec<_>>()
};
let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
let expected_with_keyword = vec![
(
"fn".to_string(),
theme_highlight_id(&theme_with_keyword, "keyword"),
),
(
"main".to_string(),
theme_highlight_id(&theme_with_keyword, "function"),
),
];
assert_eq!(highlighted_chunks(&snapshot), expected_with_keyword);
assert_eq!(
highlighted_chunks(&snapshot),
expected_with_keyword,
"repeated chunking of the same snapshot must return the same highlights"
);
language.set_theme(&theme_without_keyword);
assert_eq!(
highlighted_chunks(&snapshot),
vec![(
"main".to_string(),
theme_highlight_id(&theme_without_keyword, "function")
)],
"a theme change must invalidate highlights of an existing snapshot"
);
buffer.update(cx, |buffer, cx| {
buffer.edit([(3..7, "launch")], None, cx);
});
cx.run_until_parked();
let edited_snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
assert_eq!(
highlighted_chunks(&edited_snapshot),
vec![(
"launch".to_string(),
theme_highlight_id(&theme_without_keyword, "function")
)],
"an edit must invalidate the cached highlights"
);
}
#[gpui::test]
fn test_chunk_highlights_across_row_chunk_seeks(cx: &mut TestAppContext) {
cx.update(|cx| init_settings(cx, |_| {}));
let language = keyword_and_function_lang();
let theme = keyword_and_function_theme();
language.set_theme(&theme);
let short_row = "fn short() {}\n";
let last_row = "fn omega() {}";
let text = format!(
"{}{last_row}",
short_row.repeat(MAX_ROWS_IN_A_CHUNK as usize)
);
let buffer = cx.new(|cx| {
let mut buffer = Buffer::local(text, cx);
buffer.set_language(Some(language.clone()), cx);
buffer
});
cx.run_until_parked();
let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
let keyword = theme_highlight_id(&theme, "keyword");
let function = theme_highlight_id(&theme, "function");
assert_eq!(
merged_highlight_runs(&snapshot, 0..short_row.len()),
vec![("fn".to_string(), keyword), ("short".to_string(), function)],
);
let last_row_start = short_row.len() * MAX_ROWS_IN_A_CHUNK as usize;
let expected_last_row_runs = vec![("fn".to_string(), keyword), ("omega".to_string(), function)];
assert_eq!(
merged_highlight_runs(&snapshot, last_row_start..snapshot.len()),
expected_last_row_runs,
);
let mut chunks = snapshot.chunks(
0..short_row.len(),
LanguageAwareStyling {
tree_sitter: true,
diagnostics: false,
},
);
chunks.seek(last_row_start..snapshot.len());
let mut runs_after_seek = Vec::new();
for chunk in chunks {
merge_highlighted_chunk(&mut runs_after_seek, &chunk);
}
assert_eq!(
runs_after_seek, expected_last_row_runs,
"seeking into another row chunk must refetch that chunk's highlights"
);
}
#[gpui::test]
fn test_oversized_chunks_bypass_the_highlight_cache(cx: &mut TestAppContext) {
if std::env::var_os("ZED_DISABLE_HIGHLIGHT_CACHE").is_some() {
return;
}
cx.update(|cx| init_settings(cx, |_| {}));
let language = keyword_and_function_lang();
let theme = keyword_and_function_theme();
language.set_theme(&theme);
let short_row = "fn short() {}\n";
let giant_row = format!(
"fn omega() {{}}{}",
" ".repeat(MAX_BYTES_TO_HIGHLIGHT_IN_A_CHUNK)
);
let text = format!(
"{}{giant_row}",
short_row.repeat(MAX_ROWS_IN_A_CHUNK as usize)
);
let buffer = cx.new(|cx| {
let mut buffer = Buffer::local(text, cx);
buffer.set_language(Some(language.clone()), cx);
buffer
});
cx.run_until_parked();
let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
let keyword = theme_highlight_id(&theme, "keyword");
let function = theme_highlight_id(&theme, "function");
let expected_giant_row_runs =
vec![("fn".to_string(), keyword), ("omega".to_string(), function)];
let last_row_start = short_row.len() * MAX_ROWS_IN_A_CHUNK as usize;
assert_eq!(
snapshot.cached_highlight_runs(last_row_start..snapshot.len()),
None,
"chunks larger than the byte cap must not be cached"
);
assert!(
snapshot.cached_highlight_runs(0..short_row.len()).is_some(),
"chunks within the byte cap must still be cached"
);
assert_eq!(
snapshot.cached_highlight_runs(last_row_start + 1..last_row_start + 1),
Some(Vec::new()),
"an empty range must not compute captures for its oversized chunk"
);
assert_eq!(
merged_highlight_runs(&snapshot, last_row_start..snapshot.len()),
expected_giant_row_runs,
"oversized chunks must fall back to direct highlighting"
);
let mut chunks = snapshot.chunks(
0..short_row.len(),
LanguageAwareStyling {
tree_sitter: true,
diagnostics: false,
},
);
chunks.seek(last_row_start..snapshot.len());
let mut runs_after_seek = Vec::new();
for chunk in chunks {
merge_highlighted_chunk(&mut runs_after_seek, &chunk);
}
assert_eq!(
runs_after_seek, expected_giant_row_runs,
"seeking into an oversized chunk must fall back to direct highlighting"
);
}
#[gpui::test]
fn test_language_change_invalidates_cached_chunk_highlights(cx: &mut TestAppContext) {
cx.update(|cx| init_settings(cx, |_| {}));
let rust = keyword_and_function_lang();
let identifiers_only = Arc::new(
Language::new(
LanguageConfig {
name: "Identifiers".into(),
..LanguageConfig::default()
},
Some(tree_sitter_rust::LANGUAGE.into()),
)
.with_highlights_query("(identifier) @variable")
.unwrap(),
);
let theme = SyntaxTheme::new([
("keyword".to_string(), gpui::rgba(0xff0000ff).into()),
("function".to_string(), gpui::rgba(0x00ff00ff).into()),
("variable".to_string(), gpui::rgba(0x0000ffff).into()),
]);
rust.set_theme(&theme);
identifiers_only.set_theme(&theme);
let buffer = cx.new(|cx| {
let mut buffer = Buffer::local("fn main() {}", cx);
buffer.set_language(Some(rust), cx);
buffer
});
cx.run_until_parked();
let keyword = theme_highlight_id(&theme, "keyword");
let function = theme_highlight_id(&theme, "function");
let variable = theme_highlight_id(&theme, "variable");
let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
assert_eq!(
merged_highlight_runs(&snapshot, 0..snapshot.len()),
vec![("fn".to_string(), keyword), ("main".to_string(), function)],
);
buffer.update(cx, |buffer, cx| {
buffer.set_sync_parse_timeout(None);
buffer.set_language(Some(identifiers_only), cx);
});
let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
assert_eq!(
merged_highlight_runs(&snapshot, 0..snapshot.len()),
Vec::new(),
"a language change must not serve the old language's cached highlights while the new parse is pending"
);
cx.run_until_parked();
let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
assert_eq!(
merged_highlight_runs(&snapshot, 0..snapshot.len()),
vec![("main".to_string(), variable)],
);
}
#[gpui::test(iterations = 100)]
fn test_random_chunk_bitmaps(cx: &mut App, mut rng: StdRng) {
use util::RandomCharIter;
@ -5288,3 +5551,63 @@ fn test_formatted_chunks(cx: &mut gpui::App) {
}
}
}
fn merged_highlight_runs(
snapshot: &BufferSnapshot,
range: Range<usize>,
) -> Vec<(String, HighlightId)> {
let chunks = snapshot.chunks(
range,
LanguageAwareStyling {
tree_sitter: true,
diagnostics: false,
},
);
let mut runs = Vec::new();
for chunk in chunks {
merge_highlighted_chunk(&mut runs, &chunk);
}
runs
}
fn merge_highlighted_chunk(runs: &mut Vec<(String, HighlightId)>, chunk: &Chunk<'_>) {
let Some(highlight_id) = chunk.syntax_highlight_id else {
return;
};
match runs.last_mut() {
Some((last_text, last_highlight_id)) if *last_highlight_id == highlight_id => {
last_text.push_str(chunk.text);
}
_ => runs.push((chunk.text.to_string(), highlight_id)),
}
}
fn keyword_and_function_lang() -> Arc<Language> {
Arc::new(
Language::new(
LanguageConfig {
name: "Rust".into(),
..LanguageConfig::default()
},
Some(tree_sitter_rust::LANGUAGE.into()),
)
.with_highlights_query(
r#"
"fn" @keyword
(identifier) @function
"#,
)
.unwrap(),
)
}
fn keyword_and_function_theme() -> SyntaxTheme {
SyntaxTheme::new([
("keyword".to_string(), gpui::rgba(0xff0000ff).into()),
("function".to_string(), gpui::rgba(0x00ff00ff).into()),
])
}
fn theme_highlight_id(theme: &SyntaxTheme, capture_name: &str) -> HighlightId {
HighlightId::new(theme.highlight_id(capture_name).unwrap())
}

12
crates/language/src/language.rs
View file

@ -1183,7 +1183,17 @@ impl Language {
.as_ref()
.map(|config| &config.query)
});
Arc::from(flattened_highlight_regions(captures, 0..text.len()))
flattened_highlight_regions(captures, 0..text.len())
.into_iter()
.map(|region| CapturedRange {
range: region.range,
capture_ids: region
.stack
.iter()
.map(|capture| capture.capture_id)
.collect(),
})
.collect()
}
pub fn path_suffixes(&self) -> &[String] {

41
crates/language/src/syntax_map.rs
View file

@ -2,12 +2,14 @@
mod syntax_map_tests;
use crate::{
CaptureId, CapturedRange, Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry,
QUERY_CURSORS, with_parser,
CaptureId, Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry, QUERY_CURSORS,
with_parser,
};
use collections::HashMap;
use futures::FutureExt;
use gpui::SharedString;
use language_core::highlight_cache::HighlightCaptureRef;
use smallvec::SmallVec;
use std::{
borrow::Cow,
cmp::{self, Ordering, Reverse},
@ -74,7 +76,7 @@ impl Drop for SyntaxSnapshot {
pub struct SyntaxMapCaptures<'a> {
layers: Vec<SyntaxMapCapturesLayer<'a>>,
active_layer_count: usize,
grammars: Vec<&'a Grammar>,
grammars: Vec<&'a Arc<Grammar>>,
}
#[derive(Default)]
@ -91,34 +93,45 @@ pub struct SyntaxMapCapture<'a> {
pub grammar_index: usize,
}
pub(crate) struct HighlightCaptureRegion {
pub range: Range<usize>,
pub stack: SmallVec<[HighlightCaptureRef; 4]>,
}
pub(crate) fn flattened_highlight_regions(
mut captures: SyntaxMapCaptures<'_>,
range: Range<usize>,
) -> Vec<CapturedRange> {
) -> Vec<HighlightCaptureRegion> {
let capture_refs = iter::from_fn(move || {
let capture = captures.next()?;
Some((capture.node.byte_range(), CaptureId(capture.index)))
Some((
capture.node.byte_range(),
HighlightCaptureRef {
grammar_index: capture.grammar_index,
capture_id: CaptureId(capture.index),
},
))
});
flatten_capture_regions(range, capture_refs)
}
fn flatten_capture_regions(
range: Range<usize>,
mut captures: impl Iterator<Item = (Range<usize>, CaptureId)>,
) -> Vec<CapturedRange> {
mut captures: impl Iterator<Item = (Range<usize>, HighlightCaptureRef)>,
) -> Vec<HighlightCaptureRegion> {
let mut result = Vec::new();
let mut stack = Vec::<(Range<usize>, CaptureId)>::new();
let mut stack = Vec::<(Range<usize>, HighlightCaptureRef)>::new();
let mut offset = range.start;
let mut next_capture = captures.next();
loop {
stack.retain(|(capture_range, _)| capture_range.end > offset);
while let Some((capture_range, capture_id)) = next_capture.take() {
while let Some((capture_range, capture_ref)) = next_capture.take() {
if capture_range.start > offset {
next_capture = Some((capture_range, capture_id));
next_capture = Some((capture_range, capture_ref));
break;
}
if capture_range.end > offset {
stack.push((capture_range, capture_id));
stack.push((capture_range, capture_ref));
}
next_capture = captures.next();
}
@ -134,9 +147,9 @@ fn flatten_capture_regions(
next_boundary = next_boundary.min(capture_range.start);
}
if !stack.is_empty() && next_boundary > offset {
result.push(CapturedRange {
result.push(HighlightCaptureRegion {
range: offset..next_boundary,
capture_ids: stack.iter().map(|(_, capture_id)| *capture_id).collect(),
stack: stack.iter().map(|(_, capture_ref)| *capture_ref).collect(),
});
}
if next_boundary >= range.end {
@ -1239,7 +1252,7 @@ impl<'a> SyntaxMapCaptures<'a> {
result
}
pub fn grammars(&self) -> &[&'a Grammar] {
pub fn grammars(&self) -> &[&'a Arc<Grammar>] {
&self.grammars
}

23
crates/language/src/syntax_map/syntax_map_tests.rs
View file

@ -1223,8 +1223,8 @@ fn test_random_syntax_map_edits_with_heex(rng: StdRng, cx: &mut App) {
#[test]
fn test_flatten_capture_regions_with_nested_captures() {
let outer = CaptureId(1);
let inner = CaptureId(2);
let outer = capture_ref(1);
let inner = capture_ref(2);
assert_eq!(
flattened(0..12, &[(0..10, outer), (2..5, inner)]),
vec![
@ -1237,8 +1237,8 @@ fn test_flatten_capture_regions_with_nested_captures() {
#[test]
fn test_flatten_capture_regions_with_overlapping_captures() {
let first = CaptureId(1);
let second = CaptureId(2);
let first = capture_ref(1);
let second = capture_ref(2);
assert_eq!(
flattened(0..25, &[(0..10, first), (2..20, second)]),
vec![
@ -1252,7 +1252,7 @@ fn test_flatten_capture_regions_with_overlapping_captures() {
#[test]
fn test_flatten_capture_regions_clips_to_the_requested_range() {
let capture = CaptureId(1);
let capture = capture_ref(1);
assert_eq!(
flattened(5..8, &[(0..10, capture)]),
vec![(5..8, vec![capture])],
@ -1672,13 +1672,20 @@ fn comment_lang() -> Language {
)
}
fn capture_ref(capture_id: u32) -> HighlightCaptureRef {
HighlightCaptureRef {
grammar_index: 0,
capture_id: CaptureId(capture_id),
}
}
fn flattened(
range: Range<usize>,
captures: &[(Range<usize>, CaptureId)],
) -> Vec<(Range<usize>, Vec<CaptureId>)> {
captures: &[(Range<usize>, HighlightCaptureRef)],
) -> Vec<(Range<usize>, Vec<HighlightCaptureRef>)> {
flatten_capture_regions(range, captures.iter().cloned())
.into_iter()
.map(|region| (region.range, region.capture_ids.to_vec()))
.map(|region| (region.range, region.stack.to_vec()))
.collect()
}

80
crates/language_core/src/highlight_cache.rs
View file

@ -1,16 +1,20 @@
use crate::highlight_map::CapturedRange;
use crate::grammar::Grammar;
use crate::highlight_map::{CaptureId, CapturedRange};
use collections::FxHasher;
use lru::LruCache;
use parking_lot::Mutex;
use smallvec::{Array, SmallVec};
use std::{
fmt,
hash::{Hash, Hasher as _},
ops::Range,
sync::Arc,
};
pub const MAX_TEXT_CAPTURES_ENTRY_BYTES: usize = MAX_TEXT_CAPTURES_CACHE_BYTES / 8;
const MAX_TEXT_CAPTURES_CACHE_BYTES: usize = 4 * 1024 * 1024;
const MAX_CHUNK_HIGHLIGHT_CACHE_BYTES: usize = 10 * 1024 * 1024;
const APPROXIMATE_LRU_NODE_BYTES: usize = 4 * size_of::<usize>();
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
@ -82,6 +86,62 @@ impl TextHighlightCache {
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct HighlightCaptureRef {
pub grammar_index: usize,
pub capture_id: CaptureId,
}
#[derive(Clone, Debug)]
pub struct ChunkCaptureRun {
pub range: Range<usize>,
pub stack: SmallVec<[HighlightCaptureRef; 4]>,
}
#[derive(Clone)]
pub struct ChunkCaptures {
pub grammars: SmallVec<[Arc<Grammar>; 2]>,
pub runs: Arc<[ChunkCaptureRun]>,
}
pub struct ChunkHighlightCache(Mutex<CostBudgetedLru<usize, ChunkCaptures>>);
impl Default for ChunkHighlightCache {
fn default() -> Self {
Self(Mutex::new(CostBudgetedLru::new(
MAX_CHUNK_HIGHLIGHT_CACHE_BYTES,
MAX_CHUNK_HIGHLIGHT_CACHE_BYTES,
)))
}
}
impl fmt::Debug for ChunkHighlightCache {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ChunkHighlightCache")
.finish_non_exhaustive()
}
}
impl ChunkHighlightCache {
pub fn get(&self, chunk_id: usize) -> Option<ChunkCaptures> {
self.0.lock().get(&chunk_id).cloned()
}
pub fn insert(&self, chunk_id: usize, captures: ChunkCaptures) {
let cost = captures.grammars.len() * size_of::<Arc<Grammar>>()
+ captures
.runs
.iter()
.map(|run| size_of::<ChunkCaptureRun>() + small_vec_heap_bytes(&run.stack))
.sum::<usize>();
self.0.lock().insert(chunk_id, captures, cost);
}
pub fn clear(&mut self) {
self.0.get_mut().clear();
}
}
struct CostBudgetedLru<K: Hash + Eq, V> {
entries: LruCache<K, (V, usize)>,
total_cost: usize,
@ -122,6 +182,11 @@ impl<K: Hash + Eq, V> CostBudgetedLru<K, V> {
self.total_cost -= evicted_cost;
}
}
fn clear(&mut self) {
self.entries.clear();
self.total_cost = 0;
}
}
struct TextCapturesEntry {
@ -151,8 +216,6 @@ fn chunks_match_text<'a>(text: &str, text_chunks: impl Iterator<Item = &'a str>)
#[cfg(test)]
mod tests {
use super::*;
use crate::highlight_map::CaptureId;
use std::ops::Range;
#[test]
fn test_budget_evicts_least_recently_used() {
@ -192,6 +255,17 @@ mod tests {
assert_eq!(cache.total_cost, 40 + overhead);
}
#[test]
fn test_clear_resets_the_budget() {
let overhead = CostBudgetedLru::<&str, u32>::ENTRY_OVERHEAD_BYTES;
let mut cache = CostBudgetedLru::<&str, u32>::new(90 + overhead, 100);
cache.insert("a", 1, 90);
cache.clear();
assert_eq!(cache.total_cost, 0);
cache.insert("b", 2, 90);
assert_eq!(cache.get(&"b"), Some(&2));
}
#[test]
fn test_zero_cost_entries_still_consume_budget() {
let overhead = CostBudgetedLru::<usize, u32>::ENTRY_OVERHEAD_BYTES;