From 7998d6450126261406f75ba65d3b5e74cf012780 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Wed, 10 Sep 2025 00:11:53 +0800 Subject: [PATCH] highlighter: Improve Syntax Highlighter performance. (#1230) Now can support up to 50K lines with code highlight. https://github.com/user-attachments/assets/3bbba2cd-ea56-4778-b471-6beea323fbaa ---- Ref Links - https://zed.dev/blog/syntax-aware-editing --- crates/ui/src/highlighter/highlighter.rs | 107 +++++++++-------------- crates/ui/src/input/element.rs | 2 +- crates/ui/src/input/mode.rs | 2 +- crates/ui/src/input/state.rs | 1 + crates/ui/src/text/node.rs | 4 +- 5 files changed, 44 insertions(+), 72 deletions(-) diff --git a/crates/ui/src/highlighter/highlighter.rs b/crates/ui/src/highlighter/highlighter.rs index 746edf2f..d72325c9 100644 --- a/crates/ui/src/highlighter/highlighter.rs +++ b/crates/ui/src/highlighter/highlighter.rs @@ -10,7 +10,6 @@ use std::{ ops::Range, usize, }; -use sum_tree::{Bias, SumTree}; use tree_sitter::{ InputEdit, Node, Parser, Point, Query, QueryCursor, QueryMatch, StreamingIterator, Tree, }; @@ -22,9 +21,6 @@ pub struct SyntaxHighlighter { language: SharedString, query: Option, injection_queries: HashMap, - parser: Parser, - old_tree: Option, - text: Rope, locals_pattern_index: usize, highlights_pattern_index: usize, @@ -37,8 +33,11 @@ pub struct SyntaxHighlighter { local_def_value_capture_index: Option, local_ref_capture_index: Option, - /// Cache of highlight, the range is offset of the token in the tree. - cache: SumTree, + /// The last parsed source text. + text: Rope, + parser: Parser, + /// The last parsed tree. + tree: Option, } struct TextProvider<'a>(&'a Rope); @@ -274,10 +273,7 @@ impl SyntaxHighlighter { language: config.name.clone(), query: Some(query), injection_queries, - parser, - old_tree: None, - text: Rope::new(), - cache: sum_tree::SumTree::new(&()), + locals_pattern_index, highlights_pattern_index, non_local_variable_patterns, @@ -287,6 +283,9 @@ impl SyntaxHighlighter { local_def_capture_index, local_def_value_capture_index, local_ref_capture_index, + text: Rope::new(), + parser, + tree: None, }) } @@ -295,8 +294,9 @@ impl SyntaxHighlighter { } /// Highlight the given text, returning a map from byte ranges to highlight captures. - /// Uses incremental parsing, detects changed ranges, and caches unchanged results. - pub fn update(&mut self, edit: Option, text: &Rope, cx: &App) { + /// + /// Uses incremental parsing by `edit` to efficiently update the highlighter's state. + pub fn update(&mut self, edit: Option, text: &Rope) { if self.text.eq(text) { return; } @@ -311,7 +311,7 @@ impl SyntaxHighlighter { }); let mut old_tree = self - .old_tree + .tree .take() .unwrap_or(self.parser.parse("", None).unwrap()); old_tree.edit(&edit); @@ -330,38 +330,26 @@ impl SyntaxHighlighter { return; }; - // let changed_ranges = new_tree.changed_ranges(&old_tree); - - // Update state - self.old_tree = Some(new_tree); + self.tree = Some(new_tree); self.text = text.clone(); - - // let measure = crate::Measure::new("build_styles"); - self.build_styles(cx); - // measure.end(); } - /// NOTE: 10K lines, about 180ms - /// FIXME: To improve the performance when there more than 5K lines, use partial update. - /// Ref: https://github.com/longbridge/gpui-component/pull/1197 - fn build_styles(&mut self, cx: &App) { - let Some(tree) = &self.old_tree else { - return; + /// Match the visible ranges of nodes in the Tree for highlighting. + fn match_styles(&self, range: Range, cx: &App) -> Vec { + let mut highlights = vec![]; + let Some(tree) = &self.tree else { + return highlights; }; let Some(query) = &self.query else { - return; + return highlights; }; let root_node = tree.root_node(); - // Remove the changed items from the cache. - let new_cache = sum_tree::SumTree::new(&()); - self.cache = new_cache; - - let source = self.text.clone(); - + let source = &self.text; let mut cursor = QueryCursor::new(); + cursor.set_byte_range(range); let mut matches = cursor.matches(&query, root_node, TextProvider(&source)); while let Some(query_match) = matches.next() { @@ -372,8 +360,7 @@ impl SyntaxHighlighter { { let styles = self.handle_injection(&language_name, content_node, cx); for (node_range, highlight_name) in styles { - self.cache - .push(HighlightItem::new(node_range.clone(), highlight_name), &()); + highlights.push(HighlightItem::new(node_range.clone(), highlight_name)); } continue; @@ -390,42 +377,37 @@ impl SyntaxHighlighter { let highlight_name = SharedString::from(highlight_name.to_string()); // Merge near range and same highlight name - let last_item = self.cache.last(); + let last_item = highlights.last(); let last_range = last_item.map(|item| &item.range).unwrap_or(&(0..0)); let last_highlight_name = last_item.map(|item| item.name.clone()); if last_range.end <= node_range.start && last_highlight_name.as_ref() == Some(&highlight_name) { - self.cache.push( - HighlightItem::new( - last_range.start..node_range.end, - highlight_name.clone(), - ), - &(), - ); + highlights.push(HighlightItem::new( + last_range.start..node_range.end, + highlight_name.clone(), + )); } else if last_range == &node_range { // case: // last_range: 213..220, last_highlight_name: Some("property") // last_range: 213..220, last_highlight_name: Some("string") - self.cache.push( - HighlightItem::new( - node_range, - last_highlight_name.unwrap_or(highlight_name), - ), - &(), - ); + highlights.push(HighlightItem::new( + node_range, + last_highlight_name.unwrap_or(highlight_name), + )); } else { - self.cache - .push(HighlightItem::new(node_range, highlight_name.clone()), &()); + highlights.push(HighlightItem::new(node_range, highlight_name.clone())); } } } // DO NOT REMOVE THIS PRINT, it's useful for debugging - // for item in self.cache.iter() { + // for item in highlights { // println!("item: {:?}", item); // } + + highlights } /// TODO: Use incremental parsing to handle the injection. @@ -570,25 +552,15 @@ impl SyntaxHighlighter { &self, range: &Range, theme: &HighlightTheme, + cx: &App, ) -> Vec<(Range, HighlightStyle)> { let mut styles = vec![]; let start_offset = range.start; - let mut cursor = self.cache.cursor::(&()); - let bias = if start_offset == 0 { - Bias::Right - } else { - Bias::Left - }; - - let left_items = cursor.slice(&start_offset, bias); - let mut filter = left_items.filter::<_, Range>(&(), move |sum| { - range.start <= sum.max_end && range.end >= sum.min_start - }); - filter.next(); + let highlights = self.match_styles(range.clone(), cx); // let mut iter_count = 0; - while let Some(item) = filter.item() { + for item in highlights { // iter_count += 1; let node_range = &item.range; let name = &item.name; @@ -600,7 +572,6 @@ impl SyntaxHighlighter { } styles.push((node_range, theme.style(name.as_ref()).unwrap_or_default())); - filter.next(); } // dbg!(iter_count); diff --git a/crates/ui/src/input/element.rs b/crates/ui/src/input/element.rs index 455b626e..70dd3ed0 100644 --- a/crates/ui/src/input/element.rs +++ b/crates/ui/src/input/element.rs @@ -427,7 +427,7 @@ impl TextElement { // +1 for `\n` let line_len = line.len() + 1; let range = offset..offset + line_len; - let line_styles = highlighter.styles(&range, &theme); + let line_styles = highlighter.styles(&range, &theme, cx); styles = gpui::combine_highlights(styles, line_styles).collect(); offset = range.end; diff --git a/crates/ui/src/input/mode.rs b/crates/ui/src/input/mode.rs index a13be6a1..ba558f11 100644 --- a/crates/ui/src/input/mode.rs +++ b/crates/ui/src/input/mode.rs @@ -217,7 +217,7 @@ impl InputMode { ), }; - highlighter.update(Some(edit), text, cx); + highlighter.update(Some(edit), text); } _ => {} } diff --git a/crates/ui/src/input/state.rs b/crates/ui/src/input/state.rs index 1adbfe75..b9d6d902 100644 --- a/crates/ui/src/input/state.rs +++ b/crates/ui/src/input/state.rs @@ -391,6 +391,7 @@ impl InputState { /// - Syntax Highlighting /// - Auto Indent /// - Line Number + /// - Large Text support, up to 50K lines. pub fn code_editor(mut self, language: impl Into) -> Self { let language: SharedString = language.into(); self.mode = InputMode::CodeEditor { diff --git a/crates/ui/src/text/node.rs b/crates/ui/src/text/node.rs index 46ea1232..e1f4a6d5 100644 --- a/crates/ui/src/text/node.rs +++ b/crates/ui/src/text/node.rs @@ -292,8 +292,8 @@ impl CodeBlock { let mut styles = vec![]; if let Some(lang) = &lang { let mut highlighter = SyntaxHighlighter::new(&lang, cx); - highlighter.update(None, &Rope::from(code.as_str()), cx); - styles = highlighter.styles(&(0..code.len()), &theme); + highlighter.update(None, &Rope::from(code.as_str())); + styles = highlighter.styles(&(0..code.len()), &theme, cx); }; let state = InlineState::default();