gpui-component/crates/ui/src/highlighter/highlighter.rs
Jason Lee a9e6b49a15
highlighter: Fix #1204 changes cause to incorrect highlight result. (#1218)
This PR for fix some incorrect changes in #1204 

## Before

<img width="854" height="864" alt="image"
src="https://github.com/user-attachments/assets/eebd07e5-3437-444d-a55d-2941228e3b08"
/>

## After

<img width="954" height="962" alt="image"
src="https://github.com/user-attachments/assets/60761848-8d4f-4fe9-88b9-f0b0f0985d2f"
/>
<img width="1515" height="867" alt="image"
src="https://github.com/user-attachments/assets/3d4654a7-f977-4919-9c9e-f013fcdd8fde"
/>
2025-09-08 16:14:07 +08:00

824 lines
28 KiB
Rust

use super::HighlightTheme;
use crate::highlighter::LanguageRegistry;
use anyhow::{anyhow, Context, Result};
use gpui::{App, HighlightStyle, SharedString};
use ropey::Rope;
use std::{
collections::{BTreeSet, HashMap},
ops::Range,
slice::Chunks,
usize,
};
use sum_tree::{Bias, SumTree};
use tree_sitter::{
InputEdit, Node, Parser, Point, Query, QueryCursor, QueryMatch, StreamingIterator, Tree,
};
/// A syntax highlighter that supports incremental parsing, multiline text,
/// and caching of highlight results.
#[allow(unused)]
pub struct SyntaxHighlighter {
language: SharedString,
query: Option<Query>,
injection_queries: HashMap<SharedString, Query>,
parser: Parser,
old_tree: Option<Tree>,
text: Rope,
locals_pattern_index: usize,
highlights_pattern_index: usize,
// highlight_indices: Vec<Option<Highlight>>,
non_local_variable_patterns: Vec<bool>,
injection_content_capture_index: Option<u32>,
injection_language_capture_index: Option<u32>,
local_scope_capture_index: Option<u32>,
local_def_capture_index: Option<u32>,
local_def_value_capture_index: Option<u32>,
local_ref_capture_index: Option<u32>,
/// Cache of highlight, the range is offset of the token in the tree.
cache: SumTree<HighlightItem>,
}
struct TextProvider<'a>(&'a Rope);
impl<'a> tree_sitter::TextProvider<&'a [u8]> for TextProvider<'a> {
type I = Chunks<'a, u8>;
fn text(&mut self, node: tree_sitter::Node) -> Self::I {
let slice = self.0.byte_slice(node.start_byte()..node.end_byte());
slice.as_str().unwrap_or_default().as_bytes().chunks(64)
}
}
#[derive(Debug, Default, Clone)]
struct HighlightSummary {
count: usize,
start: usize,
end: usize,
min_start: usize,
max_end: usize,
}
/// The highlight item, the range is offset of the token in the tree.
#[derive(Debug, Default, Clone)]
struct HighlightItem {
/// The byte range of the highlight in the text.
range: Range<usize>,
/// The highlight name, like `function`, `string`, `comment`, etc.
name: SharedString,
}
impl HighlightItem {
pub fn new(range: Range<usize>, name: impl Into<SharedString>) -> Self {
Self {
range,
name: name.into(),
}
}
}
impl sum_tree::Item for HighlightItem {
type Summary = HighlightSummary;
fn summary(&self, _cx: &()) -> Self::Summary {
HighlightSummary {
count: 1,
start: self.range.start,
end: self.range.end,
min_start: self.range.start,
max_end: self.range.end,
}
}
}
impl sum_tree::Summary for HighlightSummary {
type Context = ();
fn zero(_: &Self::Context) -> Self {
HighlightSummary {
count: 0,
start: usize::MIN,
end: usize::MAX,
min_start: usize::MAX,
max_end: usize::MIN,
}
}
fn add_summary(&mut self, other: &Self, _: &Self::Context) {
self.min_start = self.min_start.min(other.min_start);
self.max_end = self.max_end.max(other.max_end);
self.start = other.start;
self.end = other.end;
self.count += other.count;
}
}
impl<'a> sum_tree::Dimension<'a, HighlightSummary> for usize {
fn zero(_: &()) -> Self {
0
}
fn add_summary(&mut self, _: &'a HighlightSummary, _: &()) {}
}
impl<'a> sum_tree::Dimension<'a, HighlightSummary> for Range<usize> {
fn zero(_: &()) -> Self {
Default::default()
}
fn add_summary(&mut self, summary: &'a HighlightSummary, _: &()) {
self.start = summary.start;
self.end = summary.end;
}
}
impl SyntaxHighlighter {
/// Create a new SyntaxHighlighter for HTML.
pub fn new(lang: &str, cx: &App) -> Self {
match Self::build_combined_injections_query(&lang, cx) {
Ok(result) => result,
Err(err) => {
tracing::warn!(
"SyntaxHighlighter init failed, fallback to use `text`, {}",
err
);
Self::build_combined_injections_query("text", cx).unwrap()
}
}
}
/// Build the combined injections query for the given language.
///
/// https://github.com/tree-sitter/tree-sitter/blob/v0.25.5/highlight/src/lib.rs#L336
fn build_combined_injections_query(lang: &str, cx: &App) -> Result<Self> {
let registry = LanguageRegistry::global(cx);
let Some(config) = registry.language(&lang) else {
return Err(anyhow!(
"language {:?} is not registered in `LanguageRegistry`",
lang
));
};
let mut parser = Parser::new();
parser
.set_language(&config.language)
.context("parse set_language")?;
// Concatenate the query strings, keeping track of the start offset of each section.
let mut query_source = String::new();
query_source.push_str(&config.injections);
let locals_query_offset = query_source.len();
query_source.push_str(&config.locals);
let highlights_query_offset = query_source.len();
query_source.push_str(&config.highlights);
// Construct a single query by concatenating the three query strings, but record the
// range of pattern indices that belong to each individual string.
let query = Query::new(&config.language, &query_source).context("new query")?;
let mut locals_pattern_index = 0;
let mut highlights_pattern_index = 0;
for i in 0..(query.pattern_count()) {
let pattern_offset = query.start_byte_for_pattern(i);
if pattern_offset < highlights_query_offset {
if pattern_offset < highlights_query_offset {
highlights_pattern_index += 1;
}
if pattern_offset < locals_query_offset {
locals_pattern_index += 1;
}
}
}
// let Some(mut combined_injections_query) =
// Query::new(&config.language, &config.injections).ok()
// else {
// return None;
// };
// let mut has_combined_queries = false;
// for pattern_index in 0..locals_pattern_index {
// let settings = query.property_settings(pattern_index);
// if settings.iter().any(|s| &*s.key == "injection.combined") {
// has_combined_queries = true;
// query.disable_pattern(pattern_index);
// } else {
// combined_injections_query.disable_pattern(pattern_index);
// }
// }
// let combined_injections_query = if has_combined_queries {
// Some(combined_injections_query)
// } else {
// None
// };
// Find all of the highlighting patterns that are disabled for nodes that
// have been identified as local variables.
let non_local_variable_patterns = (0..query.pattern_count())
.map(|i| {
query
.property_predicates(i)
.iter()
.any(|(prop, positive)| !*positive && prop.key.as_ref() == "local")
})
.collect();
// Store the numeric ids for all of the special captures.
let mut injection_content_capture_index = None;
let mut injection_language_capture_index = None;
let mut local_def_capture_index = None;
let mut local_def_value_capture_index = None;
let mut local_ref_capture_index = None;
let mut local_scope_capture_index = None;
for (i, name) in query.capture_names().iter().enumerate() {
let i = Some(i as u32);
match *name {
"injection.content" => injection_content_capture_index = i,
"injection.language" => injection_language_capture_index = i,
"local.definition" => local_def_capture_index = i,
"local.definition-value" => local_def_value_capture_index = i,
"local.reference" => local_ref_capture_index = i,
"local.scope" => local_scope_capture_index = i,
_ => {}
}
}
let mut injection_queries = HashMap::new();
for inj_language in config.injection_languages.iter() {
if let Some(inj_config) = registry.language(&inj_language) {
match Query::new(&inj_config.language, &inj_config.highlights) {
Ok(q) => {
injection_queries.insert(inj_config.name.clone(), q);
}
Err(e) => {
tracing::error!(
"failed to build injection query for {:?}: {:?}",
inj_config.name,
e
);
}
}
}
}
// let highlight_indices = vec![None; query.capture_names().len()];
Ok(Self {
language: config.name.clone(),
query: Some(query),
injection_queries,
parser,
old_tree: None,
text: Rope::new(),
cache: sum_tree::SumTree::new(&()),
locals_pattern_index,
highlights_pattern_index,
non_local_variable_patterns,
injection_content_capture_index,
injection_language_capture_index,
local_scope_capture_index,
local_def_capture_index,
local_def_value_capture_index,
local_ref_capture_index,
})
}
pub fn is_empty(&self) -> bool {
self.text.len_bytes() == 0
}
/// Highlight the given text, returning a map from byte ranges to highlight captures.
/// Uses incremental parsing, detects changed ranges, and caches unchanged results.
pub fn update(&mut self, edit: Option<InputEdit>, text: &Rope, cx: &App) {
if &self.text == text {
return;
}
let edit = edit.unwrap_or(InputEdit {
start_byte: 0,
old_end_byte: 0,
new_end_byte: text.len_bytes(),
start_position: Point::new(0, 0),
old_end_position: Point::new(0, 0),
new_end_position: Point::new(0, 0),
});
let mut old_tree = self
.old_tree
.take()
.unwrap_or(self.parser.parse("", None).unwrap());
old_tree.edit(&edit);
let new_tree = self.parser.parse_with_options(
&mut |offset, _| {
if offset >= text.len_bytes() {
""
} else {
let (chunk, chunk_byte_ix, _, _) = text.chunk_at_byte(offset);
&chunk[offset - chunk_byte_ix..]
}
},
Some(&old_tree),
None,
);
let Some(new_tree) = new_tree else {
return;
};
// let changed_ranges = new_tree.changed_ranges(&old_tree);
// Update state
self.old_tree = Some(new_tree);
self.text = text.clone();
// let measure = crate::Measure::new("build_styles");
self.build_styles(cx);
// measure.end();
}
/// NOTE: 10K lines, about 180ms
/// FIXME: To improve the performance when there more than 5K lines, use partial update.
/// Ref: https://github.com/longbridge/gpui-component/pull/1197
fn build_styles(&mut self, cx: &App) {
let Some(tree) = &self.old_tree else {
return;
};
let Some(query) = &self.query else {
return;
};
let root_node = tree.root_node();
// Remove the changed items from the cache.
let new_cache = sum_tree::SumTree::new(&());
self.cache = new_cache;
let source = self.text.clone();
let mut cursor = QueryCursor::new();
let mut matches = cursor.matches(&query, root_node, TextProvider(&source));
while let Some(query_match) = matches.next() {
// Ref:
// https://github.com/tree-sitter/tree-sitter/blob/460118b4c82318b083b4d527c9c750426730f9c0/highlight/src/lib.rs#L556
if let (Some(language_name), Some(content_node), _) =
self.injection_for_match(None, query, query_match)
{
let styles = self.handle_injection(&language_name, content_node, cx);
for (node_range, highlight_name) in styles {
self.cache
.push(HighlightItem::new(node_range.clone(), highlight_name), &());
}
continue;
}
for cap in query_match.captures {
let node = cap.node;
let Some(highlight_name) = query.capture_names().get(cap.index as usize) else {
continue;
};
let node_range: Range<usize> = node.start_byte()..node.end_byte();
let highlight_name = SharedString::from(highlight_name.to_string());
// Merge near range and same highlight name
let last_item = self.cache.last();
let last_range = last_item.map(|item| &item.range).unwrap_or(&(0..0));
let last_highlight_name = last_item.map(|item| item.name.clone());
if last_range.end <= node_range.start
&& last_highlight_name.as_ref() == Some(&highlight_name)
{
self.cache.push(
HighlightItem::new(
last_range.start..node_range.end,
highlight_name.clone(),
),
&(),
);
} else if last_range == &node_range {
// case:
// last_range: 213..220, last_highlight_name: Some("property")
// last_range: 213..220, last_highlight_name: Some("string")
self.cache.push(
HighlightItem::new(
node_range,
last_highlight_name.unwrap_or(highlight_name),
),
&(),
);
} else {
self.cache
.push(HighlightItem::new(node_range, highlight_name.clone()), &());
}
}
}
// DO NOT REMOVE THIS PRINT, it's useful for debugging
// for item in self.cache.iter() {
// println!("item: {:?}", item);
// }
}
/// TODO: Use incremental parsing to handle the injection.
fn handle_injection(
&self,
injection_language: &str,
node: Node,
cx: &App,
) -> Vec<(Range<usize>, String)> {
let start_offset = node.start_byte();
let end_offset = node.end_byte();
let mut cache = vec![];
let Some(query) = &self.injection_queries.get(injection_language) else {
return cache;
};
let content = self.text.byte_slice(node.start_byte()..node.end_byte());
if content.len_bytes() == 0 {
return cache;
};
let Some(config) = LanguageRegistry::global(cx).language(injection_language) else {
return cache;
};
let mut parser = Parser::new();
if parser.set_language(&config.language).is_err() {
return cache;
}
let source = content.as_str().unwrap_or_default().as_bytes();
let Some(tree) = parser.parse(source, None) else {
return cache;
};
let mut query_cursor = QueryCursor::new();
let mut matches = query_cursor.matches(query, tree.root_node(), source);
let mut last_end = start_offset;
while let Some(m) = matches.next() {
for cap in m.captures {
let cap_node = cap.node;
let node_range: Range<usize> =
start_offset + cap_node.start_byte()..start_offset + cap_node.end_byte();
if node_range.start < last_end {
continue;
}
if node_range.end > end_offset {
break;
}
if let Some(highlight_name) = query.capture_names().get(cap.index as usize) {
last_end = node_range.end;
cache.push((node_range, highlight_name.to_string()));
}
}
}
cache
}
/// Ref:
/// https://github.com/tree-sitter/tree-sitter/blob/v0.25.5/highlight/src/lib.rs#L1229
///
/// Returns:
/// - `language_name`: The language name of the injection.
/// - `content_node`: The content node of the injection.
/// - `include_children`: Whether to include the children of the content node.
fn injection_for_match<'a>(
&self,
parent_name: Option<SharedString>,
query: &'a Query,
query_match: &QueryMatch<'a, 'a>,
) -> (Option<SharedString>, Option<Node<'a>>, bool) {
let content_capture_index = self.injection_content_capture_index;
// let language_capture_index = self.injection_language_capture_index;
let mut language_name: Option<SharedString> = None;
let mut content_node = None;
for capture in query_match.captures {
let index = Some(capture.index);
if index == content_capture_index {
content_node = Some(capture.node);
}
}
let mut include_children = false;
for prop in query.property_settings(query_match.pattern_index) {
match prop.key.as_ref() {
// In addition to specifying the language name via the text of a
// captured node, it can also be hard-coded via a `#set!` predicate
// that sets the injection.language key.
"injection.language" => {
if language_name.is_none() {
language_name = prop
.value
.as_ref()
.map(std::convert::AsRef::as_ref)
.map(ToString::to_string)
.map(SharedString::from);
}
}
// Setting the `injection.self` key can be used to specify that the
// language name should be the same as the language of the current
// layer.
"injection.self" => {
if language_name.is_none() {
language_name = Some(self.language.clone());
}
}
// Setting the `injection.parent` key can be used to specify that
// the language name should be the same as the language of the
// parent layer
"injection.parent" => {
if language_name.is_none() {
language_name = parent_name.clone();
}
}
// By default, injections do not include the *children* of an
// `injection.content` node - only the ranges that belong to the
// node itself. This can be changed using a `#set!` predicate that
// sets the `injection.include-children` key.
"injection.include-children" => include_children = true,
_ => {}
}
}
(language_name, content_node, include_children)
}
/// The argument `range` is the range of the line in the text.
///
/// Returns `range` is the range in the line.
pub(crate) fn styles(
&self,
range: &Range<usize>,
theme: &HighlightTheme,
) -> Vec<(Range<usize>, HighlightStyle)> {
let mut styles = vec![];
let start_offset = range.start;
let mut cursor = self.cache.cursor::<usize>(&());
let bias = if start_offset == 0 {
Bias::Right
} else {
Bias::Left
};
let left_items = cursor.slice(&start_offset, bias);
let mut filter = left_items.filter::<_, Range<usize>>(&(), move |sum| {
range.start <= sum.max_end && range.end >= sum.min_start
});
filter.next();
// let mut iter_count = 0;
while let Some(item) = filter.item() {
// iter_count += 1;
let node_range = &item.range;
let name = &item.name;
// Avoid start larger than end
let mut node_range = node_range.start.max(range.start)..node_range.end.min(range.end);
if node_range.start > node_range.end {
node_range.end = node_range.start;
}
styles.push((node_range, theme.style(name.as_ref()).unwrap_or_default()));
filter.next();
}
// dbg!(iter_count);
// If the matched styles is empty, return a default range.
if styles.len() == 0 {
return vec![(start_offset..range.end, HighlightStyle::default())];
}
let styles = unique_styles(&range, styles);
// NOTE: DO NOT remove this comment, it is used for debugging.
// for style in &styles {
// println!("---- style: {:?} - {:?}", style.0, style.1.color);
// }
// println!("--------------------------------");
styles
}
}
/// To merge intersection ranges, let the subsequent range cover
/// the previous overlapping range and split the previous range.
///
/// From:
///
/// AA
/// BBB
/// CCCCC
/// DD
/// EEEE
///
/// To:
///
/// AABCCDDCEEEE
pub(crate) fn unique_styles(
total_range: &Range<usize>,
styles: Vec<(Range<usize>, HighlightStyle)>,
) -> Vec<(Range<usize>, HighlightStyle)> {
if styles.is_empty() {
return styles;
}
let mut intervals = BTreeSet::new();
let mut significant_intervals = BTreeSet::new();
// For example
//
// from: [(6..11), (6..11), (11..17), (17..25), (16..19), (25..59))]
// to: [6, 11, 16, 17, 19, 25, 59]
intervals.insert(total_range.start);
intervals.insert(total_range.end);
for (range, _) in &styles {
intervals.insert(range.start);
intervals.insert(range.end);
significant_intervals.insert(range.end); // End points are significant for merging decisions
}
let intervals: Vec<usize> = intervals.into_iter().collect();
let mut result = Vec::with_capacity(intervals.len().saturating_sub(1));
// For each interval between boundaries, find the top-most style
//
// Result e.g.:
//
// [(6..11, red), (11..16, green), (16..17, blue), (17..19, red), (19..25, clean), (25..59, blue)]
for i in 0..intervals.len().saturating_sub(1) {
let interval = intervals[i]..intervals[i + 1];
if interval.start >= interval.end {
continue;
}
// Find the last (top-most) style that covers this interval
let mut top_style: Option<HighlightStyle> = None;
for (range, style) in &styles {
if range.start <= interval.start && interval.end <= range.end {
if let Some(top_style) = &mut top_style {
merge_highlight_style(top_style, style);
} else {
top_style = Some(*style);
}
}
}
if let Some(style) = top_style {
result.push((interval, style));
} else {
result.push((interval, HighlightStyle::default()));
}
}
// Merge adjacent ranges with the same style, but not across significant boundaries
let mut merged: Vec<(Range<usize>, HighlightStyle)> = Vec::with_capacity(result.len());
for (range, style) in result {
if let Some((last_range, last_style)) = merged.last_mut() {
if last_range.end == range.start
&& *last_style == style
&& !significant_intervals.contains(&range.start)
{
// Merge adjacent ranges with same style, but not across significant boundaries
last_range.end = range.end;
continue;
}
}
merged.push((range, style));
}
merged
}
/// Merge other style (Other on top)
fn merge_highlight_style(style: &mut HighlightStyle, other: &HighlightStyle) {
if let Some(color) = other.color {
style.color = Some(color);
}
if let Some(font_weight) = other.font_weight {
style.font_weight = Some(font_weight);
}
if let Some(font_style) = other.font_style {
style.font_style = Some(font_style);
}
if let Some(background_color) = other.background_color {
style.background_color = Some(background_color);
}
if let Some(underline) = other.underline {
style.underline = Some(underline);
}
if let Some(strikethrough) = other.strikethrough {
style.strikethrough = Some(strikethrough);
}
if let Some(fade_out) = other.fade_out {
style.fade_out = Some(fade_out);
}
}
#[cfg(test)]
mod tests {
use gpui::Hsla;
use super::*;
use crate::Colorize as _;
fn color_style(color: Hsla) -> HighlightStyle {
let mut style = HighlightStyle::default();
style.color = Some(color);
style
}
#[track_caller]
fn assert_unique_styles(
range: Range<usize>,
left: Vec<(Range<usize>, HighlightStyle)>,
right: Vec<(Range<usize>, HighlightStyle)>,
) {
fn color_name(c: Option<Hsla>) -> String {
match c {
Some(c) => {
if c == gpui::red() {
"red".to_string()
} else if c == gpui::green() {
"green".to_string()
} else if c == gpui::blue() {
"blue".to_string()
} else {
c.to_hex()
}
}
None => "clean".to_string(),
}
}
let left = unique_styles(&range, left);
if left.len() != right.len() {
println!("\n---------------------------------------------");
for (range, style) in left.iter() {
println!("({:?}, {})", range, color_name(style.color));
}
println!("---------------------------------------------");
panic!("left {} styles, right {} styles", left.len(), right.len());
}
for (left, right) in left.into_iter().zip(right) {
if left.1.color != right.1.color || left.0 != right.0 {
panic!(
"\n left: ({:?}, {})\nright: ({:?}, {})\n",
left.0,
color_name(left.1.color),
right.0,
color_name(right.1.color)
);
}
}
}
#[test]
fn test_unique_styles() {
let red = color_style(gpui::red());
let green = color_style(gpui::green());
let blue = color_style(gpui::blue());
let clean = HighlightStyle::default();
assert_unique_styles(
0..65,
vec![
(2..10, clean),
(2..10, clean),
(5..11, red),
(2..6, clean),
(10..15, green),
(15..30, clean),
(29..35, blue),
(35..40, green),
(45..60, blue),
],
vec![
(0..5, clean),
(5..6, red),
(6..10, red),
(10..11, green),
(11..15, green),
(15..29, clean),
(29..30, blue),
(30..35, blue),
(35..40, green),
(40..45, clean),
(45..60, blue),
(60..65, clean),
],
);
}
}