From c3ec1deb976bff79561cc5f07fbac469c10a0d4a Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Mon, 9 Jun 2025 14:27:33 +0800 Subject: [PATCH] code-editor: Update CodeEditor to support external language. (#928) Add [Navi](https://navi-lang.org) language for example to external language for CodeEditor. image --- Cargo.lock | 12 ++++ crates/macros/Cargo.toml | 2 +- crates/story/Cargo.toml | 1 + crates/story/examples/code-editor.rs | 86 +++++++++++++++++----- crates/story/examples/fixtures/test.nv | 18 +++++ crates/ui/src/highlighter/highlighter.rs | 90 ++++++++++++++---------- crates/ui/src/highlighter/languages.rs | 24 +++++-- crates/ui/src/highlighter/registry.rs | 34 ++++++--- crates/ui/src/input/element.rs | 20 ++++-- crates/ui/src/input/mode.rs | 34 +++++++-- crates/ui/src/input/state.rs | 42 +++++------ crates/ui/src/text/element.rs | 2 +- 12 files changed, 262 insertions(+), 103 deletions(-) create mode 100644 crates/story/examples/fixtures/test.nv diff --git a/Cargo.lock b/Cargo.lock index 10564754..d865ecfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6772,6 +6772,7 @@ dependencies = [ "serde", "serde_json", "tracing-subscriber", + "tree-sitter-navi", "unindent", ] @@ -7706,6 +7707,17 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-navi" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff190c235d2c0106d8d4c567e990f7776b80a2643b7d201a672cae308675424d" +dependencies = [ + "cc", + "tree-sitter", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-proto" version = "0.2.0" diff --git a/crates/macros/Cargo.toml b/crates/macros/Cargo.toml index 14150c72..0892acee 100644 --- a/crates/macros/Cargo.toml +++ b/crates/macros/Cargo.toml @@ -1,5 +1,5 @@ [package] -edition = "2024" +edition = "2021" name = "gpui-component-macros" version = "0.1.0" diff --git a/crates/story/Cargo.toml b/crates/story/Cargo.toml index c0175e5e..7e644500 100644 --- a/crates/story/Cargo.toml +++ b/crates/story/Cargo.toml @@ -20,6 +20,7 @@ serde = "1" serde_json = "1" unindent = "0.2.3" tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +tree-sitter-navi = "0.2.2" [target.'cfg(target_os = "linux")'.dependencies] gtk = { version = "0.18" } diff --git a/crates/story/examples/code-editor.rs b/crates/story/examples/code-editor.rs index cd2988cb..12d799c0 100644 --- a/crates/story/examples/code-editor.rs +++ b/crates/story/examples/code-editor.rs @@ -3,38 +3,87 @@ use gpui_component::{ checkbox::Checkbox, dropdown::{Dropdown, DropdownEvent, DropdownState}, h_flex, - highlighter::Language, + highlighter::{Language, LanguageConfig, LanguageRegistry}, input::{InputEvent, InputState, Marker, TabSize, TextInput}, v_flex, }; use story::Assets; +fn init(cx: &mut App) { + LanguageRegistry::global_mut(cx).register( + "navi", + &LanguageConfig::new( + tree_sitter_navi::LANGUAGE.into(), + vec![], + tree_sitter_navi::HIGHLIGHTS_QUERY, + "", + "", + ), + ); +} + pub struct Example { input_state: Entity, language_state: Entity>>, - language: Language, + language: Lang, line_number: bool, need_update: bool, _subscribes: Vec, } -const LANGUAGES: [(Language, &'static str); 8] = [ - (Language::Rust, include_str!("./fixtures/test.rs")), - (Language::JavaScript, include_str!("./fixtures/test.js")), - (Language::Go, include_str!("./fixtures/test.go")), - (Language::Python, include_str!("./fixtures/test.py")), - (Language::Ruby, include_str!("./fixtures/test.rb")), - (Language::Zig, include_str!("./fixtures/test.zig")), - (Language::C, include_str!("./fixtures/test.c")), - (Language::Sql, include_str!("./fixtures/test.sql")), +#[derive(Debug, Clone, PartialEq, Eq)] +enum Lang { + BuiltIn(Language), + External(&'static str), +} + +impl Lang { + fn name(&self) -> &str { + match self { + Lang::BuiltIn(lang) => lang.name(), + Lang::External(lang) => lang, + } + } +} + +const LANGUAGES: [(Lang, &'static str); 8] = [ + ( + Lang::BuiltIn(Language::Rust), + include_str!("./fixtures/test.rs"), + ), + ( + Lang::BuiltIn(Language::JavaScript), + include_str!("./fixtures/test.js"), + ), + ( + Lang::BuiltIn(Language::Go), + include_str!("./fixtures/test.go"), + ), + ( + Lang::BuiltIn(Language::Python), + include_str!("./fixtures/test.py"), + ), + ( + Lang::BuiltIn(Language::Ruby), + include_str!("./fixtures/test.rb"), + ), + ( + Lang::BuiltIn(Language::Zig), + include_str!("./fixtures/test.zig"), + ), + ( + Lang::BuiltIn(Language::Sql), + include_str!("./fixtures/test.sql"), + ), + (Lang::External("navi"), include_str!("./fixtures/test.nv")), ]; impl Example { pub fn new(window: &mut Window, cx: &mut Context) -> Self { - let default_language = LANGUAGES[0]; + let default_language = LANGUAGES[0].clone(); let input_state = cx.new(|cx| { InputState::new(window, cx) - .code_editor(default_language.0.name()) + .code_editor(default_language.0.name().to_string()) .line_number(true) .tab_size(TabSize { tab_size: 4, @@ -61,9 +110,11 @@ impl Example { |this, state, _: &DropdownEvent>, cx| { if let Some(val) = state.read(cx).selected_value() { if let Some(language) = Language::from_str(&val) { - this.language = language; - this.need_update = true; + this.language = Lang::BuiltIn(language); + } else { + this.language = Lang::External("navi"); } + this.need_update = true; cx.notify(); } }, @@ -108,8 +159,8 @@ impl Example { return; } - let language = self.language; - let code = LANGUAGES.iter().find(|s| s.0 == language).unwrap().1; + let language = self.language.name().to_string(); + let code = LANGUAGES.iter().find(|s| s.0.name() == language).unwrap().1; self.input_state.update(cx, |state, cx| { state.set_value(code, window, cx); state.set_highlighter(language, cx); @@ -166,6 +217,7 @@ fn main() { app.run(move |cx| { story::init(cx); + init(cx); cx.activate(true); story::create_new_window("Code Editor", Example::view, cx); diff --git a/crates/story/examples/fixtures/test.nv b/crates/story/examples/fixtures/test.nv new file mode 100644 index 00000000..1eb1388e --- /dev/null +++ b/crates/story/examples/fixtures/test.nv @@ -0,0 +1,18 @@ +use std.net.http.client.{HttpClient, Request}; +use std.net.http.OK; + +fn main() throws { + let client = HttpClient.new( + max_redirect_count: 5, + user_agent: "navi-client", + ); + let req = try Request.get("https://httpbin.org/get"); + let res = try client.request(req); + + if (res.status() != OK) { + try println("Failed to fetch repo", res.text()); + return; + } + + try println(res.text()); +} diff --git a/crates/ui/src/highlighter/highlighter.rs b/crates/ui/src/highlighter/highlighter.rs index 335a0a2e..37d1fca3 100644 --- a/crates/ui/src/highlighter/highlighter.rs +++ b/crates/ui/src/highlighter/highlighter.rs @@ -9,16 +9,17 @@ use tree_sitter::{ }; use tree_sitter_highlight::{HighlightConfiguration, Highlighter}; +use crate::highlighter::LanguageRegistry; + use super::{HighlightTheme, Language}; /// A syntax highlighter that supports incremental parsing, multiline text, /// and caching of highlight results. #[allow(unused)] pub struct SyntaxHighlighter { - language_name: &'static str, - language: Option, + language: SharedString, query: Option, - injection_queries: HashMap<&'static str, Query>, + injection_queries: HashMap, parser: Parser, old_tree: Option, text: SharedString, @@ -47,19 +48,22 @@ pub struct SyntaxHighlighter { impl SyntaxHighlighter { /// Create a new SyntaxHighlighter for HTML. - pub fn new(lang: &str) -> Self { - Self::build_combined_injections_query(&lang).unwrap() + pub fn new(lang: &str, cx: &App) -> Self { + Self::build_combined_injections_query(&lang, cx).unwrap_or_else(|| panic!( + "failed to build language {}, please make sure have registered the language in LanguageRegistry", + lang + )) } /// Build the combined injections query for the given language. /// /// https://github.com/tree-sitter/tree-sitter/blob/v0.25.5/highlight/src/lib.rs#L336 - fn build_combined_injections_query(lang: &str) -> Option { - let language = Language::from_str(&lang); - let Some(language) = language else { + fn build_combined_injections_query(lang: &str, cx: &App) -> Option { + let registry = LanguageRegistry::global(cx); + let config = registry.language(&lang); + let Some(config) = config else { return None; }; - let config = language.config(); let mut parser = Parser::new(); _ = parser.set_language(&config.language); @@ -146,19 +150,19 @@ impl SyntaxHighlighter { } let mut injection_queries = HashMap::new(); - for inj_language in language.injection_languages() { - let inj_config = inj_language.config(); - - match Query::new(&inj_config.language, &inj_config.highlights) { - Ok(q) => { - injection_queries.insert(inj_language.name(), q); - } - Err(e) => { - tracing::error!( - "failed to build injection query for {:?}: {:?}", - inj_language, - e - ); + for inj_language in config.injection_languages.iter() { + if let Some(inj_config) = registry.language(&inj_language) { + match Query::new(&inj_config.language, &inj_config.highlights) { + Ok(q) => { + injection_queries.insert(inj_config.name.clone(), q); + } + Err(e) => { + tracing::error!( + "failed to build injection query for {:?}: {:?}", + inj_config.name, + e + ); + } } } } @@ -166,8 +170,7 @@ impl SyntaxHighlighter { // let highlight_indices = vec![None; query.capture_names().len()]; Some(Self { - language_name: language.name(), - language: Some(language), + language: config.name.clone(), query: Some(query), injection_queries, parser, @@ -188,21 +191,28 @@ impl SyntaxHighlighter { }) } - pub fn set_language(&mut self, lang: impl Into) { - let lang = lang.into(); - let language = Language::from_str(&lang); + pub fn set_language(&mut self, language: impl Into, cx: &App) { + let language = language.into(); + if self.language == language { return; } // FIXME: use build_combined_injections_query to build the query. - - if let Some(language) = language { + self.query = None; + if let Some(language) = Language::from_str(&language) { + self.query = Some(language.query()); _ = self.parser.set_language(&language.config().language); + } else { + if let Some(config) = LanguageRegistry::global(cx).language(&language) { + _ = self.parser.set_language(&config.language); + if let Ok(query) = tree_sitter::Query::new(&config.language, &config.highlights) { + self.query = Some(query); + } + } } self.language = language; - self.query = language.map(|l| l.query()); self.old_tree = None; self.text = SharedString::new(""); self.highlighter = Highlighter::new(); @@ -464,21 +474,26 @@ impl SyntaxHighlighter { /// - `include_children`: Whether to include the children of the content node. fn injection_for_match<'a>( &self, - parent_name: Option<&'a str>, + parent_name: Option, query: &'a Query, query_match: &QueryMatch<'a, 'a>, source: &'a [u8], - ) -> (Option<&'a str>, Option>, bool) { + ) -> (Option, Option>, bool) { let content_capture_index = self.injection_content_capture_index; let language_capture_index = self.injection_language_capture_index; - let mut language_name = None; + let mut language_name: Option = None; let mut content_node = None; for capture in query_match.captures { let index = Some(capture.index); if index == language_capture_index { - language_name = capture.node.utf8_text(source).ok(); + language_name = capture + .node + .utf8_text(source) + .ok() + .map(ToString::to_string) + .map(SharedString::from); } else if index == content_capture_index { content_node = Some(capture.node); } @@ -496,7 +511,8 @@ impl SyntaxHighlighter { .value .as_ref() .map(std::convert::AsRef::as_ref) - .to_owned(); + .map(ToString::to_string) + .map(SharedString::from); } } @@ -505,7 +521,7 @@ impl SyntaxHighlighter { // layer. "injection.self" => { if language_name.is_none() { - language_name = Some(self.language_name); + language_name = Some(self.language.clone()); } } @@ -514,7 +530,7 @@ impl SyntaxHighlighter { // parent layer "injection.parent" => { if language_name.is_none() { - language_name = parent_name; + language_name = parent_name.clone(); } } diff --git a/crates/ui/src/highlighter/languages.rs b/crates/ui/src/highlighter/languages.rs index a7a64ef6..115960cd 100644 --- a/crates/ui/src/highlighter/languages.rs +++ b/crates/ui/src/highlighter/languages.rs @@ -38,7 +38,9 @@ pub enum Language { #[derive(Debug, Clone, PartialEq, Eq)] pub struct LanguageConfig { + pub name: SharedString, pub language: tree_sitter::Language, + pub injection_languages: Vec, pub highlights: SharedString, pub injections: SharedString, pub locals: SharedString, @@ -47,12 +49,15 @@ pub struct LanguageConfig { impl LanguageConfig { pub fn new( language: tree_sitter::Language, + injection_languages: Vec, highlights: &str, injections: &str, locals: &str, ) -> Self { Self { + name: language.name().unwrap_or_default().into(), language, + injection_languages, highlights: SharedString::from(highlights.to_string()), injections: SharedString::from(injections.to_string()), locals: SharedString::from(locals.to_string()), @@ -144,14 +149,17 @@ impl Language { } #[allow(unused)] - pub(super) fn injection_languages(&self) -> Vec { + pub(super) fn injection_languages(&self) -> Vec { match self { - Self::Markdown => vec![Self::MarkdownInline, Self::Html, Self::Toml, Self::Yaml], + Self::Markdown => vec!["markdown-inline", "html", "toml", "yaml"], Self::MarkdownInline => vec![], - Self::Html => vec![Self::JavaScript, Self::Css], - Self::Rust => vec![Self::Rust], + Self::Html => vec!["javascript", "css"], + Self::Rust => vec!["rust"], _ => vec![], } + .into_iter() + .map(|s| s.into()) + .collect() } pub(super) fn query(&self) -> Query { @@ -329,7 +337,13 @@ impl Language { let language = tree_sitter::Language::new(language); - LanguageConfig::new(language, query, injection, locals) + LanguageConfig::new( + language, + self.injection_languages(), + query, + injection, + locals, + ) } } diff --git a/crates/ui/src/highlighter/registry.rs b/crates/ui/src/highlighter/registry.rs index 64e9d66a..a9e39c4e 100644 --- a/crates/ui/src/highlighter/registry.rs +++ b/crates/ui/src/highlighter/registry.rs @@ -8,7 +8,16 @@ use std::{ }; use super::LanguageConfig; -use crate::ThemeMode; +use crate::{highlighter::languages, ThemeMode}; + +pub(super) fn init(cx: &mut App) { + let mut register = LanguageRegistry::new(); + for language in languages::Language::all() { + register.register(language.name(), &language.config()); + } + + cx.set_global(register); +} pub(super) const HIGHLIGHT_NAMES: [&str; 40] = [ "attribute", @@ -359,10 +368,6 @@ impl HighlightTheme { } } -pub fn init(cx: &mut App) { - cx.set_global(LanguageRegistry::new()); -} - /// Registry for code highlighter languages. #[derive(Clone)] pub struct LanguageRegistry { @@ -394,13 +399,12 @@ impl LanguageRegistry { self.languages.insert(lang.to_string(), config.clone()); } - #[allow(unused)] - pub(crate) fn set_theme(&mut self, light_theme: &HighlightTheme, dark_theme: &HighlightTheme) { - self.light_theme = Arc::new(light_theme.clone()); - self.dark_theme = Arc::new(dark_theme.clone()); + /// Set highlighter theme. + pub fn set_theme(&mut self, light: &HighlightTheme, dark: &HighlightTheme) { + self.light_theme = Arc::new(light.clone()); + self.dark_theme = Arc::new(dark.clone()); } - #[allow(unused)] pub(crate) fn theme(&self, is_dark: bool) -> &Arc { if is_dark { &self.dark_theme @@ -408,6 +412,16 @@ impl LanguageRegistry { &self.light_theme } } + + /// Returns a reference to the map of registered languages. + pub fn languages(&self) -> &HashMap { + &self.languages + } + + /// Returns the language configuration for the given language name. + pub fn language(&self, name: &str) -> Option<&LanguageConfig> { + self.languages.get(name) + } } #[cfg(test)] diff --git a/crates/ui/src/input/element.rs b/crates/ui/src/input/element.rs index 1033eadf..424781e8 100644 --- a/crates/ui/src/input/element.rs +++ b/crates/ui/src/input/element.rs @@ -8,7 +8,10 @@ use gpui::{ }; use smallvec::SmallVec; -use crate::{highlighter::LanguageRegistry, ActiveTheme as _, Root}; +use crate::{ + highlighter::{LanguageRegistry, SyntaxHighlighter}, + ActiveTheme as _, Root, +}; use super::{mode::InputMode, InputState, LastLayout}; @@ -363,12 +366,22 @@ impl TextElement { let theme = LanguageRegistry::global(cx) .theme(cx.theme().is_dark()) .clone(); - self.input.update(cx, |state, _| match &state.mode { + self.input.update(cx, |state, cx| match &state.mode { InputMode::CodeEditor { + language, highlighter, markers, .. } => { + // Init highlighter if not initialized + let mut highlighter = highlighter.borrow_mut(); + if highlighter.is_none() { + highlighter.replace(SyntaxHighlighter::new(language, cx)); + }; + let Some(highlighter) = highlighter.as_ref() else { + return None; + }; + let mut offset = 0; let mut skipped_offset = 0; let mut styles = vec![]; @@ -386,8 +399,7 @@ impl TextElement { } let range = offset..offset + line_len; - let line_styles = highlighter.borrow().styles(&range, &theme); - + let line_styles = highlighter.styles(&range, &theme); styles = gpui::combine_highlights(styles, line_styles).collect(); offset = range.end; diff --git a/crates/ui/src/input/mode.rs b/crates/ui/src/input/mode.rs index c9b3353a..543f11ce 100644 --- a/crates/ui/src/input/mode.rs +++ b/crates/ui/src/input/mode.rs @@ -1,7 +1,7 @@ -use std::cell::RefCell; use std::rc::Rc; +use std::{cell::RefCell, ops::Range}; -use gpui::{DefiniteLength, SharedString}; +use gpui::{App, DefiniteLength, SharedString}; use crate::{highlighter::SyntaxHighlighter, input::marker::Marker}; @@ -49,7 +49,8 @@ pub enum InputMode { height: Option, /// Show line number line_number: bool, - highlighter: Rc>, + language: SharedString, + highlighter: Rc>>, markers: Vec, }, AutoGrow { @@ -154,11 +155,30 @@ impl InputMode { } } - #[allow(unused)] - pub(super) fn highlighter(&self) -> Option<&Rc>> { + pub(super) fn update_highlighter( + &mut self, + selected_range: &Range, + full_text: SharedString, + new_text: &str, + cx: &mut App, + ) { match &self { - InputMode::CodeEditor { highlighter, .. } => Some(highlighter), - _ => None, + InputMode::CodeEditor { + language, + highlighter, + .. + } => { + let mut highlighter = highlighter.borrow_mut(); + if highlighter.is_none() { + let new_highlighter = SyntaxHighlighter::new(language, cx); + highlighter.replace(new_highlighter); + } + + if let Some(highlighter) = highlighter.as_mut() { + highlighter.update(selected_range, full_text, new_text, cx); + } + } + _ => {} } } diff --git a/crates/ui/src/input/state.rs b/crates/ui/src/input/state.rs index d3008955..d6c76662 100644 --- a/crates/ui/src/input/state.rs +++ b/crates/ui/src/input/state.rs @@ -30,7 +30,6 @@ use super::{ number_input, text_wrapper::TextWrapper, }; -use crate::highlighter::SyntaxHighlighter; use crate::input::marker::Marker; use crate::{history::History, scroll::ScrollbarState, Root}; @@ -379,7 +378,8 @@ impl InputState { self.mode = InputMode::CodeEditor { rows: 2, tab: TabSize::default(), - highlighter: Rc::new(RefCell::new(SyntaxHighlighter::new(&language))), + language, + highlighter: Rc::new(RefCell::new(None)), line_number: true, height: Some(relative(1.)), markers: vec![], @@ -442,11 +442,20 @@ impl InputState { self } - /// Set highlighter, only for [`InputMode::CodeEditor`] mode. - pub fn set_highlighter(&mut self, language: impl Into, cx: &mut Context) { + /// Set highlighter language for for [`InputMode::CodeEditor`] mode. + pub fn set_highlighter( + &mut self, + new_language: impl Into, + cx: &mut Context, + ) { match &mut self.mode { - InputMode::CodeEditor { highlighter, .. } => { - highlighter.borrow_mut().set_language(language); + InputMode::CodeEditor { + language, + highlighter, + .. + } => { + *language = new_language.into(); + *highlighter.borrow_mut() = None; } _ => {} } @@ -1998,11 +2007,8 @@ impl EntityInputHandler for InputState { self.push_history(&range, &new_text, window, cx); self.text = mask_text.clone(); - if let Some(highlighter) = self.mode.highlighter() { - highlighter - .borrow_mut() - .update(&range, self.text.clone(), &new_text, cx); - } + self.mode + .update_highlighter(&range, self.text.clone(), &new_text, cx); self.mode.clear_markers(); self.text_wrapper.update(self.text.clone(), false, cx); self.selected_range = new_pos..new_pos; @@ -2042,11 +2048,8 @@ impl EntityInputHandler for InputState { self.push_history(&range, new_text, window, cx); self.text = pending_text; - if let Some(highlighter) = self.mode.highlighter() { - highlighter - .borrow_mut() - .update(&range, self.text.clone(), &new_text, cx); - } + self.mode + .update_highlighter(&range, self.text.clone(), &new_text, cx); self.mode.clear_markers(); self.text_wrapper.update(self.text.clone(), false, cx); if new_text.is_empty() { @@ -2151,11 +2154,8 @@ impl Focusable for InputState { impl Render for InputState { fn render(&mut self, _: &mut Window, cx: &mut Context) -> impl IntoElement { self.text_wrapper.update(self.text.clone(), false, cx); - if let Some(highlighter) = self.mode.highlighter() { - highlighter - .borrow_mut() - .update(&(0..0), self.text.clone(), "", cx); - } + self.mode + .update_highlighter(&(0..0), self.text.clone(), "", cx); div() .id("text-element") diff --git a/crates/ui/src/text/element.rs b/crates/ui/src/text/element.rs index ef96ef3d..d13099a0 100644 --- a/crates/ui/src/text/element.rs +++ b/crates/ui/src/text/element.rs @@ -222,7 +222,7 @@ impl CodeBlock { .clone(); let mut styles = vec![]; if let Some(lang) = &lang { - let mut highlighter = SyntaxHighlighter::new(&lang); + let mut highlighter = SyntaxHighlighter::new(&lang, cx); highlighter.update(&(0..0), code.clone(), "", cx); styles = highlighter.styles(&(0..code.len()), &theme); };