syntax: Split parsing and highlighting

pull/1563/head^2
Blaž Hrastnik 3 years ago
parent 83bde1004d
commit 6728e44490

1
Cargo.lock generated

@ -381,6 +381,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"similar", "similar",
"slotmap",
"smallvec", "smallvec",
"tendril", "tendril",
"toml", "toml",

@ -22,6 +22,7 @@ unicode-segmentation = "1.8"
unicode-width = "0.1" unicode-width = "0.1"
unicode-general-category = "0.4" unicode-general-category = "0.4"
# slab = "0.4.2" # slab = "0.4.2"
slotmap = "1.0"
tree-sitter = "0.20" tree-sitter = "0.20"
once_cell = "1.9" once_cell = "1.9"
arc-swap = "1" arc-swap = "1"

@ -454,7 +454,7 @@ where
let language_config = loader.language_config_for_scope("source.rust").unwrap(); let language_config = loader.language_config_for_scope("source.rust").unwrap();
let highlight_config = language_config.highlight_config(&[]).unwrap(); let highlight_config = language_config.highlight_config(&[]).unwrap();
let syntax = Syntax::new(&doc, highlight_config.clone()); let syntax = Syntax::new(&doc, highlight_config.clone(), std::sync::Arc::new(loader));
let text = doc.slice(..); let text = doc.slice(..);
let tab_width = 4; let tab_width = 4;

File diff suppressed because it is too large Load Diff

@ -68,13 +68,12 @@ impl EditorView {
surface: &mut Surface, surface: &mut Surface,
theme: &Theme, theme: &Theme,
is_focused: bool, is_focused: bool,
loader: &syntax::Loader,
config: &helix_view::editor::Config, config: &helix_view::editor::Config,
) { ) {
let inner = view.inner_area(); let inner = view.inner_area();
let area = view.area; let area = view.area;
let highlights = Self::doc_syntax_highlights(doc, view.offset, inner.height, theme, loader); let highlights = Self::doc_syntax_highlights(doc, view.offset, inner.height, theme);
let highlights = syntax::merge(highlights, Self::doc_diagnostics_highlights(doc, theme)); let highlights = syntax::merge(highlights, Self::doc_diagnostics_highlights(doc, theme));
let highlights: Box<dyn Iterator<Item = HighlightEvent>> = if is_focused { let highlights: Box<dyn Iterator<Item = HighlightEvent>> = if is_focused {
Box::new(syntax::merge( Box::new(syntax::merge(
@ -121,8 +120,7 @@ impl EditorView {
doc: &'doc Document, doc: &'doc Document,
offset: Position, offset: Position,
height: u16, height: u16,
theme: &Theme, _theme: &Theme,
loader: &syntax::Loader,
) -> Box<dyn Iterator<Item = HighlightEvent> + 'doc> { ) -> Box<dyn Iterator<Item = HighlightEvent> + 'doc> {
let text = doc.text().slice(..); let text = doc.text().slice(..);
let last_line = std::cmp::min( let last_line = std::cmp::min(
@ -142,25 +140,8 @@ impl EditorView {
// TODO: range doesn't actually restrict source, just highlight range // TODO: range doesn't actually restrict source, just highlight range
let highlights = match doc.syntax() { let highlights = match doc.syntax() {
Some(syntax) => { Some(syntax) => {
let scopes = theme.scopes();
syntax syntax
.highlight_iter(text.slice(..), Some(range), None, |language| { .highlight_iter(text.slice(..), Some(range), None)
loader.language_configuration_for_injection_string(language)
.and_then(|language_config| {
let config = language_config.highlight_config(scopes)?;
let config_ref = config.as_ref();
// SAFETY: the referenced `HighlightConfiguration` behind
// the `Arc` is guaranteed to remain valid throughout the
// duration of the highlight.
let config_ref = unsafe {
std::mem::transmute::<
_,
&'static syntax::HighlightConfiguration,
>(config_ref)
};
Some(config_ref)
})
})
.map(|event| event.unwrap()) .map(|event| event.unwrap())
.collect() // TODO: we collect here to avoid holding the lock, fix later .collect() // TODO: we collect here to avoid holding the lock, fix later
} }
@ -1070,7 +1051,6 @@ impl Component for EditorView {
for (view, is_focused) in cx.editor.tree.views() { for (view, is_focused) in cx.editor.tree.views() {
let doc = cx.editor.document(view.doc).unwrap(); let doc = cx.editor.document(view.doc).unwrap();
let loader = &cx.editor.syn_loader;
self.render_view( self.render_view(
doc, doc,
view, view,
@ -1078,7 +1058,6 @@ impl Component for EditorView {
surface, surface,
&cx.editor.theme, &cx.editor.theme,
is_focused, is_focused,
loader,
&cx.editor.config, &cx.editor.config,
); );
} }

@ -38,7 +38,7 @@ impl Markdown {
fn parse<'a>( fn parse<'a>(
contents: &'a str, contents: &'a str,
theme: Option<&Theme>, theme: Option<&Theme>,
loader: &syntax::Loader, loader: Arc<syntax::Loader>,
) -> tui::text::Text<'a> { ) -> tui::text::Text<'a> {
// // also 2021-03-04T16:33:58.553 helix_lsp::transport [INFO] <- {"contents":{"kind":"markdown","value":"\n```rust\ncore::num\n```\n\n```rust\npub const fn saturating_sub(self, rhs:Self) ->Self\n```\n\n---\n\n```rust\n```"},"range":{"end":{"character":61,"line":101},"start":{"character":47,"line":101}}} // // also 2021-03-04T16:33:58.553 helix_lsp::transport [INFO] <- {"contents":{"kind":"markdown","value":"\n```rust\ncore::num\n```\n\n```rust\npub const fn saturating_sub(self, rhs:Self) ->Self\n```\n\n---\n\n```rust\n```"},"range":{"end":{"character":61,"line":101},"start":{"character":47,"line":101}}}
// let text = "\n```rust\ncore::iter::traits::iterator::Iterator\n```\n\n```rust\nfn collect<B: FromIterator<Self::Item>>(self) -> B\nwhere\n Self: Sized,\n```\n\n---\n\nTransforms an iterator into a collection.\n\n`collect()` can take anything iterable, and turn it into a relevant\ncollection. This is one of the more powerful methods in the standard\nlibrary, used in a variety of contexts.\n\nThe most basic pattern in which `collect()` is used is to turn one\ncollection into another. You take a collection, call [`iter`](https://doc.rust-lang.org/nightly/core/iter/traits/iterator/trait.Iterator.html) on it,\ndo a bunch of transformations, and then `collect()` at the end.\n\n`collect()` can also create instances of types that are not typical\ncollections. For example, a [`String`](https://doc.rust-lang.org/nightly/core/iter/std/string/struct.String.html) can be built from [`char`](type@char)s,\nand an iterator of [`Result<T, E>`](https://doc.rust-lang.org/nightly/core/result/enum.Result.html) items can be collected\ninto `Result<Collection<T>, E>`. See the examples below for more.\n\nBecause `collect()` is so general, it can cause problems with type\ninference. As such, `collect()` is one of the few times you'll see\nthe syntax affectionately known as the 'turbofish': `::<>`. This\nhelps the inference algorithm understand specifically which collection\nyou're trying to collect into.\n\n# Examples\n\nBasic usage:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled: Vec<i32> = a.iter()\n .map(|&x| x * 2)\n .collect();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nNote that we needed the `: Vec<i32>` on the left-hand side. This is because\nwe could collect into, for example, a [`VecDeque<T>`](https://doc.rust-lang.org/nightly/core/iter/std/collections/struct.VecDeque.html) instead:\n\n```rust\nuse std::collections::VecDeque;\n\nlet a = [1, 2, 3];\n\nlet doubled: VecDeque<i32> = a.iter().map(|&x| x * 2).collect();\n\nassert_eq!(2, doubled[0]);\nassert_eq!(4, doubled[1]);\nassert_eq!(6, doubled[2]);\n```\n\nUsing the 'turbofish' instead of annotating `doubled`:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled = a.iter().map(|x| x * 2).collect::<Vec<i32>>();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nBecause `collect()` only cares about what you're collecting into, you can\nstill use a partial type hint, `_`, with the turbofish:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled = a.iter().map(|x| x * 2).collect::<Vec<_>>();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nUsing `collect()` to make a [`String`](https://doc.rust-lang.org/nightly/core/iter/std/string/struct.String.html):\n\n```rust\nlet chars = ['g', 'd', 'k', 'k', 'n'];\n\nlet hello: String = chars.iter()\n .map(|&x| x as u8)\n .map(|x| (x + 1) as char)\n .collect();\n\nassert_eq!(\"hello\", hello);\n```\n\nIf you have a list of [`Result<T, E>`](https://doc.rust-lang.org/nightly/core/result/enum.Result.html)s, you can use `collect()` to\nsee if any of them failed:\n\n```rust\nlet results = [Ok(1), Err(\"nope\"), Ok(3), Err(\"bad\")];\n\nlet result: Result<Vec<_>, &str> = results.iter().cloned().collect();\n\n// gives us the first error\nassert_eq!(Err(\"nope\"), result);\n\nlet results = [Ok(1), Ok(3)];\n\nlet result: Result<Vec<_>, &str> = results.iter().cloned().collect();\n\n// gives us the list of answers\nassert_eq!(Ok(vec![1, 3]), result);\n```"; // let text = "\n```rust\ncore::iter::traits::iterator::Iterator\n```\n\n```rust\nfn collect<B: FromIterator<Self::Item>>(self) -> B\nwhere\n Self: Sized,\n```\n\n---\n\nTransforms an iterator into a collection.\n\n`collect()` can take anything iterable, and turn it into a relevant\ncollection. This is one of the more powerful methods in the standard\nlibrary, used in a variety of contexts.\n\nThe most basic pattern in which `collect()` is used is to turn one\ncollection into another. You take a collection, call [`iter`](https://doc.rust-lang.org/nightly/core/iter/traits/iterator/trait.Iterator.html) on it,\ndo a bunch of transformations, and then `collect()` at the end.\n\n`collect()` can also create instances of types that are not typical\ncollections. For example, a [`String`](https://doc.rust-lang.org/nightly/core/iter/std/string/struct.String.html) can be built from [`char`](type@char)s,\nand an iterator of [`Result<T, E>`](https://doc.rust-lang.org/nightly/core/result/enum.Result.html) items can be collected\ninto `Result<Collection<T>, E>`. See the examples below for more.\n\nBecause `collect()` is so general, it can cause problems with type\ninference. As such, `collect()` is one of the few times you'll see\nthe syntax affectionately known as the 'turbofish': `::<>`. This\nhelps the inference algorithm understand specifically which collection\nyou're trying to collect into.\n\n# Examples\n\nBasic usage:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled: Vec<i32> = a.iter()\n .map(|&x| x * 2)\n .collect();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nNote that we needed the `: Vec<i32>` on the left-hand side. This is because\nwe could collect into, for example, a [`VecDeque<T>`](https://doc.rust-lang.org/nightly/core/iter/std/collections/struct.VecDeque.html) instead:\n\n```rust\nuse std::collections::VecDeque;\n\nlet a = [1, 2, 3];\n\nlet doubled: VecDeque<i32> = a.iter().map(|&x| x * 2).collect();\n\nassert_eq!(2, doubled[0]);\nassert_eq!(4, doubled[1]);\nassert_eq!(6, doubled[2]);\n```\n\nUsing the 'turbofish' instead of annotating `doubled`:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled = a.iter().map(|x| x * 2).collect::<Vec<i32>>();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nBecause `collect()` only cares about what you're collecting into, you can\nstill use a partial type hint, `_`, with the turbofish:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled = a.iter().map(|x| x * 2).collect::<Vec<_>>();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nUsing `collect()` to make a [`String`](https://doc.rust-lang.org/nightly/core/iter/std/string/struct.String.html):\n\n```rust\nlet chars = ['g', 'd', 'k', 'k', 'n'];\n\nlet hello: String = chars.iter()\n .map(|&x| x as u8)\n .map(|x| (x + 1) as char)\n .collect();\n\nassert_eq!(\"hello\", hello);\n```\n\nIf you have a list of [`Result<T, E>`](https://doc.rust-lang.org/nightly/core/result/enum.Result.html)s, you can use `collect()` to\nsee if any of them failed:\n\n```rust\nlet results = [Ok(1), Err(\"nope\"), Ok(3), Err(\"bad\")];\n\nlet result: Result<Vec<_>, &str> = results.iter().cloned().collect();\n\n// gives us the first error\nassert_eq!(Err(\"nope\"), result);\n\nlet results = [Ok(1), Ok(3)];\n\nlet result: Result<Vec<_>, &str> = results.iter().cloned().collect();\n\n// gives us the list of answers\nassert_eq!(Ok(vec![1, 3]), result);\n```";
@ -98,14 +98,13 @@ fn parse<'a>(
let syntax = loader let syntax = loader
.language_configuration_for_injection_string(language) .language_configuration_for_injection_string(language)
.and_then(|config| config.highlight_config(theme.scopes())) .and_then(|config| config.highlight_config(theme.scopes()))
.map(|config| Syntax::new(&rope, config)); .map(|config| Syntax::new(&rope, config, loader.clone()));
if let Some(syntax) = syntax { if let Some(syntax) = syntax {
// if we have a syntax available, highlight_iter and generate spans // if we have a syntax available, highlight_iter and generate spans
let mut highlights = Vec::new(); let mut highlights = Vec::new();
for event in syntax.highlight_iter(rope.slice(..), None, None, |_| None) for event in syntax.highlight_iter(rope.slice(..), None, None) {
{
match event.unwrap() { match event.unwrap() {
HighlightEvent::HighlightStart(span) => { HighlightEvent::HighlightStart(span) => {
highlights.push(span); highlights.push(span);
@ -211,7 +210,11 @@ impl Component for Markdown {
fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) { fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) {
use tui::widgets::{Paragraph, Widget, Wrap}; use tui::widgets::{Paragraph, Widget, Wrap};
let text = parse(&self.contents, Some(&cx.editor.theme), &self.config_loader); let text = parse(
&self.contents,
Some(&cx.editor.theme),
self.config_loader.clone(),
);
let par = Paragraph::new(text) let par = Paragraph::new(text)
.wrap(Wrap { trim: false }) .wrap(Wrap { trim: false })
@ -229,7 +232,7 @@ impl Component for Markdown {
if padding >= viewport.1 || padding >= viewport.0 { if padding >= viewport.1 || padding >= viewport.0 {
return None; return None;
} }
let contents = parse(&self.contents, None, &self.config_loader); let contents = parse(&self.contents, None, self.config_loader.clone());
// TODO: account for tab width // TODO: account for tab width
let max_text_width = (viewport.0 - padding).min(120); let max_text_width = (viewport.0 - padding).min(120);
let mut text_width = 0; let mut text_width = 0;

@ -221,13 +221,8 @@ impl<T: 'static> Component for FilePicker<T> {
let offset = Position::new(first_line, 0); let offset = Position::new(first_line, 0);
let highlights = EditorView::doc_syntax_highlights( let highlights =
doc, EditorView::doc_syntax_highlights(doc, offset, area.height, &cx.editor.theme);
offset,
area.height,
&cx.editor.theme,
&cx.editor.syn_loader,
);
EditorView::render_text_highlights( EditorView::render_text_highlights(
doc, doc,
offset, offset,

@ -359,7 +359,7 @@ impl Document {
path: &Path, path: &Path,
encoding: Option<&'static encoding::Encoding>, encoding: Option<&'static encoding::Encoding>,
theme: Option<&Theme>, theme: Option<&Theme>,
config_loader: Option<&syntax::Loader>, config_loader: Option<Arc<syntax::Loader>>,
) -> Result<Self, Error> { ) -> Result<Self, Error> {
// Open the file if it exists, otherwise assume it is a new file (and thus empty). // Open the file if it exists, otherwise assume it is a new file (and thus empty).
let (rope, encoding) = if path.exists() { let (rope, encoding) = if path.exists() {
@ -498,12 +498,12 @@ impl Document {
} }
/// Detect the programming language based on the file type. /// Detect the programming language based on the file type.
pub fn detect_language(&mut self, theme: Option<&Theme>, config_loader: &syntax::Loader) { pub fn detect_language(&mut self, theme: Option<&Theme>, config_loader: Arc<syntax::Loader>) {
if let Some(path) = &self.path { if let Some(path) = &self.path {
let language_config = config_loader let language_config = config_loader
.language_config_for_file_name(path) .language_config_for_file_name(path)
.or_else(|| config_loader.language_config_for_shebang(self.text())); .or_else(|| config_loader.language_config_for_shebang(self.text()));
self.set_language(theme, language_config); self.set_language(theme, language_config, Some(config_loader));
} }
} }
@ -579,11 +579,12 @@ impl Document {
&mut self, &mut self,
theme: Option<&Theme>, theme: Option<&Theme>,
language_config: Option<Arc<helix_core::syntax::LanguageConfiguration>>, language_config: Option<Arc<helix_core::syntax::LanguageConfiguration>>,
loader: Option<Arc<helix_core::syntax::Loader>>,
) { ) {
if let Some(language_config) = language_config { if let (Some(language_config), Some(loader)) = (language_config, loader) {
let scopes = theme.map(|theme| theme.scopes()).unwrap_or(&[]); let scopes = theme.map(|theme| theme.scopes()).unwrap_or(&[]);
if let Some(highlight_config) = language_config.highlight_config(scopes) { if let Some(highlight_config) = language_config.highlight_config(scopes) {
let syntax = Syntax::new(&self.text, highlight_config); let syntax = Syntax::new(&self.text, highlight_config, loader);
self.syntax = Some(syntax); self.syntax = Some(syntax);
// TODO: config.configure(scopes) is now delayed, is that ok? // TODO: config.configure(scopes) is now delayed, is that ok?
} }
@ -605,7 +606,7 @@ impl Document {
) { ) {
let language_config = config_loader.language_config_for_scope(scope); let language_config = config_loader.language_config_for_scope(scope);
self.set_language(theme, language_config); self.set_language(theme, language_config, Some(config_loader));
} }
/// Set the LSP. /// Set the LSP.

@ -283,7 +283,7 @@ impl Editor {
/// Refreshes the language server for a given document /// Refreshes the language server for a given document
pub fn refresh_language_server(&mut self, doc_id: DocumentId) -> Option<()> { pub fn refresh_language_server(&mut self, doc_id: DocumentId) -> Option<()> {
let doc = self.documents.get_mut(&doc_id)?; let doc = self.documents.get_mut(&doc_id)?;
doc.detect_language(Some(&self.theme), &self.syn_loader); doc.detect_language(Some(&self.theme), self.syn_loader.clone());
Self::launch_language_server(&mut self.language_servers, doc) Self::launch_language_server(&mut self.language_servers, doc)
} }
@ -462,7 +462,12 @@ impl Editor {
let id = if let Some(id) = id { let id = if let Some(id) = id {
id id
} else { } else {
let mut doc = Document::open(&path, None, Some(&self.theme), Some(&self.syn_loader))?; let mut doc = Document::open(
&path,
None,
Some(&self.theme),
Some(self.syn_loader.clone()),
)?;
let _ = Self::launch_language_server(&mut self.language_servers, &mut doc); let _ = Self::launch_language_server(&mut self.language_servers, &mut doc);

Loading…
Cancel
Save