Update tree-sitter to 0.20

0.20 includes querying improvements, we no longer have to convert
fragments to strings but can return an iterator of chunks instead.
pull/717/head
Blaž Hrastnik 3 years ago
parent 57ed5180e0
commit 3cb95be452

4
Cargo.lock generated

@ -1034,9 +1034,9 @@ dependencies = [
[[package]] [[package]]
name = "tree-sitter" name = "tree-sitter"
version = "0.19.5" version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad726ec26496bf4c083fff0f43d4eb3a2ad1bba305323af5ff91383c0b6ecac0" checksum = "63ec02a07a782abef91279b72fe8fd2bee4c168a22112cedec5d3b0d49b9e4f9"
dependencies = [ dependencies = [
"cc", "cc",
"regex", "regex",

@ -22,7 +22,7 @@ unicode-segmentation = "1.8"
unicode-width = "0.1" unicode-width = "0.1"
unicode-general-category = "0.4" unicode-general-category = "0.4"
# slab = "0.4.2" # slab = "0.4.2"
tree-sitter = "0.19" tree-sitter = "0.20"
once_cell = "1.8" once_cell = "1.8"
arc-swap = "1" arc-swap = "1"
regex = "1" regex = "1"

@ -276,16 +276,6 @@ fn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<st
Cow::from(source.slice(start_char..end_char)) Cow::from(source.slice(start_char..end_char))
} }
fn node_to_bytes<'a>(node: Node, source: RopeSlice<'a>) -> Cow<'a, [u8]> {
let start_char = source.byte_to_char(node.start_byte());
let end_char = source.byte_to_char(node.end_byte());
let fragment = source.slice(start_char..end_char);
match fragment.as_str() {
Some(fragment) => Cow::Borrowed(fragment.as_bytes()),
None => Cow::Owned(String::from(fragment).into_bytes()),
}
}
impl Syntax { impl Syntax {
// buffer, grammar, config, grammars, sync_timeout? // buffer, grammar, config, grammars, sync_timeout?
pub fn new( pub fn new(
@ -380,14 +370,11 @@ impl Syntax {
// TODO: if reusing cursors this might need resetting // TODO: if reusing cursors this might need resetting
if let Some(range) = &range { if let Some(range) = &range {
cursor_ref.set_byte_range(range.start, range.end); cursor_ref.set_byte_range(range.clone());
} }
let captures = cursor_ref let captures = cursor_ref
.captures(query_ref, tree_ref.root_node(), move |n: Node| { .captures(query_ref, tree_ref.root_node(), RopeProvider(source))
// &source[n.byte_range()]
node_to_bytes(n, source)
})
.peekable(); .peekable();
// manually craft the root layer based on the existing tree // manually craft the root layer based on the existing tree
@ -501,10 +488,7 @@ impl LanguageLayer {
// let mut injections_by_pattern_index = // let mut injections_by_pattern_index =
// vec![(None, Vec::new(), false); combined_injections_query.pattern_count()]; // vec![(None, Vec::new(), false); combined_injections_query.pattern_count()];
// let matches = // let matches =
// cursor.matches(combined_injections_query, tree.root_node(), |n: Node| { // cursor.matches(combined_injections_query, tree.root_node(), RopeProvider(source));
// // &source[n.byte_range()]
// node_to_bytes(n, source)
// });
// for mat in matches { // for mat in matches {
// let entry = &mut injections_by_pattern_index[mat.pattern_index]; // let entry = &mut injections_by_pattern_index[mat.pattern_index];
// let (language_name, content_node, include_children) = // let (language_name, content_node, include_children) =
@ -716,7 +700,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use std::{iter, mem, ops, str, usize}; use std::{iter, mem, ops, str, usize};
use tree_sitter::{ use tree_sitter::{
Language as Grammar, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError, Language as Grammar, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError,
QueryMatch, Range, Tree, QueryMatch, Range, TextProvider, Tree,
}; };
const CANCELLATION_CHECK_INTERVAL: usize = 100; const CANCELLATION_CHECK_INTERVAL: usize = 100;
@ -776,7 +760,7 @@ struct LocalScope<'a> {
} }
#[derive(Debug)] #[derive(Debug)]
struct HighlightIter<'a, 'tree: 'a, F> struct HighlightIter<'a, F>
where where
F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a, F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
{ {
@ -784,16 +768,41 @@ where
byte_offset: usize, byte_offset: usize,
injection_callback: F, injection_callback: F,
cancellation_flag: Option<&'a AtomicUsize>, cancellation_flag: Option<&'a AtomicUsize>,
layers: Vec<HighlightIterLayer<'a, 'tree>>, layers: Vec<HighlightIterLayer<'a>>,
iter_count: usize, iter_count: usize,
next_event: Option<HighlightEvent>, next_event: Option<HighlightEvent>,
last_highlight_range: Option<(usize, usize, usize)>, last_highlight_range: Option<(usize, usize, usize)>,
} }
struct HighlightIterLayer<'a, 'tree: 'a> { // Adapter to convert rope chunks to bytes
struct ChunksBytes<'a> {
chunks: ropey::iter::Chunks<'a>,
}
impl<'a> Iterator for ChunksBytes<'a> {
type Item = &'a [u8];
fn next(&mut self) -> Option<Self::Item> {
self.chunks.next().map(str::as_bytes)
}
}
struct RopeProvider<'a>(RopeSlice<'a>);
impl<'a> TextProvider<'a> for RopeProvider<'a> {
type I = ChunksBytes<'a>;
fn text(&mut self, node: Node) -> Self::I {
let start_char = self.0.byte_to_char(node.start_byte());
let end_char = self.0.byte_to_char(node.end_byte());
let fragment = self.0.slice(start_char..end_char);
ChunksBytes {
chunks: fragment.chunks(),
}
}
}
struct HighlightIterLayer<'a> {
_tree: Option<Tree>, _tree: Option<Tree>,
cursor: QueryCursor, cursor: QueryCursor,
captures: iter::Peekable<QueryCaptures<'a, 'tree, Cow<'a, [u8]>>>, captures: iter::Peekable<QueryCaptures<'a, 'a, RopeProvider<'a>>>,
config: &'a HighlightConfiguration, config: &'a HighlightConfiguration,
highlight_end_stack: Vec<usize>, highlight_end_stack: Vec<usize>,
scope_stack: Vec<LocalScope<'a>>, scope_stack: Vec<LocalScope<'a>>,
@ -801,7 +810,7 @@ struct HighlightIterLayer<'a, 'tree: 'a> {
depth: usize, depth: usize,
} }
impl<'a, 'tree: 'a> fmt::Debug for HighlightIterLayer<'a, 'tree> { impl<'a> fmt::Debug for HighlightIterLayer<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("HighlightIterLayer").finish() f.debug_struct("HighlightIterLayer").finish()
} }
@ -972,7 +981,7 @@ impl HighlightConfiguration {
} }
} }
impl<'a, 'tree: 'a> HighlightIterLayer<'a, 'tree> { impl<'a> HighlightIterLayer<'a> {
/// Create a new 'layer' of highlighting for this document. /// Create a new 'layer' of highlighting for this document.
/// ///
/// In the even that the new layer contains "combined injections" (injections where multiple /// In the even that the new layer contains "combined injections" (injections where multiple
@ -1029,10 +1038,7 @@ impl<'a, 'tree: 'a> HighlightIterLayer<'a, 'tree> {
let matches = cursor.matches( let matches = cursor.matches(
combined_injections_query, combined_injections_query,
tree.root_node(), tree.root_node(),
|n: Node| { RopeProvider(source),
// &source[n.byte_range()]
node_to_bytes(n, source)
},
); );
for mat in matches { for mat in matches {
let entry = &mut injections_by_pattern_index[mat.pattern_index]; let entry = &mut injections_by_pattern_index[mat.pattern_index];
@ -1079,10 +1085,7 @@ impl<'a, 'tree: 'a> HighlightIterLayer<'a, 'tree> {
let cursor_ref = let cursor_ref =
unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) }; unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
let captures = cursor_ref let captures = cursor_ref
.captures(&config.query, tree_ref.root_node(), move |n: Node| { .captures(&config.query, tree_ref.root_node(), RopeProvider(source))
// &source[n.byte_range()]
node_to_bytes(n, source)
})
.peekable(); .peekable();
result.push(HighlightIterLayer { result.push(HighlightIterLayer {
@ -1236,7 +1239,7 @@ impl<'a, 'tree: 'a> HighlightIterLayer<'a, 'tree> {
} }
} }
impl<'a, 'tree: 'a, F> HighlightIter<'a, 'tree, F> impl<'a, F> HighlightIter<'a, F>
where where
F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a, F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
{ {
@ -1287,7 +1290,7 @@ where
} }
} }
fn insert_layer(&mut self, mut layer: HighlightIterLayer<'a, 'tree>) { fn insert_layer(&mut self, mut layer: HighlightIterLayer<'a>) {
if let Some(sort_key) = layer.sort_key() { if let Some(sort_key) = layer.sort_key() {
let mut i = 1; let mut i = 1;
while i < self.layers.len() { while i < self.layers.len() {
@ -1306,7 +1309,7 @@ where
} }
} }
impl<'a, 'tree: 'a, F> Iterator for HighlightIter<'a, 'tree, F> impl<'a, F> Iterator for HighlightIter<'a, F>
where where
F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a, F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
{ {
@ -1570,7 +1573,7 @@ where
fn injection_for_match<'a>( fn injection_for_match<'a>(
config: &HighlightConfiguration, config: &HighlightConfiguration,
query: &'a Query, query: &'a Query,
query_match: &QueryMatch<'a>, query_match: &QueryMatch<'a, 'a>,
source: RopeSlice<'a>, source: RopeSlice<'a>,
) -> (Option<Cow<'a, str>>, Option<Node<'a>>, bool) { ) -> (Option<Cow<'a, str>>, Option<Node<'a>>, bool) {
let content_capture_index = config.injection_content_capture_index; let content_capture_index = config.injection_content_capture_index;

@ -11,7 +11,7 @@ homepage = "https://helix-editor.com"
include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"] include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"]
[dependencies] [dependencies]
tree-sitter = "0.19" tree-sitter = "0.20"
libloading = "0.7" libloading = "0.7"
anyhow = "1" anyhow = "1"

Loading…
Cancel
Save