@ -9,6 +9,7 @@ use crate::{
pub use helix_syntax ::get_language ; pub use helix_syntax ::get_language ;
use arc_swap ::ArcSwap ; use arc_swap ::ArcSwap ;
use slotmap ::{ DefaultKey as LayerId , HopSlotMap } ;
use std ::{ use std ::{
borrow ::Cow , borrow ::Cow ,
@ -388,9 +389,9 @@ thread_local! {
#[ derive(Debug) ] #[ derive(Debug) ]
pub struct Syntax { pub struct Syntax {
config: Arc < HighlightConfiguration > , layers: HopSlotMap < LayerId , LanguageLayer > ,
root : LayerId ,
root_layer: LanguageLayer , loader: Arc < Loader > ,
} }
fn byte_range_to_str ( range : std ::ops ::Range < usize > , source : RopeSlice ) -> Cow < str > { fn byte_range_to_str ( range : std ::ops ::Range < usize > , source : RopeSlice ) -> Cow < str > {
@ -400,38 +401,36 @@ fn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<st
} }
impl Syntax { impl Syntax {
// buffer, grammar, config, grammars, sync_timeout?
pub fn new ( source : & Rope , config : Arc < HighlightConfiguration > , loader : Arc < Loader > ) -> Self {
pub fn new ( let root_layer = LanguageLayer {
/* language: Lang, */ source : & Rope , tree : None ,
config : Arc < HighlightConfiguration > , config ,
) -> Self { depth : 0 ,
let root_layer = LanguageLayer { tree : None } ; ranges : vec ! [ Range {
start_byte : 0 ,
end_byte : usize ::MAX ,
start_point : Point ::new ( 0 , 0 ) ,
end_point : Point ::new ( usize ::MAX , usize ::MAX ) ,
} ] ,
} ;
// track markers of injections
// track markers of injections
// track scope_descriptor: a Vec of scopes for item in tree
// track scope_descriptor: a Vec of scopes for item in tree
let mut layers = HopSlotMap ::default ( ) ;
let root = layers . insert ( root_layer ) ;
let mut syntax = Self { let mut syntax = Self {
// grammar,
// grammar,
config , root ,
root_layer , layers ,
loader ,
} ; } ;
// update root layer
syntax
PARSER . with ( | ts_parser | { . update ( source , source , & ChangeSet ::new ( & source ) )
// TODO: handle the returned `Result` properly.
. unwrap ( ) ;
let _ = syntax . root_layer . parse (
& mut ts_parser . borrow_mut ( ) ,
& syntax . config ,
source ,
0 ,
vec! [ Range {
start_byte : 0 ,
end_byte : usize ::MAX ,
start_point : Point ::new ( 0 , 0 ) ,
end_point : Point ::new ( usize ::MAX , usize ::MAX ) ,
} ] ,
) ;
} ) ;
syntax syntax
} }
@ -441,30 +440,197 @@ impl Syntax {
source : & Rope , source : & Rope ,
changeset : & ChangeSet , changeset : & ChangeSet ,
) -> Result < ( ) , Error > { ) -> Result < ( ) , Error > {
PARSER . with ( | ts_parser | { use std ::collections ::VecDeque ;
self . root_layer . update ( let mut queue = VecDeque ::new ( ) ;
& mut ts_parser . borrow_mut ( ) , // let source = source.slice(..);
& self . config , let injection_callback = | language : & str | {
old_source , self . loader
source , . language_configuration_for_injection_string ( language )
changeset , . and_then ( | language_config | {
// TODO: get these theme.scopes from somewhere, probably make them settable on Loader
let scopes = & [
"attribute" ,
"constant" ,
"function.builtin" ,
"function" ,
"keyword" ,
"operator" ,
"property" ,
"punctuation" ,
"punctuation.bracket" ,
"punctuation.delimiter" ,
"string" ,
"string.special" ,
"tag" ,
"type" ,
"type.builtin" ,
"variable" ,
"variable.builtin" ,
"variable.parameter" ,
] ;
language_config . highlight_config (
& scopes
. iter ( )
. map ( | scope | scope . to_string ( ) )
. collect ::< Vec < _ > > ( ) ,
) )
} ) } )
} ;
queue . push_back ( self . root ) ;
// HAXX: for now, clear all layers except root so they get re-parsed
self . layers . retain ( | id , _ | id = = self . root ) ;
// Workaround for Syntax::new() with empty changeset
if ! changeset . is_empty ( ) {
// TODO: do this in a recursive way
// Notify the tree about all the changes
let edits = generate_edits ( old_source . slice ( .. ) , changeset ) ;
let tree = self . layers [ self . root ] . tree . as_mut ( ) . unwrap ( ) ;
for edit in edits . iter ( ) . rev ( ) {
// apply the edits in reverse. If we applied them in order then edit 1 would disrupt
// the positioning of edit 2
tree . edit ( edit ) ;
}
}
PARSER . with ( | ts_parser | {
let ts_parser = & mut ts_parser . borrow_mut ( ) ;
let mut cursor = ts_parser . cursors . pop ( ) . unwrap_or_else ( QueryCursor ::new ) ;
// TODO: might need to set cursor range
while let Some ( layer_id ) = queue . pop_front ( ) {
// Re-parse the tree.
self . layers [ layer_id ] . parse ( ts_parser , source ) ? ;
let source = source . slice ( .. ) ;
let layer = & self . layers [ layer_id ] ;
// Process injections.
let matches = cursor . matches (
& layer . config . injections_query ,
layer . tree ( ) . root_node ( ) ,
RopeProvider ( source ) ,
) ;
let mut injections = Vec ::new ( ) ;
for mat in matches {
let ( language_name , content_node , include_children ) = injection_for_match (
& layer . config ,
& layer . config . injections_query ,
& mat ,
source ,
) ;
// TODO: deal with injections and update them too
// Explicitly remove this match so that none of its other captures will remain
// in the stream of captures.
mat . remove ( ) ; // TODO: is this still necessary?
// If a language is found with the given name, then add a new language layer
// to the highlighted document.
if let ( Some ( language_name ) , Some ( content_node ) ) = ( language_name , content_node )
{
if let Some ( config ) = ( injection_callback ) ( & language_name ) {
let ranges =
intersect_ranges ( & layer . ranges , & [ content_node ] , include_children ) ;
if ! ranges . is_empty ( ) {
log ::info ! ( "{} {:?}" , language_name , ranges ) ;
injections . push ( ( config , ranges ) ) ;
}
}
}
} }
// fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers
// Process combined injections.
if let Some ( combined_injections_query ) = & layer . config . combined_injections_query {
let mut injections_by_pattern_index =
vec! [ ( None , Vec ::new ( ) , false ) ; combined_injections_query . pattern_count ( ) ] ;
let matches = cursor . matches (
combined_injections_query ,
layer . tree ( ) . root_node ( ) ,
RopeProvider ( source ) ,
) ;
for mat in matches {
let entry = & mut injections_by_pattern_index [ mat . pattern_index ] ;
let ( language_name , content_node , include_children ) = injection_for_match (
& layer . config ,
combined_injections_query ,
& mat ,
source ,
) ;
if language_name . is_some ( ) {
entry . 0 = language_name ;
}
if let Some ( content_node ) = content_node {
entry . 1. push ( content_node ) ;
}
entry . 2 = include_children ;
}
for ( lang_name , content_nodes , includes_children ) in injections_by_pattern_index
{
if let ( Some ( lang_name ) , false ) = ( lang_name , content_nodes . is_empty ( ) ) {
if let Some ( config ) = ( injection_callback ) ( & lang_name ) {
let ranges = intersect_ranges (
& layer . ranges ,
& content_nodes ,
includes_children ,
) ;
if ! ranges . is_empty ( ) {
injections . push ( ( config , ranges ) ) ;
}
}
}
}
}
let depth = layer . depth + 1 ;
// TODO: can't inline this since matches borrows self.layers
for ( config , ranges ) in injections {
let layer_id = self . layers . insert ( LanguageLayer {
tree : None ,
config ,
depth ,
ranges ,
} ) ;
queue . push_back ( layer_id ) ;
}
}
// Return the cursor back in the pool.
ts_parser . cursors . push ( cursor ) ;
Ok ( ( ) ) // so we can use the try operator
} ) ? ;
Ok ( ( ) )
}
// fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers
// call this on transaction.apply() -> buffer_changed(changes)
// call this on transaction.apply() -> buffer_changed(changes)
//
// fn parse(language, old_tree, ranges)
//
pub fn tree ( & self ) -> & Tree { pub fn tree ( & self ) -> & Tree {
self . root_layer . tree ( ) self . layers [ self . root ] . tree ( )
} }
// root: Tree
// injections: Vec<(Tree, Range marker)>
// handle updates that go over a part of the layer by truncating them to start/end appropriately
// injections tracked by marker:
// if marker areas match it's fine and update
// if not found add new layer
// if length 0 then area got removed, clean up the layer
//
//
// <!--update_for_injection(grammar)-->
// layer update:
// if range.len = 0 then remove the layer
// for change in changes { tree.edit(change) }
// tree = parser.parse(.., tree, ..)
// calculate affected range and update injections
// injection update:
// look for existing injections
// if present, range = (first injection start, last injection end)
// Highlighting
// Highlighting
@ -474,31 +640,37 @@ impl Syntax {
source : RopeSlice < ' a > , source : RopeSlice < ' a > ,
range : Option < std ::ops ::Range < usize > > , range : Option < std ::ops ::Range < usize > > ,
cancellation_flag : Option < & ' a AtomicUsize > , cancellation_flag : Option < & ' a AtomicUsize > ,
injection_callback : impl FnMut ( & str ) -> Option < & ' a HighlightConfiguration > + ' a ,
) -> impl Iterator < Item = Result < HighlightEvent , Error > > + ' a { ) -> impl Iterator < Item = Result < HighlightEvent , Error > > + ' a {
// The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
let mut layers = self
// prevents them from being moved. But both of these values are really just
. layers
// pointers, so it's actually ok to move them.
. iter ( )
. map ( | ( _ , layer ) | {
// reuse a cursor from the pool if possible // Reuse a cursor from the pool if available.
let mut cursor = PARSER . with ( | ts_parser | { let mut cursor = PARSER . with ( | ts_parser | {
let highlighter = & mut ts_parser . borrow_mut ( ) ; let highlighter = & mut ts_parser . borrow_mut ( ) ;
highlighter . cursors . pop ( ) . unwrap_or_else ( QueryCursor ::new ) highlighter . cursors . pop ( ) . unwrap_or_else ( QueryCursor ::new )
} ) ; } ) ;
let tree_ref = self . tree ( ) ;
let cursor_ref = unsafe { mem ::transmute ::< _ , & ' static mut QueryCursor > ( & mut cursor ) } ; // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
let query_ref = & self . config . query ; // prevents them from being moved. But both of these values are really just
let config_ref = self . config . as_ref ( ) ; // pointers, so it's actually ok to move them.
let cursor_ref =
unsafe { mem ::transmute ::< _ , & ' static mut QueryCursor > ( & mut cursor ) } ;
// if reusing cursors & no range this resets to whole range
// if reusing cursors & no range this resets to whole range
cursor_ref . set_byte_range ( range . clone ( ) . unwrap_or ( 0 .. usize ::MAX ) ) ; // TODO: handle intersect (range & layer.range)
// cursor_ref.set_byte_range(range.clone().unwrap_or(0..usize::MAX));
cursor_ref . set_byte_range ( 0 .. usize ::MAX ) ;
let captures = cursor_ref let captures = cursor_ref
. captures ( query_ref , tree_ref . root_node ( ) , RopeProvider ( source ) ) . captures (
& layer . config . query ,
layer . tree ( ) . root_node ( ) ,
RopeProvider ( source ) ,
)
. peekable ( ) ; . peekable ( ) ;
// manually craft the root layer based on the existing tree
HighlightIterLayer {
let layer = HighlightIterLayer {
highlight_end_stack : Vec ::new ( ) , highlight_end_stack : Vec ::new ( ) ,
scope_stack : vec ! [ LocalScope { scope_stack : vec ! [ LocalScope {
inherits : false , inherits : false ,
@ -506,29 +678,38 @@ impl Syntax {
local_defs : Vec ::new ( ) , local_defs : Vec ::new ( ) ,
} ] , } ] ,
cursor , cursor ,
depth : 0 ,
_tree : None , _tree : None ,
captures , captures ,
config : config_ref , config : layer . config . as_ref ( ) , // TODO: just reuse
ranges : vec ! [ Range { depth : layer . depth , // TODO: just reuse
start_byte : 0 , ranges : layer . ranges . clone ( ) ,
end_byte : usize ::MAX , }
start_point : Point ::new ( 0 , 0 ) , } )
end_point : Point ::new ( usize ::MAX , usize ::MAX ) , . collect ::< Vec < _ > > ( ) ;
} ] ,
} ; log ::info ! ( "--" ) ;
// HAXX: arrange layers by byte range, with deeper layers positioned first
layers . sort_by_key ( | layer | {
(
layer . ranges . first ( ) . cloned ( ) ,
std ::cmp ::Reverse ( layer . depth ) ,
)
} ) ;
let mut result = HighlightIter { let mut result = HighlightIter {
source , source ,
byte_offset : range . map_or ( 0 , | r | r . start ) , // TODO: simplify
byte_offset : range . map_or ( 0 , | r | r . start ) , // TODO: simplify
injection_callback ,
cancellation_flag , cancellation_flag ,
iter_count : 0 , iter_count : 0 ,
layers : vec ! [ layer ] , layers ,
next_event : None , next_event : None ,
last_highlight_range : None , last_highlight_range : None ,
} ; } ;
result . sort_layers ( ) ; result . sort_layers ( ) ;
for layer in & result . layers {
log ::info ! ( "> {:?} {:?}" , layer . depth , layer . ranges ) ; // <- for some reason layers are reversed here
}
result result
} }
// on_tokenize
// on_tokenize
@ -556,32 +737,24 @@ impl Syntax {
pub struct LanguageLayer { pub struct LanguageLayer {
// mode
// mode
// grammar
// grammar
// depth
pub config : Arc < HighlightConfiguration > ,
pub ( crate ) tree : Option < Tree > , pub ( crate ) tree : Option < Tree > ,
pub ranges : Vec < Range > ,
pub depth : usize ,
} }
impl LanguageLayer { impl LanguageLayer {
// pub fn new() -> Self {
// Self { tree: None }
// }
pub fn tree ( & self ) -> & Tree { pub fn tree ( & self ) -> & Tree {
// TODO: no unwrap
// TODO: no unwrap
self . tree . as_ref ( ) . unwrap ( ) self . tree . as_ref ( ) . unwrap ( )
} }
fn parse ( fn parse ( & mut self , ts_parser : & mut TsParser , source : & Rope ) -> Result < ( ) , Error > {
& mut self , ts_parser . parser . set_included_ranges ( & self . ranges ) . unwrap ( ) ;
ts_parser : & mut TsParser ,
config : & HighlightConfiguration ,
source : & Rope ,
_depth : usize ,
ranges : Vec < Range > ,
) -> Result < ( ) , Error > {
if ts_parser . parser . set_included_ranges ( & ranges ) . is_ok ( ) {
ts_parser ts_parser
. parser . parser
. set_language ( config . language ) . set_language ( self . config . language )
. map_err ( | _ | Error ::InvalidLanguage ) ? ; . map_err ( | _ | Error ::InvalidLanguage ) ? ;
// unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };
// unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };
@ -600,11 +773,11 @@ impl LanguageLayer {
self . tree . as_ref ( ) , self . tree . as_ref ( ) ,
) )
. ok_or ( Error ::Cancelled ) ? ; . ok_or ( Error ::Cancelled ) ? ;
// unsafe { ts_parser.parser.set_cancellation_flag(None) };
self . tree = Some ( tree ) self . tree = Some ( tree ) ;
}
Ok ( ( ) ) Ok ( ( ) )
} }
}
pub ( crate ) fn generate_edits ( pub ( crate ) fn generate_edits (
old_text : RopeSlice , old_text : RopeSlice ,
@ -713,77 +886,6 @@ impl LanguageLayer {
edits edits
} }
fn update (
& mut self ,
ts_parser : & mut TsParser ,
config : & HighlightConfiguration ,
old_source : & Rope ,
source : & Rope ,
changeset : & ChangeSet ,
) -> Result < ( ) , Error > {
if changeset . is_empty ( ) {
return Ok ( ( ) ) ;
}
let edits = Self ::generate_edits ( old_source . slice ( .. ) , changeset ) ;
// Notify the tree about all the changes
for edit in edits . iter ( ) . rev ( ) {
// apply the edits in reverse. If we applied them in order then edit 1 would disrupt
// the positioning of edit 2
self . tree . as_mut ( ) . unwrap ( ) . edit ( edit ) ;
}
self . parse (
ts_parser ,
config ,
source ,
0 ,
// TODO: what to do about this range on update
vec! [ Range {
start_byte : 0 ,
end_byte : usize ::MAX ,
start_point : Point ::new ( 0 , 0 ) ,
end_point : Point ::new ( usize ::MAX , usize ::MAX ) ,
} ] ,
)
}
// fn highlight_iter() -> same as Mode but for this layer. Mode composits these
// fn buffer_changed
// fn update(range)
// fn update_injections()
}
// -- refactored from tree-sitter-highlight to be able to retain state
// TODO: add seek() to iter
// problem: any time a layer is updated it must update it's injections on the parent (potentially
// removing some from use)
// can't modify to vec and exist in it at the same time since that would violate borrows
// maybe we can do with an arena
// maybe just caching on the top layer and nevermind the injections for now?
//
// Grammar {
// layers: Vec<Box<Layer>> to prevent memory moves when vec is modified
// }
// injections tracked by marker:
// if marker areas match it's fine and update
// if not found add new layer
// if length 0 then area got removed, clean up the layer
//
// layer update:
// if range.len = 0 then remove the layer
// for change in changes { tree.edit(change) }
// tree = parser.parse(.., tree, ..)
// calculate affected range and update injections
// injection update:
// look for existing injections
// if present, range = (first injection start, last injection end)
//
// For now cheat and just throw out non-root layers if they exist. This should still improve
// parsing in majority of cases.
use std ::sync ::atomic ::{ AtomicUsize , Ordering } ; use std ::sync ::atomic ::{ AtomicUsize , Ordering } ;
use std ::{ iter , mem , ops , str , usize } ; use std ::{ iter , mem , ops , str , usize } ;
use tree_sitter ::{ use tree_sitter ::{
@ -820,8 +922,8 @@ pub enum HighlightEvent {
pub struct HighlightConfiguration { pub struct HighlightConfiguration {
pub language : Grammar , pub language : Grammar ,
pub query : Query , pub query : Query ,
injections_query : Query ,
combined_injections_query : Option < Query > , combined_injections_query : Option < Query > ,
locals_pattern_index : usize ,
highlights_pattern_index : usize , highlights_pattern_index : usize ,
highlight_indices : ArcSwap < Vec < Option < Highlight > > > , highlight_indices : ArcSwap < Vec < Option < Highlight > > > ,
non_local_variable_patterns : Vec < bool > , non_local_variable_patterns : Vec < bool > ,
@ -848,13 +950,9 @@ struct LocalScope<'a> {
} }
#[ derive(Debug) ] #[ derive(Debug) ]
struct HighlightIter < ' a , F > struct HighlightIter < ' a > {
where
F : FnMut ( & str ) -> Option < & ' a HighlightConfiguration > + ' a ,
{
source : RopeSlice < ' a > , source : RopeSlice < ' a > ,
byte_offset : usize , byte_offset : usize ,
injection_callback : F ,
cancellation_flag : Option < & ' a AtomicUsize > , cancellation_flag : Option < & ' a AtomicUsize > ,
layers : Vec < HighlightIterLayer < ' a > > , layers : Vec < HighlightIterLayer < ' a > > ,
iter_count : usize , iter_count : usize ,
@ -894,8 +992,8 @@ struct HighlightIterLayer<'a> {
config : & ' a HighlightConfiguration , config : & ' a HighlightConfiguration ,
highlight_end_stack : Vec < usize > , highlight_end_stack : Vec < usize > ,
scope_stack : Vec < LocalScope < ' a > > , scope_stack : Vec < LocalScope < ' a > > ,
ranges : Vec < Range > ,
depth : usize , depth : usize ,
ranges : Vec < Range > , // TEMP
} }
impl < ' a > fmt ::Debug for HighlightIterLayer < ' a > { impl < ' a > fmt ::Debug for HighlightIterLayer < ' a > {
@ -927,38 +1025,32 @@ impl HighlightConfiguration {
) -> Result < Self , QueryError > { ) -> Result < Self , QueryError > {
// Concatenate the query strings, keeping track of the start offset of each section.
// Concatenate the query strings, keeping track of the start offset of each section.
let mut query_source = String ::new ( ) ; let mut query_source = String ::new ( ) ;
query_source . push_str ( injection_query ) ;
let locals_query_offset = query_source . len ( ) ;
query_source . push_str ( locals_query ) ; query_source . push_str ( locals_query ) ;
let highlights_query_offset = query_source . len ( ) ; let highlights_query_offset = query_source . len ( ) ;
query_source . push_str ( highlights_query ) ; query_source . push_str ( highlights_query ) ;
// Construct a single query by concatenating the three query strings, but record the
// Construct a single query by concatenating the three query strings, but record the
// range of pattern indices that belong to each individual string.
// range of pattern indices that belong to each individual string.
let mut query = Query ::new ( language , & query_source ) ? ; let query = Query ::new ( language , & query_source ) ? ;
let mut locals_pattern_index = 0 ;
let mut highlights_pattern_index = 0 ; let mut highlights_pattern_index = 0 ;
for i in 0 .. ( query . pattern_count ( ) ) { for i in 0 .. ( query . pattern_count ( ) ) {
let pattern_offset = query . start_byte_for_pattern ( i ) ; let pattern_offset = query . start_byte_for_pattern ( i ) ;
if pattern_offset < highlights_query_offset {
if pattern_offset < highlights_query_offset { if pattern_offset < highlights_query_offset {
highlights_pattern_index + = 1 ; highlights_pattern_index + = 1 ;
} }
if pattern_offset < locals_query_offset {
locals_pattern_index + = 1 ;
}
}
} }
let mut injections_query = Query ::new ( language , injection_query ) ? ;
// Construct a separate query just for dealing with the 'combined injections'.
// Construct a separate query just for dealing with the 'combined injections'.
// Disable the combined injection patterns in the main query.
// Disable the combined injection patterns in the main query.
let mut combined_injections_query = Query ::new ( language , injection_query ) ? ; let mut combined_injections_query = Query ::new ( language , injection_query ) ? ;
let mut has_combined_queries = false ; let mut has_combined_queries = false ;
for pattern_index in 0 .. locals_pattern_index { for pattern_index in 0 .. injections_query. pattern_count ( ) {
let settings = . property_settings ( pattern_index ) ; let settings = injections_ query. property_settings ( pattern_index ) ;
if settings . iter ( ) . any ( | s | & * s . key = = "injection.combined" ) { if settings . iter ( ) . any ( | s | & * s . key = = "injection.combined" ) {
has_combined_queries = true ; has_combined_queries = true ;
. disable_pattern ( pattern_index ) ; injections_ query. disable_pattern ( pattern_index ) ;
} else { } else {
combined_injections_query . disable_pattern ( pattern_index ) ; combined_injections_query . disable_pattern ( pattern_index ) ;
} }
@ -990,8 +1082,6 @@ impl HighlightConfiguration {
for ( i , name ) in query . capture_names ( ) . iter ( ) . enumerate ( ) { for ( i , name ) in query . capture_names ( ) . iter ( ) . enumerate ( ) {
let i = Some ( i as u32 ) ; let i = Some ( i as u32 ) ;
match name . as_str ( ) { match name . as_str ( ) {
"injection.content" = > injection_content_capture_index = i ,
"injection.language" = > injection_language_capture_index = i ,
"local.definition" = > local_def_capture_index = i , "local.definition" = > local_def_capture_index = i ,
"local.definition-value" = > local_def_value_capture_index = i , "local.definition-value" = > local_def_value_capture_index = i ,
"local.reference" = > local_ref_capture_index = i , "local.reference" = > local_ref_capture_index = i ,
@ -1000,12 +1090,21 @@ impl HighlightConfiguration {
} }
} }
for ( i , name ) in injections_query . capture_names ( ) . iter ( ) . enumerate ( ) {
let i = Some ( i as u32 ) ;
match name . as_str ( ) {
"injection.content" = > injection_content_capture_index = i ,
"injection.language" = > injection_language_capture_index = i ,
_ = > { }
}
}
let highlight_indices = ArcSwap ::from_pointee ( vec! [ None ; query . capture_names ( ) . len ( ) ] ) ; let highlight_indices = ArcSwap ::from_pointee ( vec! [ None ; query . capture_names ( ) . len ( ) ] ) ;
Ok ( Self { Ok ( Self {
language , language ,
query , query ,
injections_query ,
combined_injections_query , combined_injections_query ,
locals_pattern_index ,
highlights_pattern_index , highlights_pattern_index ,
highlight_indices , highlight_indices ,
non_local_variable_patterns , non_local_variable_patterns ,
@ -1070,142 +1169,29 @@ impl HighlightConfiguration {
} }
impl < ' a > HighlightIterLayer < ' a > { impl < ' a > HighlightIterLayer < ' a > {
/// Create a new 'layer' of highlighting for this document.
// First, sort scope boundaries by their byte offset in the document. At a
///
// given position, emit scope endings before scope beginnings. Finally, emit
/// In the even that the new layer contains "combined injections" (injections where multiple
// scope boundaries from deeper layers first.
/// disjoint ranges are parsed as one syntax tree), these will be eagerly processed and
fn sort_key ( & mut self ) -> Option < ( usize , bool , isize ) > {
/// added to the returned vector.
let depth = - ( self . depth as isize ) ;
fn new < F : FnMut ( & str ) -> Option < & ' a HighlightConfiguration > + ' a > ( let next_start = self
source : RopeSlice < ' a > , . captures
cancellation_flag : Option < & ' a AtomicUsize > , . peek ( )
injection_callback : & mut F , . map ( | ( m , i ) | m . captures [ * i ] . node . start_byte ( ) ) ;
mut config : & ' a HighlightConfiguration , let next_end = self . highlight_end_stack . last ( ) . cloned ( ) ;
mut depth : usize , match ( next_start , next_end ) {
mut ranges : Vec < Range > , ( Some ( start ) , Some ( end ) ) = > {
) -> Result < Vec < Self > , Error > { if start < end {
let mut result = Vec ::with_capacity ( 1 ) ; Some ( ( start , true , depth ) )
let mut queue = Vec ::new ( ) ;
loop {
// --> Tree parsing part
PARSER . with ( | ts_parser | {
let highlighter = & mut ts_parser . borrow_mut ( ) ;
if highlighter . parser . set_included_ranges ( & ranges ) . is_ok ( ) {
highlighter
. parser
. set_language ( config . language )
. map_err ( | _ | Error ::InvalidLanguage ) ? ;
unsafe { highlighter . parser . set_cancellation_flag ( cancellation_flag ) } ;
let tree = highlighter
. parser
. parse_with (
& mut | byte , _ | {
if byte < = source . len_bytes ( ) {
let ( chunk , start_byte , _ , _ ) = source . chunk_at_byte ( byte ) ;
chunk [ byte - start_byte .. ] . as_bytes ( )
} else { } else {
// out of range
Some ( ( end , false , depth ) )
& [ ]
}
} ,
None ,
)
. ok_or ( Error ::Cancelled ) ? ;
unsafe { highlighter . parser . set_cancellation_flag ( None ) } ;
let mut cursor = highlighter . cursors . pop ( ) . unwrap_or_else ( QueryCursor ::new ) ;
// Process combined injections.
if let Some ( combined_injections_query ) = & config . combined_injections_query {
let mut injections_by_pattern_index = vec! [
( None , Vec ::new ( ) , false ) ;
combined_injections_query
. pattern_count ( )
] ;
let matches = cursor . matches (
combined_injections_query ,
tree . root_node ( ) ,
RopeProvider ( source ) ,
) ;
for mat in matches {
let entry = & mut injections_by_pattern_index [ mat . pattern_index ] ;
let ( language_name , content_node , include_children ) =
injection_for_match (
config ,
combined_injections_query ,
& mat ,
source ,
) ;
if language_name . is_some ( ) {
entry . 0 = language_name ;
}
if let Some ( content_node ) = content_node {
entry . 1. push ( content_node ) ;
}
entry . 2 = include_children ;
}
for ( lang_name , content_nodes , includes_children ) in
injections_by_pattern_index
{
if let ( Some ( lang_name ) , false ) = ( lang_name , content_nodes . is_empty ( ) )
{
if let Some ( next_config ) = ( injection_callback ) ( & lang_name ) {
let ranges = Self ::intersect_ranges (
& ranges ,
& content_nodes ,
includes_children ,
) ;
if ! ranges . is_empty ( ) {
queue . push ( ( next_config , depth + 1 , ranges ) ) ;
}
}
}
} }
} }
( Some ( i ) , None ) = > Some ( ( i , true , depth ) ) ,
// --> Highlighting query part
( None , Some ( j ) ) = > Some ( ( j , false , depth ) ) ,
_ = > None ,
// The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
// prevents them from being moved. But both of these values are really just
// pointers, so it's actually ok to move them.
let tree_ref = unsafe { mem ::transmute ::< _ , & ' static Tree > ( & tree ) } ;
let cursor_ref =
unsafe { mem ::transmute ::< _ , & ' static mut QueryCursor > ( & mut cursor ) } ;
let captures = cursor_ref
. captures ( & config . query , tree_ref . root_node ( ) , RopeProvider ( source ) )
. peekable ( ) ;
result . push ( HighlightIterLayer {
highlight_end_stack : Vec ::new ( ) ,
scope_stack : vec ! [ LocalScope {
inherits : false ,
range : 0 .. usize ::MAX ,
local_defs : Vec ::new ( ) ,
} ] ,
cursor ,
depth ,
_tree : Some ( tree ) ,
captures ,
config ,
ranges ,
} ) ;
}
Ok ( ( ) ) // so we can use the try operator
} ) ? ;
if queue . is_empty ( ) {
break ;
} }
let ( next_config , next_depth , next_ranges ) = queue . remove ( 0 ) ;
config = next_config ;
depth = next_depth ;
ranges = next_ranges ;
} }
Ok ( result )
} }
// Compute the ranges that should be included when parsing an injection.
// Compute the ranges that should be included when parsing an injection.
@ -1302,35 +1288,7 @@ impl<'a> HighlightIterLayer<'a> {
result result
} }
// First, sort scope boundaries by their byte offset in the document. At a
impl < ' a > HighlightIter < ' a > {
// given position, emit scope endings before scope beginnings. Finally, emit
// scope boundaries from deeper layers first.
fn sort_key ( & mut self ) -> Option < ( usize , bool , isize ) > {
let depth = - ( self . depth as isize ) ;
let next_start = self
. captures
. peek ( )
. map ( | ( m , i ) | m . captures [ * i ] . node . start_byte ( ) ) ;
let next_end = self . highlight_end_stack . last ( ) . cloned ( ) ;
match ( next_start , next_end ) {
( Some ( start ) , Some ( end ) ) = > {
if start < end {
Some ( ( start , true , depth ) )
} else {
Some ( ( end , false , depth ) )
}
}
( Some ( i ) , None ) = > Some ( ( i , true , depth ) ) ,
( None , Some ( j ) ) = > Some ( ( j , false , depth ) ) ,
_ = > None ,
}
}
}
impl < ' a , F > HighlightIter < ' a , F >
where
F : FnMut ( & str ) -> Option < & ' a HighlightConfiguration > + ' a ,
{
fn emit_event ( fn emit_event (
& mut self , & mut self ,
offset : usize , offset : usize ,
@ -1361,6 +1319,12 @@ where
i + = 1 ; i + = 1 ;
continue ; continue ;
} }
} else {
let layer = self . layers . remove ( i + 1 ) ;
PARSER . with ( | ts_parser | {
let highlighter = & mut ts_parser . borrow_mut ( ) ;
highlighter . cursors . push ( layer . cursor ) ;
} ) ;
} }
break ; break ;
} }
@ -1377,30 +1341,9 @@ where
} }
} }
} }
fn insert_layer ( & mut self , mut layer : HighlightIterLayer < ' a > ) {
if let Some ( sort_key ) = layer . sort_key ( ) {
let mut i = 1 ;
while i < self . layers . len ( ) {
if let Some ( sort_key_i ) = self . layers [ i ] . sort_key ( ) {
if sort_key_i > sort_key {
self . layers . insert ( i , layer ) ;
return ;
}
i + = 1 ;
} else {
self . layers . remove ( i ) ;
}
}
self . layers . push ( layer ) ;
}
}
} }
impl < ' a , F > Iterator for HighlightIter < ' a , F > impl < ' a > Iterator for HighlightIter < ' a > {
where
F : FnMut ( & str ) -> Option < & ' a HighlightConfiguration > + ' a ,
{
type Item = Result < HighlightEvent , Error > ; type Item = Result < HighlightEvent , Error > ;
fn next ( & mut self ) -> Option < Self ::Item > { fn next ( & mut self ) -> Option < Self ::Item > {
@ -1460,55 +1403,12 @@ where
layer . highlight_end_stack . pop ( ) ; layer . highlight_end_stack . pop ( ) ;
return self . emit_event ( end_byte , Some ( HighlightEvent ::HighlightEnd ) ) ; return self . emit_event ( end_byte , Some ( HighlightEvent ::HighlightEnd ) ) ;
} else { } else {
// return self.emit_event(self.source.len(), None);
return self . emit_event ( self . source . len_bytes ( ) , None ) ;
return None ;
} ; } ;
let ( mut match_ , capture_index ) = layer . captures . next ( ) . unwrap ( ) ; let ( mut match_ , capture_index ) = layer . captures . next ( ) . unwrap ( ) ;
let mut capture = match_ . captures [ capture_index ] ; let mut capture = match_ . captures [ capture_index ] ;
// If this capture represents an injection, then process the injection.
if match_ . pattern_index < layer . config . locals_pattern_index {
let ( language_name , content_node , include_children ) =
injection_for_match ( layer . config , & layer . config . query , & match_ , self . source ) ;
// Explicitly remove this match so that none of its other captures will remain
// in the stream of captures.
match_ . remove ( ) ;
// If a language is found with the given name, then add a new language layer
// to the highlighted document.
if let ( Some ( language_name ) , Some ( content_node ) ) = ( language_name , content_node ) {
if let Some ( config ) = ( self . injection_callback ) ( & language_name ) {
let ranges = HighlightIterLayer ::intersect_ranges (
& self . layers [ 0 ] . ranges ,
& [ content_node ] ,
include_children ,
) ;
if ! ranges . is_empty ( ) {
match HighlightIterLayer ::new (
self . source ,
self . cancellation_flag ,
& mut self . injection_callback ,
config ,
self . layers [ 0 ] . depth + 1 ,
ranges ,
) {
Ok ( layers ) = > {
for layer in layers {
self . insert_layer ( layer ) ;
}
}
Err ( e ) = > return Some ( Err ( e ) ) ,
}
}
}
}
self . sort_layers ( ) ;
continue 'main ;
}
// Remove from the local scope stack any local scopes that have already ended.
// Remove from the local scope stack any local scopes that have already ended.
while range . start > layer . scope_stack . last ( ) . unwrap ( ) . range . end { while range . start > layer . scope_stack . last ( ) . unwrap ( ) . range . end {
layer . scope_stack . pop ( ) ; layer . scope_stack . pop ( ) ;
@ -1703,14 +1603,6 @@ fn injection_for_match<'a>(
( language_name , content_node , include_children ) ( language_name , content_node , include_children )
} }
// fn shrink_and_clear<T>(vec: &mut Vec<T>, capacity: usize) {
// if vec.len() > capacity {
// vec.truncate(capacity);
// vec.shrink_to_fit();
// }
// vec.clear();
// }
pub struct Merge < I > { pub struct Merge < I > {
iter : I , iter : I ,
spans : Box < dyn Iterator < Item = ( usize , std ::ops ::Range < usize > ) > > , spans : Box < dyn Iterator < Item = ( usize , std ::ops ::Range < usize > ) > > ,
@ -1877,6 +1769,8 @@ mod test {
. map ( String ::from ) . map ( String ::from )
. collect ( ) ; . collect ( ) ;
let loader = Loader ::new ( Configuration { language : vec ! [ ] } ) ;
let language = get_language ( & crate ::RUNTIME_DIR , "Rust" ) . unwrap ( ) ; let language = get_language ( & crate ::RUNTIME_DIR , "Rust" ) . unwrap ( ) ;
let config = HighlightConfiguration ::new ( let config = HighlightConfiguration ::new (
language , language ,
@ -1899,7 +1793,7 @@ mod test {
fn main ( ) { } fn main ( ) { }
" , " ,
) ; ) ;
let syntax = Syntax ::new ( & source , Arc ::new ( config ) ; let syntax = Syntax ::new ( & source , Arc ::new ( config ) , Arc ::new ( loader ) );
let tree = syntax . tree ( ) ; let tree = syntax . tree ( ) ;
let root = tree . root_node ( ) ; let root = tree . root_node ( ) ;
assert_eq! ( root . kind ( ) , "source_file" ) ; assert_eq! ( root . kind ( ) , "source_file" ) ;
@ -1926,7 +1820,7 @@ mod test {
& doc , & doc ,
vec! [ ( 6 , 11 , Some ( "test" . into ( ) ) ) , ( 12 , 17 , None ) ] . into_iter ( ) , vec! [ ( 6 , 11 , Some ( "test" . into ( ) ) ) , ( 12 , 17 , None ) ] . into_iter ( ) ,
) ; ) ;
let edits = LanguageLayer:: generate_edits( doc . slice ( .. ) , transaction . changes ( ) ) ; let edits = ( doc . slice ( .. ) , transaction . changes ( ) ) ;
// transaction.apply(&mut state);
// transaction.apply(&mut state);
assert_eq! ( assert_eq! (
@ -1955,7 +1849,7 @@ mod test {
let mut doc = Rope ::from ( "fn test() {}" ) ; let mut doc = Rope ::from ( "fn test() {}" ) ;
let transaction = let transaction =
Transaction ::change ( & doc , vec! [ ( 8 , 8 , Some ( "a: u32" . into ( ) ) ) ] . into_iter ( ) ) ; Transaction ::change ( & doc , vec! [ ( 8 , 8 , Some ( "a: u32" . into ( ) ) ) ] . into_iter ( ) ) ;
let edits = LanguageLayer:: generate_edits( doc . slice ( .. ) , transaction . changes ( ) ) ; let edits = ( doc . slice ( .. ) , transaction . changes ( ) ) ;
transaction . apply ( & mut doc ) ; transaction . apply ( & mut doc ) ;
assert_eq! ( doc , "fn test(a: u32) {}" ) ; assert_eq! ( doc , "fn test(a: u32) {}" ) ;
Blaž Hrastnik
3 years ago