Incremental parsing: rough draft.

.gitignore

@@ -1,2 +1,3 @@
 target
 helix-term/rustfmt.toml
+helix-syntax/languages/

Cargo.lock

@@ -369,6 +369,7 @@ name = "helix-core"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "helix-syntax",
  "ropey",
  "smallvec",
  "tendril",

Cargo.toml

@@ -6,5 +6,5 @@ members = [
 ]
 
 # Build helix-syntax in release mode to make the code path faster in development.
-[profile.dev.package."helix-syntax"]
+# [profile.dev.package."helix-syntax"]
-opt-level = 3
+# opt-level = 3

helix-core/Cargo.toml

@@ -7,6 +7,8 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+helix-syntax = { path = "../helix-syntax" }
+
 ropey = "1.2.0"
 anyhow = "1.0.31"
 smallvec = "1.4.0"

helix-core/src/lib.rs

@@ -1,6 +1,7 @@
 #![allow(unused)]
 pub mod commands;
 pub mod graphemes;
+mod position;
 mod selection;
 pub mod state;
 pub mod syntax;
@@ -9,8 +10,10 @@ mod transaction;
 pub use ropey::{Rope, RopeSlice};
 pub use tendril::StrTendril as Tendril;
 
+pub use position::Position;
 pub use selection::Range as SelectionRange;
 pub use selection::Selection;
+pub use syntax::Syntax;
 
 pub use state::State;
 

helix-core/src/position.rs

@@ -0,0 +1,57 @@
+/// Represents a single point in a text buffer. Zero indexed.
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Position {
+    pub row: usize,
+    pub col: usize,
+}
+
+impl Position {
+    pub const fn new(row: usize, col: usize) -> Self {
+        Self { row, col }
+    }
+
+    pub const fn is_zero(self) -> bool {
+        self.row == 0 && self.col == 0
+    }
+
+    // TODO: generalize
+    pub fn traverse(self, text: &crate::Tendril) -> Self {
+        let Self { mut row, mut col } = self;
+        // TODO: there should be a better way here
+        for ch in text.chars() {
+            if ch == '\n' {
+                row += 1;
+                col = 0;
+            } else {
+                col += 1;
+            }
+        }
+        Self { row, col }
+    }
+}
+
+impl From<(usize, usize)> for Position {
+    fn from(tuple: (usize, usize)) -> Self {
+        Position {
+            row: tuple.0,
+            col: tuple.1,
+        }
+    }
+}
+
+impl Into<tree_sitter::Point> for Position {
+    fn into(self) -> tree_sitter::Point {
+        tree_sitter::Point::new(self.row, self.col)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_ordering() {
+        // (0, 5) is less than (1, 0)
+        assert!(Position::new(0, 5) < Position::new(1, 0));
+    }
+}

helix-core/src/state.rs

@@ -1,5 +1,5 @@
 use crate::graphemes::{nth_next_grapheme_boundary, nth_prev_grapheme_boundary, RopeGraphemes};
-use crate::{Rope, RopeSlice, Selection, SelectionRange};
+use crate::{Position, Rope, RopeSlice, Selection, SelectionRange, Syntax};
 use anyhow::Error;
 
 use std::path::PathBuf;
@@ -17,6 +17,9 @@ pub struct State {
     pub(crate) doc: Rope,
     pub(crate) selection: Selection,
     pub(crate) mode: Mode,
+
+    //
+    pub syntax: Option<Syntax>,
 }
 
 #[derive(Copy, Clone, PartialEq, Eq)]
@@ -40,6 +43,7 @@ impl State {
             doc,
             selection: Selection::single(0, 0),
             mode: Mode::Normal,
+            syntax: None,
         }
     }
 
@@ -54,6 +58,29 @@ impl State {
         let mut state = Self::new(doc);
         state.path = Some(path);
 
+        let language = helix_syntax::get_language(&helix_syntax::LANG::Rust);
+
+        let mut highlight_config = crate::syntax::HighlightConfiguration::new(
+            language,
+            &std::fs::read_to_string(
+                "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
+            )
+            .unwrap(),
+            &std::fs::read_to_string(
+                "../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
+            )
+            .unwrap(),
+            "", // locals.scm
+        )
+        .unwrap();
+
+        // TODO: config.configure(scopes) is now delayed, is that ok?
+
+        // TODO: get_language is called twice
+        let syntax = Syntax::new(helix_syntax::LANG::Rust, &state.doc, highlight_config);
+
+        state.syntax = Some(syntax);
+
         Ok(state)
     }
 
@@ -175,29 +202,29 @@ impl State {
 }
 
 /// Coordinates are a 0-indexed line and column pair.
-type Coords = (usize, usize); // line, col
+pub type Coords = (usize, usize); // line, col
 
 /// Convert a character index to (line, column) coordinates.
-pub fn coords_at_pos(text: &RopeSlice, pos: usize) -> Coords {
+pub fn coords_at_pos(text: &RopeSlice, pos: usize) -> Position {
     let line = text.char_to_line(pos);
     let line_start = text.line_to_char(line);
     let col = text.slice(line_start..pos).len_chars();
-    (line, col)
+    Position::new(line, col)
 }
 
 /// Convert (line, column) coordinates to a character index.
-pub fn pos_at_coords(text: &RopeSlice, coords: Coords) -> usize {
+pub fn pos_at_coords(text: &RopeSlice, coords: Position) -> usize {
-    let (line, col) = coords;
+    let Position { row, col } = coords;
-    let line_start = text.line_to_char(line);
+    let line_start = text.line_to_char(row);
     nth_next_grapheme_boundary(text, line_start, col)
 }
 
 fn move_vertically(text: &RopeSlice, dir: Direction, pos: usize, count: usize) -> usize {
-    let (line, col) = coords_at_pos(text, pos);
+    let Position { row, col } = coords_at_pos(text, pos);
 
     let new_line = match dir {
-        Direction::Backward => line.saturating_sub(count),
+        Direction::Backward => row.saturating_sub(count),
-        Direction::Forward => std::cmp::min(line.saturating_add(count), text.len_lines() - 1),
+        Direction::Forward => std::cmp::min(row.saturating_add(count), text.len_lines() - 1),
     };
 
     // convert to 0-indexed, subtract another 1 because len_chars() counts \n
@@ -210,7 +237,7 @@ fn move_vertically(text: &RopeSlice, dir: Direction, pos: usize, count: usize) -
         col
     };
 
-    pos_at_coords(text, (new_line, new_col))
+    pos_at_coords(text, Position::new(new_line, new_col))
 }
 
 #[cfg(test)]
@@ -220,32 +247,32 @@ mod test {
     #[test]
     fn test_coords_at_pos() {
         let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
-        assert_eq!(coords_at_pos(&text.slice(..), 0), (0, 0));
+        assert_eq!(coords_at_pos(&text.slice(..), 0), (0, 0).into());
-        assert_eq!(coords_at_pos(&text.slice(..), 5), (0, 5)); // position on \n
+        assert_eq!(coords_at_pos(&text.slice(..), 5), (0, 5).into()); // position on \n
-        assert_eq!(coords_at_pos(&text.slice(..), 6), (1, 0)); // position on w
+        assert_eq!(coords_at_pos(&text.slice(..), 6), (1, 0).into()); // position on w
-        assert_eq!(coords_at_pos(&text.slice(..), 7), (1, 1)); // position on o
+        assert_eq!(coords_at_pos(&text.slice(..), 7), (1, 1).into()); // position on o
-        assert_eq!(coords_at_pos(&text.slice(..), 10), (1, 4)); // position on d
+        assert_eq!(coords_at_pos(&text.slice(..), 10), (1, 4).into()); // position on d
     }
 
     #[test]
     fn test_pos_at_coords() {
         let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
-        assert_eq!(pos_at_coords(&text.slice(..), (0, 0)), 0);
+        assert_eq!(pos_at_coords(&text.slice(..), (0, 0).into()), 0);
-        assert_eq!(pos_at_coords(&text.slice(..), (0, 5)), 5); // position on \n
+        assert_eq!(pos_at_coords(&text.slice(..), (0, 5).into()), 5); // position on \n
-        assert_eq!(pos_at_coords(&text.slice(..), (1, 0)), 6); // position on w
+        assert_eq!(pos_at_coords(&text.slice(..), (1, 0).into()), 6); // position on w
-        assert_eq!(pos_at_coords(&text.slice(..), (1, 1)), 7); // position on o
+        assert_eq!(pos_at_coords(&text.slice(..), (1, 1).into()), 7); // position on o
-        assert_eq!(pos_at_coords(&text.slice(..), (1, 4)), 10); // position on d
+        assert_eq!(pos_at_coords(&text.slice(..), (1, 4).into()), 10); // position on d
     }
 
     #[test]
     fn test_vertical_move() {
         let text = Rope::from("abcd\nefg\nwrs");
-        let pos = pos_at_coords(&text.slice(..), (0, 4));
+        let pos = pos_at_coords(&text.slice(..), (0, 4).into());
         let slice = text.slice(..);
 
         assert_eq!(
             coords_at_pos(&slice, move_vertically(&slice, Direction::Forward, pos, 1)),
-            (1, 2)
+            (1, 2).into()
         );
     }
 }

helix-core/src/syntax.rs

@@ -1,66 +1,397 @@
-// pub struct Syntax {
+use crate::{Change, Rope, RopeSlice, Transaction};
-//     parser: Parser,
+pub use helix_syntax::LANG;
-// }
+pub use helix_syntax::{get_language, get_language_name};
+
+pub struct Syntax {
+    grammar: Language,
+    parser: Parser,
+    cursors: Vec<QueryCursor>,
+
+    config: HighlightConfiguration,
 
-//impl Syntax {
+    root_layer: LanguageLayer,
-//    // buffer, grammar, config, grammars, sync_timeout?
+}
-//    pub fn new() -> Self {
+
-//        unimplemented!()
+impl Syntax {
-//        // make a new root layer
+    // buffer, grammar, config, grammars, sync_timeout?
-//        // track markers of injections
+    pub fn new(language: LANG, source: &Rope, config: HighlightConfiguration) -> Self {
-//        //
+        // fetch grammar for parser based on language string
-//        // track scope_descriptor: a Vec of scopes for item in tree
+        let grammar = get_language(&language);
-//        //
+        let parser = Parser::new();
-//        // fetch grammar for parser based on language string
+
-//        // update root layer
+        let root_layer = LanguageLayer::new();
-//    }
+
-
+        // track markers of injections
-//    // fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers
+        // track scope_descriptor: a Vec of scopes for item in tree
-
+
-//    // call this on transaction.apply() -> buffer_changed(changes)
+        let mut syntax = Self {
-//    //
+            grammar,
-//    // fn parse(language, old_tree, ranges)
+            parser,
-//    //
+            cursors: Vec::new(),
-//    // fn tree() -> Tree
+            config,
-//    //
+            root_layer,
-//    // <!--update_for_injection(grammar)-->
+        };
-
+
-//    // Highlighting
+        // update root layer
-//    // fn highlight_iter() -> iterates over all the scopes
+        syntax.root_layer.parse(
-//    // on_tokenize
+            &mut syntax.parser,
-//    // on_change_highlighting
+            &syntax.config,
-
+            source,
-//    // Commenting
+            0,
-//    // comment_strings_for_pos
+            vec![Range {
-//    // is_commented
+                start_byte: 0,
-
+                end_byte: usize::MAX,
-//    // Indentation
+                start_point: Point::new(0, 0),
-//    // suggested_indent_for_line_at_buffer_row
+                end_point: Point::new(usize::MAX, usize::MAX),
-//    // suggested_indent_for_buffer_row
+            }],
-//    // indent_level_for_line
+        );
-
+        syntax
-//    // TODO: Folding
+    }
-
+
-//    // Syntax APIs
+    pub fn configure(&mut self, scopes: &[String]) {
-//    // get_syntax_node_containing_range ->
+        self.config.configure(scopes)
-//    // ...
+    }
-//    // get_syntax_node_at_pos
+
-//    // buffer_range_for_scope_at_pos
+    pub fn update(&mut self, source: &Rope, changeset: &ChangeSet) -> Result<(), Error> {
-//}
+        self.root_layer
+            .update(&mut self.parser, &self.config, source, changeset)
+
+        // TODO: deal with injections and update them too
+    }
+
+    // fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers
+
+    // call this on transaction.apply() -> buffer_changed(changes)
+    //
+    // fn parse(language, old_tree, ranges)
+    //
+    fn tree(&self) -> &Tree {
+        self.root_layer.tree()
+    }
+    //
+    // <!--update_for_injection(grammar)-->
+
+    // Highlighting
+
+    /// Iterate over the highlighted regions for a given slice of source code.
+    pub fn highlight_iter<'a>(
+        &'a mut self,
+        source: &'a [u8],
+        cancellation_flag: Option<&'a AtomicUsize>,
+        mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
+    ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
+        // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
+        // prevents them from being moved. But both of these values are really just
+        // pointers, so it's actually ok to move them.
+
+        let mut cursor = QueryCursor::new(); // reuse a pool
+        let tree_ref = unsafe { mem::transmute::<_, &'static Tree>(self.tree()) };
+        let cursor_ref = unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
+        let query_ref = unsafe { mem::transmute::<_, &'static mut Query>(&mut self.config.query) };
+        let config_ref =
+            unsafe { mem::transmute::<_, &'static HighlightConfiguration>(&self.config) };
+        let captures = cursor_ref
+            .captures(query_ref, tree_ref.root_node(), move |n: Node| {
+                &source[n.byte_range()]
+            })
+            .peekable();
+
+        // manually craft the root layer based on the existing tree
+        let layer = HighlightIterLayer {
+            highlight_end_stack: Vec::new(),
+            scope_stack: vec![LocalScope {
+                inherits: false,
+                range: 0..usize::MAX,
+                local_defs: Vec::new(),
+            }],
+            cursor,
+            depth: 0,
+            _tree: None,
+            captures,
+            config: config_ref,
+            ranges: vec![Range {
+                start_byte: 0,
+                end_byte: usize::MAX,
+                start_point: Point::new(0, 0),
+                end_point: Point::new(usize::MAX, usize::MAX),
+            }],
+        };
+
+        let mut result = HighlightIter {
+            source,
+            byte_offset: 0,
+            injection_callback,
+            cancellation_flag,
+            highlighter: self,
+            iter_count: 0,
+            layers: vec![layer],
+            next_event: None,
+            last_highlight_range: None,
+        };
+        result.sort_layers();
+        Ok(result)
+    }
+    // on_tokenize
+    // on_change_highlighting
+
+    // Commenting
+    // comment_strings_for_pos
+    // is_commented
+
+    // Indentation
+    // suggested_indent_for_line_at_buffer_row
+    // suggested_indent_for_buffer_row
+    // indent_level_for_line
+
+    // TODO: Folding
+
+    // Syntax APIs
+    // get_syntax_node_containing_range ->
+    // ...
+    // get_syntax_node_at_pos
+    // buffer_range_for_scope_at_pos
+}
 
 pub struct LanguageLayer {
     // mode
-// grammar
+    // grammar
-// depth
+    // depth
-// tree: Tree,
+    tree: Option<Tree>,
 }
 
-// impl LanguageLayer {
+use crate::state::{coords_at_pos, Coords};
-//     // fn highlight_iter() -> same as Mode but for this layer. Mode composits these
+use crate::transaction::{ChangeSet, Operation};
-//     // fn buffer_changed
+use crate::Tendril;
-//     // fn update(range)
+
-//     // fn update_injections()
+impl LanguageLayer {
-// }
+    pub fn new() -> Self {
+        Self { tree: None }
+    }
+
+    fn tree(&self) -> &Tree {
+        // TODO: no unwrap
+        self.tree.as_ref().unwrap()
+    }
+
+    fn parse<'a>(
+        &mut self,
+        parser: &mut Parser,
+        config: &HighlightConfiguration,
+        source: &Rope,
+        mut depth: usize,
+        mut ranges: Vec<Range>,
+    ) -> Result<(), Error> {
+        if parser.set_included_ranges(&ranges).is_ok() {
+            parser
+                .set_language(config.language)
+                .map_err(|_| Error::InvalidLanguage)?;
+
+            // unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };
+            let tree = parser
+                .parse_with(
+                    &mut |byte, _| {
+                        if byte <= source.len_bytes() {
+                            let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);
+                            chunk[byte - start_byte..].as_bytes()
+                        } else {
+                            // out of range
+                            &[]
+                        }
+                    },
+                    self.tree.as_ref(),
+                )
+                .ok_or(Error::Cancelled)?;
+            // unsafe { syntax.parser.set_cancellation_flag(None) };
+            // let mut cursor = syntax.cursors.pop().unwrap_or_else(QueryCursor::new);
+
+            // Process combined injections. (ERB, EJS, etc https://github.com/tree-sitter/tree-sitter/pull/526)
+            // if let Some(combined_injections_query) = &config.combined_injections_query {
+            //     let mut injections_by_pattern_index =
+            //         vec![(None, Vec::new(), false); combined_injections_query.pattern_count()];
+            //     let matches =
+            //         cursor.matches(combined_injections_query, tree.root_node(), |n: Node| {
+            //             &source[n.byte_range()]
+            //         });
+            //     for mat in matches {
+            //         let entry = &mut injections_by_pattern_index[mat.pattern_index];
+            //         let (language_name, content_node, include_children) =
+            //             injection_for_match(config, combined_injections_query, &mat, source);
+            //         if language_name.is_some() {
+            //             entry.0 = language_name;
+            //         }
+            //         if let Some(content_node) = content_node {
+            //             entry.1.push(content_node);
+            //         }
+            //         entry.2 = include_children;
+            //     }
+            //     for (lang_name, content_nodes, includes_children) in injections_by_pattern_index {
+            //         if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {
+            //             if let Some(next_config) = (injection_callback)(lang_name) {
+            //                 let ranges =
+            //                     Self::intersect_ranges(&ranges, &content_nodes, includes_children);
+            //                 if !ranges.is_empty() {
+            //                     queue.push((next_config, depth + 1, ranges));
+            //                 }
+            //             }
+            //         }
+            //     }
+            // }
+            self.tree = Some(tree)
+        }
+        Ok(())
+    }
+
+    pub(crate) fn generate_edits(
+        text: &RopeSlice,
+        changeset: &ChangeSet,
+    ) -> Vec<tree_sitter::InputEdit> {
+        use Operation::*;
+        let mut old_pos = 0;
+        let mut new_pos = 0;
+
+        let mut edits = Vec::new();
+
+        let mut iter = changeset.changes.iter().peekable();
+
+        // TODO; this is a lot easier with Change instead of Operation.
+
+        fn point_at_pos(text: &RopeSlice, pos: usize) -> (usize, Point) {
+            let byte = text.char_to_byte(pos);
+            let line = text.char_to_line(pos);
+            let line_start_byte = text.line_to_byte(line);
+            let col = byte - line_start_byte;
+
+            (byte, Point::new(line, col))
+        }
+
+        fn traverse(point: Point, text: &Tendril) -> Point {
+            let Point {
+                mut row,
+                mut column,
+            } = point;
+
+            // TODO: there should be a better way here
+            for ch in text.bytes() {
+                if ch == b'\n' {
+                    row += 1;
+                    column = 0;
+                } else {
+                    column += 1;
+                }
+            }
+            Point { row, column }
+        }
+
+        while let Some(change) = iter.next() {
+            let len = match change {
+                Delete(i) | Retain(i) => *i,
+                Insert(_) => 0,
+            };
+            let old_end = old_pos + len;
+
+            match change {
+                Retain(_) => {
+                    new_pos += len;
+                }
+                Delete(_) => {
+                    let (start_byte, start_position) = point_at_pos(&text, old_pos);
+                    let (old_end_byte, old_end_position) = point_at_pos(&text, old_end);
+
+                    // TODO: Position also needs to be byte based...
+                    // let byte = char_to_byte(old_pos)
+                    // let line = char_to_line(old_pos)
+                    // let line_start_byte = line_to_byte()
+                    // Position::new(line, line_start_byte - byte)
+
+                    // a subsequent ins means a replace, consume it
+                    if let Some(Insert(s)) = iter.peek() {
+                        iter.next();
+                        let ins = s.chars().count();
+
+                        // replacement
+                        edits.push(tree_sitter::InputEdit {
+                            start_byte,                                    // old_pos to byte
+                            old_end_byte,                                  // old_end to byte
+                            new_end_byte: start_byte + s.len(), // old_pos to byte + s.len()
+                            start_position,                     // old pos to coords
+                            old_end_position,                   // old_end to coords
+                            new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
+                        });
+
+                        new_pos += ins;
+                    } else {
+                        // deletion
+                        edits.push(tree_sitter::InputEdit {
+                            start_byte,                       // old_pos to byte
+                            old_end_byte,                     // old_end to byte
+                            new_end_byte: start_byte,         // old_pos to byte
+                            start_position,                   // old pos to coords
+                            old_end_position,                 // old_end to coords
+                            new_end_position: start_position, // old pos to coords
+                        });
+                    };
+                }
+                Insert(s) => {
+                    let (start_byte, start_position) = point_at_pos(&text, old_pos);
+
+                    let ins = s.chars().count();
+
+                    // insert
+                    edits.push(tree_sitter::InputEdit {
+                        start_byte,                                    // old_pos to byte
+                        old_end_byte: start_byte,                      // same
+                        new_end_byte: start_byte + s.len(),            // old_pos + s.len()
+                        start_position,                                // old pos to coords
+                        old_end_position: start_position,              // same
+                        new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
+                    });
+
+                    new_pos += ins;
+                }
+            }
+            old_pos = old_end;
+        }
+        edits
+    }
+
+    fn update(
+        &mut self,
+        parser: &mut Parser,
+        config: &HighlightConfiguration,
+        source: &Rope,
+        changeset: &ChangeSet,
+    ) -> Result<(), Error> {
+        if changeset.is_empty() {
+            return Ok(());
+        }
+
+        let edits = Self::generate_edits(&source.slice(..), changeset);
+
+        // Notify the tree about all the changes
+        for edit in edits {
+            self.tree.as_mut().unwrap().edit(&edit);
+        }
+
+        self.parse(
+            parser,
+            config,
+            source,
+            0,
+            // TODO: what to do about this range on update
+            vec![Range {
+                start_byte: 0,
+                end_byte: usize::MAX,
+                start_point: Point::new(0, 0),
+                end_point: Point::new(usize::MAX, usize::MAX),
+            }],
+        )
+    }
+
+    // fn highlight_iter() -> same as Mode but for this layer. Mode composits these
+    // fn buffer_changed
+    // fn update(range)
+    // fn update_injections()
+}
 
 // -- refactored from tree-sitter-highlight to be able to retain state
 // TODO: add seek() to iter
@@ -169,7 +500,7 @@ where
 {
     source: &'a [u8],
     byte_offset: usize,
-    highlighter: &'a mut Highlighter,
+    highlighter: &'a mut Syntax,
     injection_callback: F,
     cancellation_flag: Option<&'a AtomicUsize>,
     layers: Vec<HighlightIterLayer<'a>>,
@@ -179,7 +510,7 @@ where
 }
 
 struct HighlightIterLayer<'a> {
-    _tree: Tree,
+    _tree: Option<Tree>,
     cursor: QueryCursor,
     captures: iter::Peekable<QueryCaptures<'a, &'a [u8]>>,
     config: &'a HighlightConfiguration,
@@ -207,43 +538,43 @@ impl Highlighter {
         &mut self.parser
     }
 
-    /// Iterate over the highlighted regions for a given slice of source code.
+    // /// Iterate over the highlighted regions for a given slice of source code.
-    pub fn highlight<'a>(
+    // pub fn highlight<'a>(
-        &'a mut self,
+    //     &'a mut self,
-        config: &'a HighlightConfiguration,
+    //     config: &'a HighlightConfiguration,
-        source: &'a [u8],
+    //     source: &'a [u8],
-        cancellation_flag: Option<&'a AtomicUsize>,
+    //     cancellation_flag: Option<&'a AtomicUsize>,
-        mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
+    //     mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
-    ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
+    // ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
-        let layers = HighlightIterLayer::new(
+    //     let layers = HighlightIterLayer::new(
-            source,
+    //         source,
-            self,
+    //         self,
-            cancellation_flag,
+    //         cancellation_flag,
-            &mut injection_callback,
+    //         &mut injection_callback,
-            config,
+    //         config,
-            0,
+    //         0,
-            vec![Range {
+    //         vec![Range {
-                start_byte: 0,
+    //             start_byte: 0,
-                end_byte: usize::MAX,
+    //             end_byte: usize::MAX,
-                start_point: Point::new(0, 0),
+    //             start_point: Point::new(0, 0),
-                end_point: Point::new(usize::MAX, usize::MAX),
+    //             end_point: Point::new(usize::MAX, usize::MAX),
-            }],
+    //         }],
-        )?;
+    //     )?;
-        assert_ne!(layers.len(), 0);
+    //     assert_ne!(layers.len(), 0);
-        let mut result = HighlightIter {
+    //     let mut result = HighlightIter {
-            source,
+    //         source,
-            byte_offset: 0,
+    //         byte_offset: 0,
-            injection_callback,
+    //         injection_callback,
-            cancellation_flag,
+    //         cancellation_flag,
-            highlighter: self,
+    //         highlighter: self,
-            iter_count: 0,
+    //         iter_count: 0,
-            layers,
+    //         layers,
-            next_event: None,
+    //         next_event: None,
-            last_highlight_range: None,
+    //         last_highlight_range: None,
-        };
+    //     };
-        result.sort_layers();
+    //     result.sort_layers();
-        Ok(result)
+    //     Ok(result)
-    }
+    // }
 }
 
 impl HighlightConfiguration {
@@ -413,7 +744,7 @@ impl<'a> HighlightIterLayer<'a> {
     /// added to the returned vector.
     fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(
         source: &'a [u8],
-        highlighter: &mut Highlighter,
+        highlighter: &mut Syntax,
         cancellation_flag: Option<&'a AtomicUsize>,
         injection_callback: &mut F,
         mut config: &'a HighlightConfiguration,
@@ -423,6 +754,8 @@ impl<'a> HighlightIterLayer<'a> {
         let mut result = Vec::with_capacity(1);
         let mut queue = Vec::new();
         loop {
+            // --> Tree parsing part
+
             if highlighter.parser.set_included_ranges(&ranges).is_ok() {
                 highlighter
                     .parser
@@ -474,6 +807,8 @@ impl<'a> HighlightIterLayer<'a> {
                     }
                 }
 
+                // --> Highlighting query part
+
                 // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
                 // prevents them from being moved. But both of these values are really just
                 // pointers, so it's actually ok to move them.
@@ -495,7 +830,7 @@ impl<'a> HighlightIterLayer<'a> {
                     }],
                     cursor,
                     depth,
-                    _tree: tree,
+                    _tree: Some(tree),
                     captures,
                     config,
                     ranges,
@@ -1016,3 +1351,106 @@ fn shrink_and_clear<T>(vec: &mut Vec<T>, capacity: usize) {
     }
     vec.clear();
 }
+
+#[test]
+fn test_parser() {
+    let highlight_names: Vec<String> = [
+        "attribute",
+        "constant",
+        "function.builtin",
+        "function",
+        "keyword",
+        "operator",
+        "property",
+        "punctuation",
+        "punctuation.bracket",
+        "punctuation.delimiter",
+        "string",
+        "string.special",
+        "tag",
+        "type",
+        "type.builtin",
+        "variable",
+        "variable.builtin",
+        "variable.parameter",
+    ]
+    .iter()
+    .cloned()
+    .map(String::from)
+    .collect();
+
+    let language = get_language(&LANG::Rust);
+    let mut config = HighlightConfiguration::new(
+        language,
+        &std::fs::read_to_string(
+            "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
+        )
+        .unwrap(),
+        &std::fs::read_to_string(
+            "../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
+        )
+        .unwrap(),
+        "", // locals.scm
+    )
+    .unwrap();
+    config.configure(&highlight_names);
+
+    let source = Rope::from_str(
+        "
+        struct Stuff {}
+        fn main() {}
+    ",
+    );
+    let syntax = Syntax::new(LANG::Rust, &source, config);
+    let tree = syntax.root_layer.tree.unwrap();
+    let root = tree.root_node();
+    assert_eq!(root.kind(), "source_file");
+
+    assert_eq!(
+        root.to_sexp(),
+        concat!(
+            "(source_file ",
+            "(struct_item name: (type_identifier) body: (field_declaration_list)) ",
+            "(function_item name: (identifier) parameters: (parameters) body: (block)))"
+        )
+    );
+
+    let struct_node = root.child(0).unwrap();
+    assert_eq!(struct_node.kind(), "struct_item");
+}
+
+#[test]
+fn test_input_edits() {
+    use crate::State;
+    use tree_sitter::InputEdit;
+
+    let mut state = State::new("hello world!\ntest 123".into());
+    let transaction = Transaction::change(
+        &state,
+        vec![(6, 11, Some("test".into())), (12, 17, None)].into_iter(),
+    );
+    let edits = LanguageLayer::generate_edits(&state.doc.slice(..), &transaction.changes);
+    // transaction.apply(&mut state);
+
+    assert_eq!(
+        edits,
+        &[
+            InputEdit {
+                start_byte: 6,
+                old_end_byte: 11,
+                new_end_byte: 10,
+                start_position: Point { row: 0, column: 6 },
+                old_end_position: Point { row: 0, column: 11 },
+                new_end_position: Point { row: 0, column: 10 }
+            },
+            InputEdit {
+                start_byte: 12,
+                old_end_byte: 17,
+                new_end_byte: 12,
+                start_position: Point { row: 0, column: 12 },
+                old_end_position: Point { row: 1, column: 4 },
+                new_end_position: Point { row: 0, column: 12 }
+            }
+        ]
+    );
+}

helix-core/src/transaction.rs

@@ -4,7 +4,7 @@ use crate::{Rope, Selection, SelectionRange, State, Tendril};
 pub type Change = (usize, usize, Option<Tendril>);
 
 #[derive(Debug, Clone, PartialEq, Eq)]
-enum Operation {
+pub(crate) enum Operation {
     /// Move cursor by n characters.
     Retain(usize),
     /// Delete n characters.
@@ -22,7 +22,7 @@ pub enum Assoc {
 // ChangeSpec = Change | ChangeSet | Vec<Change>
 #[derive(Debug)]
 pub struct ChangeSet {
-    changes: Vec<Operation>,
+    pub(crate) changes: Vec<Operation>,
     /// The required document length. Will refuse to apply changes unless it matches.
     len: usize,
 }
@@ -307,7 +307,7 @@ impl ChangeSet {
 /// a single transaction.
 pub struct Transaction {
     /// Changes made to the buffer.
-    changes: ChangeSet,
+    pub(crate) changes: ChangeSet,
     /// When set, explicitly updates the selection.
     selection: Option<Selection>,
     // effects, annotations
@@ -337,6 +337,14 @@ impl Transaction {
             .clone()
             .unwrap_or_else(|| state.selection.clone().map(&self.changes));
 
+        // TODO: no unwrap
+        state
+            .syntax
+            .as_mut()
+            .unwrap()
+            .update(&state.doc, &self.changes)
+            .unwrap();
+
         true
     }
 
@@ -359,9 +367,15 @@ impl Transaction {
         for (from, to, tendril) in changes {
             // Retain from last "to" to current "from"
             acc.push(Operation::Retain(from - last));
+            let span = to - from;
             match tendril {
-                Some(text) => acc.push(Operation::Insert(text)),
+                Some(text) => {
-                None => acc.push(Operation::Delete(to - from)),
+                    if span > 0 {
+                        acc.push(Operation::Delete(span));
+                    }
+                    acc.push(Operation::Insert(text));
+                }
+                None => acc.push(Operation::Delete(span)),
             }
             last = to;
         }
@@ -466,4 +480,15 @@ mod test {
         assert_eq!(cs.map_pos(2, Assoc::Before), 2);
         assert_eq!(cs.map_pos(2, Assoc::After), 2);
     }
+
+    #[test]
+    fn transaction_change() {
+        let mut state = State::new("hello world!\ntest 123".into());
+        let transaction = Transaction::change(
+            &state,
+            vec![(6, 11, Some("void".into())), (12, 17, None)].into_iter(),
+        );
+        transaction.apply(&mut state);
+        assert_eq!(state.doc, Rope::from_str("hello void! 123"));
+    }
 }

helix-term/Cargo.toml

@@ -11,11 +11,11 @@ name = "hx"
 path = "src/main.rs"
 
 [dependencies]
+helix-core = { path = "../helix-core" }
 helix-syntax = { path = "../helix-syntax" }
 
 anyhow = "1"
 argh = "0.1.3"
-helix-core = { path = "../helix-core" }
 crossterm = { version = "0.17", features = ["event-stream"] }
 
 smol = "1"

helix-term/src/editor.rs

@@ -38,8 +38,6 @@ pub struct Editor {
     size: (u16, u16),
     surface: Surface,
     theme: Theme,
-    highlighter: Highlighter,
-    highlight_config: HighlightConfiguration,
 }
 
 impl Editor {
@@ -49,33 +47,8 @@ impl Editor {
         let mut terminal = Terminal::new(backend)?;
         let size = terminal::size().unwrap();
         let area = Rect::new(0, 0, size.0, size.1);
-
         let theme = Theme::default();
 
-        // let mut parser = tree_sitter::Parser::new();
-        // parser.set_language(language).unwrap();
-        // let tree = parser.parse(source_code, None).unwrap();
-
-        let language = helix_syntax::get_language(&helix_syntax::LANG::Rust);
-
-        let highlighter = Highlighter::new();
-
-        let mut highlight_config = HighlightConfiguration::new(
-            language,
-            &std::fs::read_to_string(
-                "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
-            )
-            .unwrap(),
-            &std::fs::read_to_string(
-                "../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
-            )
-            .unwrap(),
-            "", // locals.scm
-        )
-        .unwrap();
-
-        highlight_config.configure(theme.scopes());
-
         let mut editor = Editor {
             terminal,
             state: None,
@@ -84,8 +57,6 @@ impl Editor {
             surface: Surface::empty(area),
             theme,
             // TODO; move to state
-            highlighter,
-            highlight_config,
         };
 
         if let Some(file) = args.files.pop() {
@@ -96,12 +67,19 @@ impl Editor {
     }
 
     pub fn open(&mut self, path: PathBuf) -> Result<(), Error> {
-        self.state = Some(State::load(path)?);
+        let mut state = State::load(path)?;
+        state
+            .syntax
+            .as_mut()
+            .unwrap()
+            .configure(self.theme.scopes());
+        self.state = Some(state);
         Ok(())
     }
 
     fn render(&mut self) {
-        match &self.state {
+        // TODO: ideally not mut but highlights require it because of cursor cache
+        match &mut self.state {
             Some(state) => {
                 let area = Rect::new(0, 0, self.size.0, self.size.1);
                 let mut surface = Surface::empty(area);
@@ -112,12 +90,13 @@ impl Editor {
 
                 // TODO: cache highlight results
                 // TODO: only recalculate when state.doc is actually modified
-                let highlights = self
+                let highlights: Vec<_> = state
-                    .highlighter
+                    .syntax
-                    .highlight(&self.highlight_config, source_code.as_bytes(), None, |_| {
+                    .as_mut()
-                        None
+                    .unwrap()
-                    })
+                    .highlight_iter(source_code.as_bytes(), None, |_| None)
-                    .unwrap();
+                    .unwrap()
+                    .collect(); // TODO: we collect here to avoid double borrow, fix later
 
                 let mut spans = Vec::new();
 
@@ -235,7 +214,7 @@ impl Editor {
                 let coords = coords_at_pos(&state.doc().slice(..), pos);
                 execute!(
                     stdout,
-                    cursor::MoveTo((coords.1 + 2) as u16, coords.0 as u16)
+                    cursor::MoveTo((coords.col + 2) as u16, coords.row as u16)
                 );
             }
             None => (),