track char_idx in DocFormatter

helix-core/src/doc_formatter.rs

@@ -37,52 +37,91 @@ pub enum GraphemeSource {
     },
 }
 
+impl GraphemeSource {
+    /// Returns whether this grapheme is virtual inline text
+    pub fn is_virtual(self) -> bool {
+        matches!(self, GraphemeSource::VirtualText { .. })
+    }
+
+    pub fn doc_chars(self) -> usize {
+        match self {
+            GraphemeSource::Document { codepoints } => codepoints as usize,
+            GraphemeSource::VirtualText { .. } => 0,
+        }
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct FormattedGrapheme<'a> {
-    pub grapheme: Grapheme<'a>,
+    pub raw: Grapheme<'a>,
     pub source: GraphemeSource,
+    pub visual_pos: Position,
+    /// Document line at the start of the grapheme
+    pub line_idx: usize,
+    /// Document char position at the start of the grapheme
+    pub char_idx: usize,
 }
 
-impl<'a> FormattedGrapheme<'a> {
-    pub fn new(
+impl FormattedGrapheme<'_> {
+    pub fn is_virtual(&self) -> bool {
+        self.source.is_virtual()
+    }
+
+    pub fn doc_chars(&self) -> usize {
+        self.source.doc_chars()
+    }
+
+    pub fn is_whitespace(&self) -> bool {
+        self.raw.is_whitespace()
+    }
+
+    pub fn width(&self) -> usize {
+        self.raw.width()
+    }
+
+    pub fn is_word_boundary(&self) -> bool {
+        self.raw.is_word_boundary()
+    }
+}
+
+#[derive(Debug, Clone)]
+struct GraphemeWithSource<'a> {
+    grapheme: Grapheme<'a>,
+    source: GraphemeSource,
+}
+
+impl<'a> GraphemeWithSource<'a> {
+    fn new(
         g: GraphemeStr<'a>,
         visual_x: usize,
         tab_width: u16,
         source: GraphemeSource,
-    ) -> FormattedGrapheme<'a> {
-        FormattedGrapheme {
+    ) -> GraphemeWithSource<'a> {
+        GraphemeWithSource {
             grapheme: Grapheme::new(g, visual_x, tab_width),
             source,
         }
     }
-    /// Returns whether this grapheme is virtual inline text
-    pub fn is_virtual(&self) -> bool {
-        matches!(self.source, GraphemeSource::VirtualText { .. })
-    }
-
-    pub fn placeholder() -> Self {
-        FormattedGrapheme {
+    fn placeholder() -> Self {
+        GraphemeWithSource {
             grapheme: Grapheme::Other { g: " ".into() },
             source: GraphemeSource::Document { codepoints: 0 },
         }
     }
 
-    pub fn doc_chars(&self) -> usize {
-        match self.source {
-            GraphemeSource::Document { codepoints } => codepoints as usize,
-            GraphemeSource::VirtualText { .. } => 0,
-        }
+    fn doc_chars(&self) -> usize {
+        self.source.doc_chars()
     }
 
-    pub fn is_whitespace(&self) -> bool {
+    fn is_whitespace(&self) -> bool {
         self.grapheme.is_whitespace()
     }
 
-    pub fn width(&self) -> usize {
+    fn width(&self) -> usize {
         self.grapheme.width()
     }
 
-    pub fn is_word_boundary(&self) -> bool {
+    fn is_word_boundary(&self) -> bool {
         self.grapheme.is_word_boundary()
     }
 }
@@ -139,9 +178,9 @@ pub struct DocumentFormatter<'t> {
     indent_level: Option<usize>,
     /// In case a long word needs to be split a single grapheme might need to be wrapped
     /// while the rest of the word stays on the same line
-    peeked_grapheme: Option<(FormattedGrapheme<'t>, usize)>,
+    peeked_grapheme: Option<GraphemeWithSource<'t>>,
     /// A first-in first-out (fifo) buffer for the Graphemes of any given word
-    word_buf: Vec<FormattedGrapheme<'t>>,
+    word_buf: Vec<GraphemeWithSource<'t>>,
     /// The index of the next grapheme that will be yielded from the `word_buf`
     word_i: usize,
 }
@@ -157,32 +196,33 @@ impl<'t> DocumentFormatter<'t> {
         text_fmt: &'t TextFormat,
         annotations: &'t TextAnnotations,
         char_idx: usize,
-    ) -> (Self, usize) {
+    ) -> Self {
         // TODO divide long lines into blocks to avoid bad performance for long lines
         let block_line_idx = text.char_to_line(char_idx.min(text.len_chars()));
         let block_char_idx = text.line_to_char(block_line_idx);
         annotations.reset_pos(block_char_idx);
-        (
-            DocumentFormatter {
-                text_fmt,
-                annotations,
-                visual_pos: Position { row: 0, col: 0 },
-                graphemes: RopeGraphemes::new(text.slice(block_char_idx..)),
-                char_pos: block_char_idx,
-                exhausted: false,
-                virtual_lines: 0,
-                indent_level: None,
-                peeked_grapheme: None,
-                word_buf: Vec::with_capacity(64),
-                word_i: 0,
-                line_pos: block_line_idx,
-                inline_anntoation_graphemes: None,
-            },
-            block_char_idx,
-        )
+
+        DocumentFormatter {
+            text_fmt,
+            annotations,
+            visual_pos: Position { row: 0, col: 0 },
+            graphemes: RopeGraphemes::new(text.slice(block_char_idx..)),
+            char_pos: block_char_idx,
+            exhausted: false,
+            virtual_lines: 0,
+            indent_level: None,
+            peeked_grapheme: None,
+            word_buf: Vec::with_capacity(64),
+            word_i: 0,
+            line_pos: block_line_idx,
+            inline_anntoation_graphemes: None,
+        }
     }
 
-    fn next_inline_annotation_grapheme(&mut self) -> Option<(&'t str, Option<Highlight>)> {
+    fn next_inline_annotation_grapheme(
+        &mut self,
+        char_pos: usize,
+    ) -> Option<(&'t str, Option<Highlight>)> {
         loop {
             if let Some(&mut (ref mut annotation, highlight)) =
                 self.inline_anntoation_graphemes.as_mut()
@@ -193,7 +233,7 @@ impl<'t> DocumentFormatter<'t> {
             }
 
             if let Some((annotation, highlight)) =
-                self.annotations.next_inline_annotation_at(self.char_pos)
+                self.annotations.next_inline_annotation_at(char_pos)
             {
                 self.inline_anntoation_graphemes = Some((
                     UnicodeSegmentation::graphemes(&*annotation.text, true),
@@ -205,21 +245,20 @@ impl<'t> DocumentFormatter<'t> {
         }
     }
 
-    fn advance_grapheme(&mut self, col: usize) -> Option<FormattedGrapheme<'t>> {
+    fn advance_grapheme(&mut self, col: usize, char_pos: usize) -> Option<GraphemeWithSource<'t>> {
         let (grapheme, source) =
-            if let Some((grapheme, highlight)) = self.next_inline_annotation_grapheme() {
+            if let Some((grapheme, highlight)) = self.next_inline_annotation_grapheme(char_pos) {
                 (grapheme.into(), GraphemeSource::VirtualText { highlight })
             } else if let Some(grapheme) = self.graphemes.next() {
                 self.virtual_lines += self.annotations.annotation_lines_at(self.char_pos);
                 let codepoints = grapheme.len_chars() as u32;
 
-                let overlay = self.annotations.overlay_at(self.char_pos);
+                let overlay = self.annotations.overlay_at(char_pos);
                 let grapheme = match overlay {
                     Some((overlay, _)) => overlay.grapheme.as_str().into(),
                     None => Cow::from(grapheme).into(),
                 };
 
-                self.char_pos += codepoints as usize;
                 (grapheme, GraphemeSource::Document { codepoints })
             } else {
                 if self.exhausted {
@@ -228,19 +267,19 @@ impl<'t> DocumentFormatter<'t> {
                 self.exhausted = true;
                 // EOF grapheme is required for rendering
                 // and correct position computations
-                return Some(FormattedGrapheme {
+                return Some(GraphemeWithSource {
                     grapheme: Grapheme::Other { g: " ".into() },
                     source: GraphemeSource::Document { codepoints: 0 },
                 });
             };
 
-        let grapheme = FormattedGrapheme::new(grapheme, col, self.text_fmt.tab_width, source);
+        let grapheme = GraphemeWithSource::new(grapheme, col, self.text_fmt.tab_width, source);
 
         Some(grapheme)
     }
 
     /// Move a word to the next visual line
-    fn wrap_word(&mut self, virtual_lines_before_word: usize) -> usize {
+    fn wrap_word(&mut self) -> usize {
         // softwrap this word to the next line
         let indent_carry_over = if let Some(indent) = self.indent_level {
             if indent as u16 <= self.text_fmt.max_indent_retain {
@@ -255,14 +294,13 @@ impl<'t> DocumentFormatter<'t> {
         };
 
         self.visual_pos.col = indent_carry_over as usize;
-        self.virtual_lines -= virtual_lines_before_word;
-        self.visual_pos.row += 1 + virtual_lines_before_word;
+        self.visual_pos.row += 1 + take(&mut self.virtual_lines);
         let mut i = 0;
         let mut word_width = 0;
         let wrap_indicator = UnicodeSegmentation::graphemes(&*self.text_fmt.wrap_indicator, true)
             .map(|g| {
                 i += 1;
-                let grapheme = FormattedGrapheme::new(
+                let grapheme = GraphemeWithSource::new(
                     g.into(),
                     self.visual_pos.col + word_width,
                     self.text_fmt.tab_width,
@@ -288,8 +326,7 @@ impl<'t> DocumentFormatter<'t> {
     fn advance_to_next_word(&mut self) {
         self.word_buf.clear();
         let mut word_width = 0;
-        let virtual_lines_before_word = self.virtual_lines;
-        let mut virtual_lines_before_grapheme = self.virtual_lines;
+        let mut word_chars = 0;
 
         loop {
             // softwrap word if necessary
@@ -301,27 +338,24 @@ impl<'t> DocumentFormatter<'t> {
                     // However if the last grapheme is multiple columns wide it might extend beyond the EOL.
                     // The condition below ensures that this grapheme is not cutoff and instead wrapped to the next line
                     if word_width + self.visual_pos.col > self.text_fmt.viewport_width as usize {
-                        self.peeked_grapheme = self.word_buf.pop().map(|grapheme| {
-                            (grapheme, self.virtual_lines - virtual_lines_before_grapheme)
-                        });
-                        self.virtual_lines = virtual_lines_before_grapheme;
+                        self.peeked_grapheme = self.word_buf.pop();
                     }
                     return;
                 }
 
-                word_width = self.wrap_word(virtual_lines_before_word);
+                word_width = self.wrap_word();
             }
 
-            virtual_lines_before_grapheme = self.virtual_lines;
-
-            let grapheme = if let Some((grapheme, virtual_lines)) = self.peeked_grapheme.take() {
-                self.virtual_lines += virtual_lines;
+            let grapheme = if let Some(grapheme) = self.peeked_grapheme.take() {
                 grapheme
-            } else if let Some(grapheme) = self.advance_grapheme(self.visual_pos.col + word_width) {
+            } else if let Some(grapheme) =
+                self.advance_grapheme(self.visual_pos.col + word_width, self.char_pos + word_chars)
+            {
                 grapheme
             } else {
                 return;
             };
+            word_chars += grapheme.doc_chars();
 
             // Track indentation
             if !grapheme.is_whitespace() && self.indent_level.is_none() {
@@ -340,19 +374,19 @@ impl<'t> DocumentFormatter<'t> {
         }
     }
 
-    /// returns the document line pos of the **next** grapheme that will be yielded
-    pub fn line_pos(&self) -> usize {
-        self.line_pos
+    /// returns the char index at the end of the last yielded grapheme
+    pub fn next_char_pos(&self) -> usize {
+        self.char_pos
     }
 
-    /// returns the visual pos of the **next** grapheme that will be yielded
-    pub fn visual_pos(&self) -> Position {
+    /// returns the visual position at the end of the last yielded grapheme
+    pub fn next_visual_pos(&self) -> Position {
         self.visual_pos
     }
 }
 
 impl<'t> Iterator for DocumentFormatter<'t> {
-    type Item = (FormattedGrapheme<'t>, Position);
+    type Item = FormattedGrapheme<'t>;
 
     fn next(&mut self) -> Option<Self::Item> {
         let grapheme = if self.text_fmt.soft_wrap {
@@ -362,15 +396,18 @@ impl<'t> Iterator for DocumentFormatter<'t> {
             }
             let grapheme = replace(
                 self.word_buf.get_mut(self.word_i)?,
-                FormattedGrapheme::placeholder(),
+                GraphemeWithSource::placeholder(),
             );
             self.word_i += 1;
             grapheme
         } else {
-            self.advance_grapheme(self.visual_pos.col)?
+            self.advance_grapheme(self.visual_pos.col, self.char_pos)?
         };
 
-        let pos = self.visual_pos;
+        let visual_pos = self.visual_pos;
+        let char_pos = self.char_pos;
+        self.char_pos += grapheme.doc_chars();
+        let line_idx = self.line_pos;
         if grapheme.grapheme == Grapheme::Newline {
             self.visual_pos.row += 1;
             self.visual_pos.row += take(&mut self.virtual_lines);
@@ -379,6 +416,12 @@ impl<'t> Iterator for DocumentFormatter<'t> {
         } else {
             self.visual_pos.col += grapheme.width();
         }
-        Some((grapheme, pos))
+        Some(FormattedGrapheme {
+            raw: grapheme.grapheme,
+            source: grapheme.source,
+            visual_pos,
+            line_idx,
+            char_idx: char_pos,
+        })
     }
 }

helix-core/src/doc_formatter/test.rs

@@ -23,18 +23,18 @@ impl<'t> DocumentFormatter<'t> {
         let viewport_width = self.text_fmt.viewport_width;
         let mut line = 0;
 
-        for (grapheme, pos) in self {
-            if pos.row != line {
+        for grapheme in self {
+            if grapheme.visual_pos.row != line {
                 line += 1;
-                assert_eq!(pos.row, line);
-                write!(res, "\n{}", ".".repeat(pos.col)).unwrap();
+                assert_eq!(grapheme.visual_pos.row, line);
+                write!(res, "\n{}", ".".repeat(grapheme.visual_pos.col)).unwrap();
                 assert!(
-                    pos.col <= viewport_width as usize,
+                    grapheme.visual_pos.col <= viewport_width as usize,
                     "softwrapped failed {}<={viewport_width}",
-                    pos.col
+                    grapheme.visual_pos.col
                 );
             }
-            write!(res, "{}", grapheme.grapheme).unwrap();
+            write!(res, "{}", grapheme.raw).unwrap();
         }
 
         res
@@ -48,7 +48,6 @@ fn softwrap_text(text: &str) -> String {
         &TextAnnotations::default(),
         0,
     )
-    .0
     .collect_to_str()
 }
 
@@ -106,7 +105,6 @@ fn overlay_text(text: &str, char_pos: usize, softwrap: bool, overlays: &[Overlay
         TextAnnotations::default().add_overlay(overlays, None),
         char_pos,
     )
-    .0
     .collect_to_str()
 }
 
@@ -143,7 +141,6 @@ fn annotate_text(text: &str, softwrap: bool, annotations: &[InlineAnnotation]) -
         TextAnnotations::default().add_inline_annotations(annotations, None),
         0,
     )
-    .0
     .collect_to_str()
 }
 
@@ -182,7 +179,6 @@ fn annotation_and_overlay() {
                 .add_overlay(overlay.as_slice(), None),
             0,
         )
-        .0
         .collect_to_str(),
         "fooo  bar "
     );

helix-core/src/position.rs

@@ -157,16 +157,14 @@ pub fn visual_offset_from_block(
     annotations: &TextAnnotations,
 ) -> (Position, usize) {
     let mut last_pos = Position::default();
-    let (formatter, block_start) =
+    let mut formatter =
         DocumentFormatter::new_at_prev_checkpoint(text, text_fmt, annotations, anchor);
-    let mut char_pos = block_start;
+    let block_start = formatter.next_char_pos();
 
-    for (grapheme, vpos) in formatter {
-        last_pos = vpos;
-        char_pos += grapheme.doc_chars();
-
-        if char_pos > pos {
-            return (last_pos, block_start);
+    while let Some(grapheme) = formatter.next() {
+        last_pos = grapheme.visual_pos;
+        if formatter.next_char_pos() > pos {
+            return (grapheme.visual_pos, block_start);
         }
     }
 
@@ -189,22 +187,21 @@ pub fn visual_offset_from_anchor(
     annotations: &TextAnnotations,
     max_rows: usize,
 ) -> Result<(Position, usize), VisualOffsetError> {
-    let (formatter, block_start) =
+    let mut formatter =
         DocumentFormatter::new_at_prev_checkpoint(text, text_fmt, annotations, anchor);
-    let mut char_pos = block_start;
     let mut anchor_line = None;
     let mut found_pos = None;
     let mut last_pos = Position::default();
 
+    let block_start = formatter.next_char_pos();
     if pos < block_start {
         return Err(VisualOffsetError::PosBeforeAnchorRow);
     }
 
-    for (grapheme, vpos) in formatter {
-        last_pos = vpos;
-        char_pos += grapheme.doc_chars();
+    while let Some(grapheme) = formatter.next() {
+        last_pos = grapheme.visual_pos;
 
-        if char_pos > pos {
+        if formatter.next_char_pos() > pos {
             if let Some(anchor_line) = anchor_line {
                 last_pos.row -= anchor_line;
                 return Ok((last_pos, block_start));
@@ -212,7 +209,7 @@ pub fn visual_offset_from_anchor(
                 found_pos = Some(last_pos);
             }
         }
-        if char_pos > anchor && anchor_line.is_none() {
+        if formatter.next_char_pos() > anchor && anchor_line.is_none() {
             if let Some(mut found_pos) = found_pos {
                 return if found_pos.row == last_pos.row {
                     found_pos.row = 0;
@@ -226,7 +223,7 @@ pub fn visual_offset_from_anchor(
         }
 
         if let Some(anchor_line) = anchor_line {
-            if vpos.row >= anchor_line + max_rows {
+            if grapheme.visual_pos.row >= anchor_line + max_rows {
                 return Err(VisualOffsetError::PosAfterMaxRow);
             }
         }
@@ -404,34 +401,33 @@ pub fn char_idx_at_visual_block_offset(
     text_fmt: &TextFormat,
     annotations: &TextAnnotations,
 ) -> (usize, usize) {
-    let (formatter, mut char_idx) =
+    let mut formatter =
         DocumentFormatter::new_at_prev_checkpoint(text, text_fmt, annotations, anchor);
-    let mut last_char_idx = char_idx;
+    let mut last_char_idx = formatter.next_char_pos();
     let mut last_char_idx_on_line = None;
     let mut last_row = 0;
-    for (grapheme, grapheme_pos) in formatter {
-        match grapheme_pos.row.cmp(&row) {
+    for grapheme in &mut formatter {
+        match grapheme.visual_pos.row.cmp(&row) {
             Ordering::Equal => {
-                if grapheme_pos.col + grapheme.width() > column {
+                if grapheme.visual_pos.col + grapheme.width() > column {
                     if !grapheme.is_virtual() {
-                        return (char_idx, 0);
+                        return (grapheme.char_idx, 0);
                     } else if let Some(char_idx) = last_char_idx_on_line {
                         return (char_idx, 0);
                     }
                 } else if !grapheme.is_virtual() {
-                    last_char_idx_on_line = Some(char_idx)
+                    last_char_idx_on_line = Some(grapheme.char_idx)
                 }
             }
             Ordering::Greater => return (last_char_idx, row - last_row),
             _ => (),
         }
 
-        last_char_idx = char_idx;
-        last_row = grapheme_pos.row;
-        char_idx += grapheme.doc_chars();
+        last_char_idx = grapheme.char_idx;
+        last_row = grapheme.visual_pos.row;
     }
 
-    (char_idx, 0)
+    (formatter.next_char_pos(), 0)
 }
 
 #[cfg(test)]

helix-term/src/ui/document.rs

@@ -179,21 +179,18 @@ pub fn render_text<'t>(
     line_decorations: &mut [Box<dyn LineDecoration + '_>],
     translated_positions: &mut [TranslatedPosition],
 ) {
-    let (
-        Position {
-            row: mut row_off, ..
-        },
-        mut char_pos,
-    ) = visual_offset_from_block(
+    let mut row_off = visual_offset_from_block(
         text,
         offset.anchor,
         offset.anchor,
         text_fmt,
         text_annotations,
-    );
+    )
+    .0
+    .row;
     row_off += offset.vertical_offset;
 
-    let (mut formatter, mut first_visible_char_idx) =
+    let mut formatter =
         DocumentFormatter::new_at_prev_checkpoint(text, text_fmt, text_annotations, offset.anchor);
     let mut syntax_styles = StyleIter {
         text_style: renderer.text_style,
@@ -226,19 +223,19 @@ pub fn render_text<'t>(
     let mut overlay_style_span = overlay_styles
         .next()
         .unwrap_or_else(|| (Style::default(), usize::MAX));
+    let mut first_visible_char_idx = formatter.next_char_pos();
 
     loop {
         // formattter.line_pos returns to line index of the next grapheme
         // so it must be called before formatter.next
-        let doc_line = formatter.line_pos();
-        let Some((grapheme, mut pos)) = formatter.next() else {
-            let mut last_pos = formatter.visual_pos();
+        let Some(mut grapheme) = formatter.next() else {
+            let mut last_pos = formatter.next_visual_pos();
             if last_pos.row >= row_off {
                 last_pos.col -= 1;
                 last_pos.row -= row_off;
                 // check if any positions translated on the fly (like cursor) are at the EOF
                 translate_positions(
-                    char_pos + 1,
+                    text.len_chars() + 1,
                     first_visible_char_idx,
                     translated_positions,
                     text_fmt,
@@ -250,46 +247,56 @@ pub fn render_text<'t>(
         };
 
         // skip any graphemes on visual lines before the block start
-        if pos.row < row_off {
-            if char_pos >= syntax_style_span.1 {
-                syntax_style_span = if let Some(syntax_style_span) = syntax_styles.next() {
-                    syntax_style_span
+        // if pos.row < row_off {
+        //     if char_pos >= syntax_style_span.1 {
+        //         syntax_style_span = if let Some(syntax_style_span) = syntax_styles.next() {
+        //             syntax_style_span
+        //         } else {
+        //             break;
+        //         }
+        //     }
+        //     if char_pos >= overlay_style_span.1 {
+        //         overlay_style_span = if let Some(overlay_style_span) = overlay_styles.next() {
+        //             overlay_style_span
+        if grapheme.visual_pos.row < row_off {
+            if grapheme.char_idx >= style_span.1 {
+                style_span = if let Some(style_span) = styles.next() {
+                    style_span
                 } else {
                     break;
-                }
-            }
-            if char_pos >= overlay_style_span.1 {
-                overlay_style_span = if let Some(overlay_style_span) = overlay_styles.next() {
-                    overlay_style_span
+                };    
+                overlay_span = if let Some(overlay_span) = overlays.next() {
+                    overlay_span
                 } else {
                     break;
-                }
+                };
             }
-            char_pos += grapheme.doc_chars();
-            first_visible_char_idx = char_pos + 1;
+            first_visible_char_idx = formatter.next_char_pos();
             continue;
         }
-        pos.row -= row_off;
+        grapheme.visual_pos.row -= row_off;
 
         // if the end of the viewport is reached stop rendering
-        if pos.row as u16 >= renderer.viewport.height {
+        if grapheme.visual_pos.row as u16 >= renderer.viewport.height + renderer.offset.row as u16 {
             break;
         }
 
         // apply decorations before rendering a new line
-        if pos.row as u16 != last_line_pos.visual_line {
-            if pos.row > 0 {
-                renderer.draw_indent_guides(last_line_indent_level, last_line_pos.visual_line);
+        if grapheme.visual_pos.row as u16 != last_line_pos.visual_line {
+            if grapheme.visual_pos.row > 0 {
+                // draw indent guides for the last line
+                renderer
+                    .draw_indent_guides(last_line_indent_level, last_line_pos.visual_line as u16);
                 is_in_indent_area = true;
                 for line_decoration in &mut *line_decorations {
-                    line_decoration.render_foreground(renderer, last_line_pos, char_pos);
+                    line_decoration.render_foreground(renderer, last_line_pos, grapheme.char_idx);
                 }
             }
             last_line_pos = LinePos {
-                first_visual_line: doc_line != last_line_pos.doc_line,
-                doc_line,
-                visual_line: pos.row as u16,
-                start_char_idx: char_pos,
+                first_visual_line: grapheme.line_idx != last_line_pos.doc_line,
+                doc_line: grapheme.line_idx,
+                visual_line: grapheme.visual_pos.row as u16,
+                start_char_idx: grapheme.char_idx,
             };
             for line_decoration in &mut *line_decorations {
                 line_decoration.render_background(renderer, last_line_pos);
@@ -297,26 +304,25 @@ pub fn render_text<'t>(
         }
 
         // acquire the correct grapheme style
-        while char_pos >= syntax_style_span.1 {
+        while  grapheme.char_idx >= syntax_style_span.1 {
             syntax_style_span = syntax_styles
                 .next()
                 .unwrap_or((Style::default(), usize::MAX));
         }
-        while char_pos >= overlay_style_span.1 {
+        while  grapheme.char_idx >= overlay_style_span.1 {
             overlay_style_span = overlay_styles
                 .next()
                 .unwrap_or((Style::default(), usize::MAX));
         }
-        char_pos += grapheme.doc_chars();
 
         // check if any positions translated on the fly (like cursor) has been reached
         translate_positions(
-            char_pos,
+            formatter.next_char_pos(),
             first_visible_char_idx,
             translated_positions,
             text_fmt,
             renderer,
-            pos,
+            grapheme.visual_pos,
         );
 
         let (syntax_style, overlay_style) =
@@ -332,27 +338,37 @@ pub fn render_text<'t>(
 
         let is_virtual = grapheme.is_virtual();
         renderer.draw_grapheme(
+<<<<<<< HEAD
             grapheme.grapheme,
             GraphemeStyle {
                 syntax_style,
                 overlay_style,
             },
             is_virtual,
+||||||| parent of 5e32edd8 (track char_idx in DocFormatter)
+            grapheme.grapheme,
+            grapheme_style,
+            virt,
+=======
+            grapheme.raw,
+            grapheme_style,
+            virt,
+>>>>>>> 5e32edd8 (track char_idx in DocFormatter)
             &mut last_line_indent_level,
             &mut is_in_indent_area,
-            pos,
+            grapheme.visual_pos,
         );
     }
 
     renderer.draw_indent_guides(last_line_indent_level, last_line_pos.visual_line);
     for line_decoration in &mut *line_decorations {
-        line_decoration.render_foreground(renderer, last_line_pos, char_pos);
+        line_decoration.render_foreground(renderer, last_line_pos, formatter.next_char_pos());
     }
 }
 
 #[derive(Debug)]
 pub struct TextRenderer<'a> {
-    pub surface: &'a mut Surface,
+    surface: &'a mut Surface,
     pub text_style: Style,
     pub whitespace_style: Style,
     pub indent_guide_char: String,