diff --git a/helix-core/src/movement.rs b/helix-core/src/movement.rs
index 6c4f3f535..8d5d2b995 100644
--- a/helix-core/src/movement.rs
+++ b/helix-core/src/movement.rs
@@ -64,7 +64,6 @@ pub fn move_vertically_visual(
     if !text_fmt.soft_wrap {
         return move_vertically(slice, range, dir, count, behaviour, text_fmt, annotations);
     }
-    annotations.clear_line_annotations();
     let pos = range.cursor(slice);
 
     // Compute the current position's 2d coordinates.
@@ -112,7 +111,6 @@ pub fn move_vertically(
     text_fmt: &TextFormat,
     annotations: &mut TextAnnotations,
 ) -> Range {
-    annotations.clear_line_annotations();
     let pos = range.cursor(slice);
     let line_idx = slice.char_to_line(pos);
     let line_start = slice.line_to_char(line_idx);
diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 0d13582f4..ba1975a29 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -595,6 +595,7 @@ fn move_impl(cx: &mut Context, move_fn: MoveFn, dir: Direction, behaviour: Movem
     let text = doc.text().slice(..);
     let text_fmt = doc.text_format(view.inner_area(doc).width, None);
     let mut annotations = view.text_annotations(doc, None);
+    annotations.clear_line_annotations();
 
     let selection = doc.selection(view.id).clone().transform(|range| {
         move_fn(