Allow explicit newlines in test DSL

The current test DSL currently has no way to express being at the end of
a line, save for putting an explicit LF or CRLF inside the `#[|]#`. The
problem with this approach is that it can add unintended extra new lines
if used in conjunction with raw strings, which insert newlines for you.

This is a simple attempt to mitigate this problem. If there is an
explicit newline character at the end of the selection, and then it
is immediately followed by the same newline character at the right end
of the selection, this following newline is removed. This way, one can
express a cursor at the end of a line explicitly.

Cargo.lock

@@ -143,9 +143,9 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.24"
+version = "0.4.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b"
+checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
 dependencies = [
  "iana-time-zone",
  "num-integer",
@@ -932,9 +932,9 @@ dependencies = [
 
 [[package]]
 name = "gix-tempfile"
-version = "5.0.0"
+version = "5.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "743bae41b5db7f085dc7acc54ed72c98853a6e5dabb355e95caa7b534f21b35c"
+checksum = "aed73ef9642f779d609fd19acc332ac1597b978ee87ec11743a68eefaed65bfa"
 dependencies = [
  "libc",
  "once_cell",
@@ -1081,6 +1081,7 @@ dependencies = [
  "hashbrown 0.13.2",
  "helix-loader",
  "imara-diff",
+ "indoc 1.0.9",
  "log",
  "once_cell",
  "quickcheck",
@@ -1176,7 +1177,7 @@ dependencies = [
  "helix-vcs",
  "helix-view",
  "ignore",
- "indoc",
+ "indoc 2.0.1",
  "libc",
  "log",
  "once_cell",
@@ -1349,6 +1350,12 @@ dependencies = [
  "hashbrown 0.12.3",
 ]
 
+[[package]]
+name = "indoc"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306"
+
 [[package]]
 name = "indoc"
 version = "2.0.1"
@@ -1532,6 +1539,15 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "nom8"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.45"
@@ -1775,18 +1791,18 @@ checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898"
 
 [[package]]
 name = "serde"
-version = "1.0.155"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71f2b4817415c6d4210bfe1c7bfcf4801b2d904cb4d0e1a8fdb651013c9e86b8"
+checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.155"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d071a94a3fac4aff69d023a7f411e33f40f3483f8c5190b1953822b6b76d7630"
+checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2120,9 +2136,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.7.3"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21"
+checksum = "f7afcae9e3f0fe2c370fd4657108972cbb2fa9db1b9f84849cefd80741b01cb6"
 dependencies = [
  "serde",
  "serde_spanned",
@@ -2141,15 +2157,15 @@ dependencies = [
 
 [[package]]
 name = "toml_edit"
-version = "0.19.6"
+version = "0.19.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08de71aa0d6e348f070457f85af8bd566e2bc452156a423ddf22861b3a953fae"
+checksum = "5e6a7712b49e1775fb9a7b998de6635b299237f48b404dde71704f2e0e7f37e5"
 dependencies = [
  "indexmap",
+ "nom8",
  "serde",
  "serde_spanned",
  "toml_datetime",
- "winnow",
 ]
 
 [[package]]
@@ -2452,15 +2468,6 @@ version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
 
-[[package]]
-name = "winnow"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee7b2c67f962bf5042bfd8b6a916178df33a26eec343ae064cb8e069f638fa6f"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "xtask"
 version = "0.6.0"

helix-core/Cargo.toml

@@ -49,3 +49,4 @@ textwrap = "0.16.0"
 
 [dev-dependencies]
 quickcheck = { version = "1", default-features = false }
+indoc = "1.0.6"

helix-core/src/movement.rs

@@ -1474,7 +1474,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection =
                 selection.transform(|r| move_prev_paragraph(text.slice(..), r, 1, Movement::Move));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(&s, &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -1497,7 +1497,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection =
                 selection.transform(|r| move_prev_paragraph(text.slice(..), r, 2, Movement::Move));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(&s, &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -1520,7 +1520,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection = selection
                 .transform(|r| move_prev_paragraph(text.slice(..), r, 1, Movement::Extend));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(&s, &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -1540,7 +1540,7 @@ mod test {
                 "a\nb\n\n#[goto\nthird\n\n|]#paragraph",
             ),
             (
-                "a\nb#[\n|]#\ngoto\nsecond\n\nparagraph",
+                "a\nb#[\n|]#\n\ngoto\nsecond\n\nparagraph",
                 "a\nb#[\n\n|]#goto\nsecond\n\nparagraph",
             ),
             (
@@ -1562,7 +1562,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection =
                 selection.transform(|r| move_next_paragraph(text.slice(..), r, 1, Movement::Move));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(&s, &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -1585,7 +1585,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection =
                 selection.transform(|r| move_next_paragraph(text.slice(..), r, 2, Movement::Move));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(&s, &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -1608,7 +1608,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection = selection
                 .transform(|r| move_next_paragraph(text.slice(..), r, 1, Movement::Extend));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(&s, &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }

helix-core/src/test.rs

@@ -2,6 +2,7 @@
 use crate::{Range, Selection};
 use smallvec::SmallVec;
 use std::cmp::Reverse;
+use unicode_segmentation::UnicodeSegmentation;
 
 /// Convert annotated test string to test string and selection.
 ///
@@ -10,6 +11,10 @@ use std::cmp::Reverse;
 /// `#[` for primary selection with head after anchor followed by `|]#`.
 /// `#(` for secondary selection with head after anchor followed by `|)#`.
 ///
+/// If the selection contains any LF or CRLF sequences, which are immediately
+/// followed by the same grapheme, then the subsequent one is removed. This is
+/// to allow representing having the cursor over the end of the line.
+///
 /// # Examples
 ///
 /// ```
@@ -30,23 +35,23 @@ use std::cmp::Reverse;
 pub fn print(s: &str) -> (String, Selection) {
     let mut primary_idx = None;
     let mut ranges = SmallVec::new();
-    let mut iter = s.chars().peekable();
+    let mut iter = UnicodeSegmentation::graphemes(s, true).peekable();
     let mut left = String::with_capacity(s.len());
 
     'outer: while let Some(c) = iter.next() {
         let start = left.chars().count();
 
-        if c != '#' {
+        if c != "#" {
-            left.push(c);
+            left.push_str(c);
             continue;
         }
 
         let (is_primary, close_pair) = match iter.next() {
-            Some('[') => (true, ']'),
+            Some("[") => (true, "]"),
-            Some('(') => (false, ')'),
+            Some("(") => (false, ")"),
             Some(ch) => {
                 left.push('#');
-                left.push(ch);
+                left.push_str(ch);
                 continue;
             }
             None => break,
@@ -56,24 +61,45 @@ pub fn print(s: &str) -> (String, Selection) {
             panic!("primary `#[` already appeared {:?} {:?}", left, s);
         }
 
-        let head_at_beg = iter.next_if_eq(&'|').is_some();
+        let head_at_beg = iter.next_if_eq(&"|").is_some();
+        let last_grapheme = |s: &str| {
+            UnicodeSegmentation::graphemes(s, true)
+                .last()
+                .map(String::from)
+        };
 
         while let Some(c) = iter.next() {
-            if !(c == close_pair && iter.peek() == Some(&'#')) {
+            let next = iter.peek();
-                left.push(c);
+            let mut prev = last_grapheme(left.as_str());
+
+            if !(c == close_pair && next == Some(&"#")) {
+                left.push_str(c);
                 continue;
             }
 
             if !head_at_beg {
-                let prev = left.pop().unwrap();
+                match &prev {
-                if prev != '|' {
+                    Some(p) if p != "|" => {
-                    left.push(prev);
+                        left.push_str(c);
-                    left.push(c);
+                        continue;
-                    continue;
+                    }
+                    Some(p) if p == "|" => {
+                        left.pop().unwrap(); // pop the |
+                        prev = last_grapheme(left.as_str());
+                    }
+                    _ => (),
                 }
             }
 
             iter.next(); // skip "#"
+            let next = iter.peek();
+
+            // skip explicit line end inside selection
+            if (prev == Some(String::from("\r\n")) || prev == Some(String::from("\n")))
+                && next.map(|n| String::from(*n)) == prev
+            {
+                iter.next();
+            }
 
             if is_primary {
                 primary_idx = Some(ranges.len());
@@ -118,11 +144,11 @@ pub fn print(s: &str) -> (String, Selection) {
 /// use smallvec::smallvec;
 ///
 /// assert_eq!(
-///     plain("abc", Selection::new(smallvec![Range::new(0, 1), Range::new(3, 2)], 0)),
+///     plain("abc", &Selection::new(smallvec![Range::new(0, 1), Range::new(3, 2)], 0)),
 ///     "#[a|]#b#(|c)#".to_owned()
 /// );
 /// ```
-pub fn plain(s: &str, selection: Selection) -> String {
+pub fn plain(s: &str, selection: &Selection) -> String {
     let primary = selection.primary_index();
     let mut out = String::with_capacity(s.len() + 5 * selection.len());
     out.push_str(s);
@@ -147,6 +173,7 @@ pub fn plain(s: &str, selection: Selection) -> String {
     out
 }
 
+#[allow(clippy::module_inception)]
 #[cfg(test)]
 #[allow(clippy::module_inception)]
 mod test {

helix-core/src/textobject.rs

@@ -437,7 +437,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection = selection
                 .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Inside, 1));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(&s, &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -460,7 +460,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection = selection
                 .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Inside, 2));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(&s, &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -491,7 +491,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection = selection
                 .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Around, 1));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(&s, &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }

helix-term/tests/test/commands.rs

@@ -201,12 +201,12 @@ async fn test_multi_selection_shell_commands() -> anyhow::Result<()> {
         .as_str(),
         "|echo foo<ret>",
         platform_line(indoc! {"\
-            #[|foo
+            #[|foo\n]#
-            ]#
+            
-            #(|foo
+            #(|foo\n)#
-            )#
+            
-            #(|foo
+            #(|foo\n)#
-            )#
+            
             "})
         .as_str(),
     ))
@@ -222,12 +222,12 @@ async fn test_multi_selection_shell_commands() -> anyhow::Result<()> {
         .as_str(),
         "!echo foo<ret>",
         platform_line(indoc! {"\
-            #[|foo
+            #[|foo\n]#
-            ]#lorem
+            lorem
-            #(|foo
+            #(|foo\n)#
-            )#ipsum
+            ipsum
-            #(|foo
+            #(|foo\n)#
-            )#dolor
+            dolor
             "})
         .as_str(),
     ))
@@ -243,12 +243,12 @@ async fn test_multi_selection_shell_commands() -> anyhow::Result<()> {
         .as_str(),
         "<A-!>echo foo<ret>",
         platform_line(indoc! {"\
-            lorem#[|foo
+            lorem#[|foo\n]#
-            ]#
+            
-            ipsum#(|foo
+            ipsum#(|foo\n)#
-            )#
+            
-            dolor#(|foo
+            dolor#(|foo\n)#
-            )#
+            
             "})
         .as_str(),
     ))
@@ -300,8 +300,8 @@ async fn test_extend_line() -> anyhow::Result<()> {
         platform_line(indoc! {"\
             #[lorem
             ipsum
-            dolor
+            dolor\n|]#
-            |]#
+            
             "})
         .as_str(),
     ))
@@ -318,8 +318,8 @@ async fn test_extend_line() -> anyhow::Result<()> {
         "2x",
         platform_line(indoc! {"\
             #[lorem
-            ipsum
+            ipsum\n|]#
-            |]#
+            
             "})
         .as_str(),
     ))