From d30a227df8d5537c42f8e46b906f9d99a09f7692 Mon Sep 17 00:00:00 2001
From: trivernis <trivernis@protonmail.com>
Date: Fri, 7 Aug 2020 16:00:00 +0200
Subject: [PATCH] Add newline token \\n

---
 Cargo.toml                   | 4 ++--
 src/elements/literal.rs      | 1 +
 src/format/mathml.rs         | 1 +
 src/lib.rs                   | 8 +++++---
 src/parsing/tokenizer.rs     | 5 ++++-
 src/parsing/tree_parser.rs   | 1 +
 src/tokens/constants/misc.rs | 1 +
 src/tokens/mod.rs            | 1 +
 8 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 998deec..d37219e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -2,7 +2,7 @@
 name = "asciimath-rs"
 description = "AsciiMath parser"
 repository = "https://github.com/trivernis/asciimath-rs"
-version = "0.4.8"
+version = "0.5.0"
 authors = ["trivernis <trivernis@protonmail.com>"]
 edition = "2018"
 readme = "README.md"
@@ -13,7 +13,7 @@ categories = ["mathematics", "parser-implementations"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-charred = "0.3.0"
+charred = "0.3.1"
 maplit = "1.0.2"
 lazy_static = "1.4.0"
 htmlescape = "0.3.1"
\ No newline at end of file
diff --git a/src/elements/literal.rs b/src/elements/literal.rs
index 76e81d0..8d13db7 100644
--- a/src/elements/literal.rs
+++ b/src/elements/literal.rs
@@ -13,6 +13,7 @@ pub enum Literal {
     Arrow(Arrow),
     Misc(Misc),
     Operation(Operation),
+    NewLine,
 }
 
 #[derive(Debug, Clone, PartialOrd, PartialEq)]
diff --git a/src/format/mathml.rs b/src/format/mathml.rs
index c6a0d8c..3a5815c 100644
--- a/src/format/mathml.rs
+++ b/src/format/mathml.rs
@@ -32,6 +32,7 @@ impl ToMathML for Literal {
             Literal::Arrow(a) => a.to_mathml(),
             Literal::Misc(m) => m.to_mathml(),
             Literal::Operation(o) => o.to_mathml(),
+            Literal::NewLine => "<mspace linebreak='newline' />".to_string(),
         }
     }
 }
diff --git a/src/lib.rs b/src/lib.rs
index 5533075..eb0e6c8 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -118,7 +118,7 @@ mod tests {
 
     #[test]
     fn it_tokenizes_expressions3() {
-        let expression = "[[1, 2],[3, 4]] //";
+        let expression = "[[1, 2],[3, 4]] // \\\n";
         let mut tokenizer = Tokenizer::new(expression.to_string());
         let tokens = tokenizer.parse();
         assert_eq!(
@@ -141,6 +141,8 @@ mod tests {
                 Token::Grouping(Grouping::LBracket),
                 Token::Text(Text::Whitespace),
                 Token::Operation(Operation::Slash),
+                Token::Text(Text::Whitespace),
+                Token::Text(Text::NewLine),
             ]
         );
     }
@@ -363,10 +365,10 @@ mod tests {
     }
 
     #[allow(dead_code)]
-    //#[test]
+    #[test]
     fn it_writes_mathml() {
         let str_expression =
-            "alpha sqrt 1 in NN implies 2^4 + <=> sum_(k = 1)^3 - ((1),(2))[[2, 3 + 3],[4, 5]]  + alpha";
+            "alpha sqrt 1 in NN implies 2^4 + \\\n<=> sum_(k = 1)^3 - ((1),(2))[[2, 3 + 3],[4, 5]]  + alpha";
         let expression = parse(str_expression.to_string());
         fs::write(
             "test-files/test.html",
diff --git a/src/parsing/tokenizer.rs b/src/parsing/tokenizer.rs
index 678c2a7..0cfa27c 100644
--- a/src/parsing/tokenizer.rs
+++ b/src/parsing/tokenizer.rs
@@ -1,6 +1,6 @@
 use crate::tokens::constants::accents::G_COLOR;
 use crate::tokens::constants::grouping::T_LPAREN;
-use crate::tokens::constants::misc::{A_TEXT, G_NUMALLOWED};
+use crate::tokens::constants::misc::{A_TEXT, G_NEWLINE, G_NUMALLOWED};
 use crate::tokens::constants::TokenPattern;
 use crate::tokens::mappings::{
     get_accent_mappings, get_arrow_mapping, get_font_mappings, get_function_mappings,
@@ -227,6 +227,9 @@ impl Tokenizer {
     }
 
     fn parse_whitespace(&mut self) -> Option<Text> {
+        if self.ctm.check_any_str_sequence(G_NEWLINE) {
+            return Some(Text::NewLine);
+        }
         if self.ctm.get_current().is_whitespace() {
             self.ctm.seek_whitespace();
             self.ctm.rewind(self.ctm.get_index() - 1);
diff --git a/src/parsing/tree_parser.rs b/src/parsing/tree_parser.rs
index a965ec5..6ad8a3a 100644
--- a/src/parsing/tree_parser.rs
+++ b/src/parsing/tree_parser.rs
@@ -197,6 +197,7 @@ impl TreeParser {
                 text: p,
                 formatting: None,
             })),
+            Text::NewLine => Some(Literal::NewLine),
             _ => None,
         }
     }
diff --git a/src/tokens/constants/misc.rs b/src/tokens/constants/misc.rs
index e579183..2505cde 100644
--- a/src/tokens/constants/misc.rs
+++ b/src/tokens/constants/misc.rs
@@ -41,5 +41,6 @@ pub const A_TEXT: char = '"';
 pub const G_T_TEX: &'static [&str] = &["text"];
 pub const A_NUMCOMMA: char = '.';
 pub const A_SCIEXP: char = 'e';
+pub const G_NEWLINE: &'static [&str] = &["\\\n"];
 
 pub const G_NUMALLOWED: &'static [char] = &[A_NUMCOMMA, A_SCIEXP];
diff --git a/src/tokens/mod.rs b/src/tokens/mod.rs
index 914fde2..89ca4eb 100644
--- a/src/tokens/mod.rs
+++ b/src/tokens/mod.rs
@@ -23,6 +23,7 @@ pub enum Text {
     Symbol(String),
     Plain(String),
     Whitespace,
+    NewLine,
 }
 
 #[derive(Debug, Clone, PartialOrd, PartialEq)]