Add MSep Token

src/lib.rs

@@ -102,6 +102,33 @@ mod tests {
         );
     }
 
+    #[test]
+    fn it_tokenizes_expressions3() {
+        let expression = "[[1, 2],[3, 4]]";
+        let mut tokenizer = Tokenizer::new(expression.to_string());
+        let tokens = tokenizer.parse();
+        assert_eq!(
+            tokens,
+            vec![
+                Token::Grouping(Grouping::RBracket),
+                Token::Grouping(Grouping::RBracket),
+                Token::Text(Text::Number("1".to_string())),
+                Token::Grouping(Grouping::MSep),
+                Token::Text(Text::Whitespace),
+                Token::Text(Text::Number("2".to_string())),
+                Token::Grouping(Grouping::LBracket),
+                Token::Grouping(Grouping::MSep),
+                Token::Grouping(Grouping::RBracket),
+                Token::Text(Text::Number("3".to_string())),
+                Token::Grouping(Grouping::MSep),
+                Token::Text(Text::Whitespace),
+                Token::Text(Text::Number("4".to_string())),
+                Token::Grouping(Grouping::LBracket),
+                Token::Grouping(Grouping::LBracket),
+            ]
+        );
+    }
+
     #[test]
     fn it_tokenizes_text1() {
         let expression = "\"just plain text\"";

src/parsing/tokenizer.rs

@@ -1,3 +1,4 @@
+use crate::elements::Element;
 use crate::tokens::constants::accents::G_COLOR;
 use crate::tokens::constants::grouping::T_LPAREN;
 use crate::tokens::constants::misc::{A_TEXT, G_NUMALLOWED};
@@ -7,6 +8,7 @@ use crate::tokens::mappings::{
     get_greek_mappings, get_grouping_mappings, get_logical_mappings, get_misc_mappings,
     get_operation_mappings, get_relation_mapping,
 };
+use crate::tokens::Grouping::MatrixEnd;
 use crate::tokens::{
     Accent, Arrow, FontCommand, Function, Greek, Grouping, Logical, Misc, Operation, Relation,
     Text, Token,

src/tokens/constants/grouping.rs

@@ -16,4 +16,6 @@ pub const G_FLOOR: &'static [&str] = &["floor"];
 pub const G_CEIL: &'static [&str] = &["ceil"];
 pub const G_NORM: &'static [&str] = &["norm"];
 
+pub const G_MATRIX_SEP: &'static [&str] = &[","];
+
 pub const T_LPAREN: char = ')';

src/tokens/mappings.rs

@@ -159,6 +159,7 @@ pub fn get_grouping_mappings() -> Vec<HashMap<TokenPattern, Grouping>> {
             G_RANGLE => Grouping::RAngle,
             G_RXPAR => Grouping::RXPar,
             G_LXPAR => Grouping::LXPar,
+            G_MATRIX_SEP => Grouping::MSep,
         },
         hashmap! {
             G_RPAREN => Grouping::RParen,

src/tokens/mod.rs

@@ -152,6 +152,11 @@ pub enum Grouping {
     Floor,
     Ceil,
     Norm,
+    MatrixBegin,
+    MatrixEnd,
+    VecBegin,
+    VecEnd,
+    MSep,
 }
 
 #[derive(Debug, Clone, PartialOrd, PartialEq)]