Add operation mappings and parsing

Cargo.toml

@@ -7,4 +7,6 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-charred = "0.2.2"
+charred = "0.3.0"
+maplit = "1.0.2"
+lazy_static = "1.4.0"

src/lib.rs

@@ -1,4 +1,7 @@
+#[macro_use] extern crate maplit;
+#[macro_use] extern crate lazy_static;
 pub mod elements;
+pub mod tokenizer;
 mod tokens;
 
 #[cfg(test)]

src/tokenizer/mod.rs

@@ -0,0 +1,29 @@
+use charred::tapemachine::CharTapeMachine;
+use crate::tokens::{Token, Operation};
+use crate::tokens::constants::operations::{G_PLUS, G_MINUS, G_CDOT, G_AST, G_STAR, G_SLASH, G_BACKSLASH, G_TIMES, G_DIV, G_LTIMES, G_RTIMES, G_BOWTIE, G_CIRC, G_OPLUS, G_ODOT, G_SUM, G_PROD, G_WEDGE, G_BIDWEDGE, G_VEE, G_BIGVEE, G_CAP, G_BIGCAP, G_CUP, G_BIGCUP};
+use std::collections::HashMap;
+use crate::tokens::mappings::get_operation_mapping;
+
+pub struct Tokenizer {
+    ctm: CharTapeMachine,
+    tokens: Vec<Token>,
+}
+
+impl Tokenizer {
+    pub fn new(text: String) -> Self {
+        Self {
+            ctm: CharTapeMachine::new(text.chars().collect()),
+            tokens: Vec::new(),
+        }
+    }
+
+    fn parse_operation(&mut self) -> Option<Operation> {
+        lazy_static! { static ref OPERATION_MAPPING: HashMap<&'static[&'static str], Operation> = get_operation_mapping();}
+        for key in OPERATION_MAPPING.keys() {
+            if self.ctm.check_any_str_sequence(*key) {
+                return Some(OPERATION_MAPPING[key].clone())
+            }
+        }
+        None
+    }
+}

src/tokens/constants/misc.rs

@@ -16,7 +16,7 @@ pub const G_THEREFORE: &'static[&str] = &[":.", "therefore"];
 pub const G_BECAUSE: &'static[&str] = &[":'", "because"];
 pub const G_ELDOTS: &'static[&str] = &["|...|", "|ldots|"];
 pub const G_ECDOTS: &'static[&str] = &["|cdots|"];
-pub const G_VDOTs: &'static[&str] = &["vdots"];
+pub const G_VDOTS: &'static[&str] = &["vdots"];
 pub const G_DDOTS: &'static[&str] = &["ddots"];
 pub const G_EPIPES: &'static[&str] = &["|\\ |"];
 pub const G_QUAD: &'static[&str] = &["|quad|"];

src/tokens/constants/operations.rs

@@ -2,7 +2,7 @@ pub const G_PLUS: &'static[&str] = &["+"];
 pub const G_MINUS: &'static[&str] = &["-"];
 pub const G_CDOT: &'static[&str] = &["*", "cdot"];
 pub const G_AST: &'static[&str] = &["**", "ast"];
-pub const G_STAR: &'static[&str] = &["***"; "star"];
+pub const G_STAR: &'static[&str] = &["***", "star"];
 pub const G_SLASH: &'static[&str] = &["////"];
 pub const G_BACKSLASH: &'static[&str] = &["\\\\", "backslash", "setminus"];
 pub const G_TIMES: &'static[&str] = &["xx", "times"];

src/tokens/mappings.rs

@@ -0,0 +1,34 @@
+use std::collections::HashMap;
+use crate::tokens::Operation;
+use crate::tokens::constants::operations::*;
+
+pub fn get_operation_mapping() -> HashMap<&'static[&'static str], Operation> {
+    hashmap! {
+        G_PLUS => Operation::Plus,
+        G_MINUS => Operation::Minus,
+        G_CDOT => Operation::CDot,
+        G_AST => Operation::Ast,
+        G_STAR => Operation::Star,
+        G_SLASH => Operation::Slash,
+        G_BACKSLASH => Operation::Backslash,
+        G_TIMES => Operation::Times,
+        G_DIV => Operation::Div,
+        G_LTIMES => Operation::LTimes,
+        G_RTIMES => Operation::RTimes,
+        G_BOWTIE => Operation::Bowtie,
+        G_CIRC => Operation::Circ,
+        G_OPLUS => Operation::OPlus,
+        G_OTIMES => Operation::OTimes,
+        G_ODOT => Operation::ODot,
+        G_SUM => Operation::Sum,
+        G_PROD => Operation::Prod,
+        G_WEDGE => Operation::Wedge,
+        G_BIDWEDGE => Operation::BidWedge,
+        G_VEE => Operation::Vee,
+        G_BIGVEE => Operation::BigVee,
+        G_CAP => Operation::Cap,
+        G_BIGCAP => Operation::BigCap,
+        G_CUP => Operation::Cup,
+        G_BIGCUP => Operation::BigCup,
+    }
+}

src/tokens/mod.rs

@@ -1,5 +1,7 @@
 pub mod constants;
+pub mod mappings;
 
+#[derive(Debug, Clone)]
 pub enum Token {
     Operation(Operation),
     Misc(Misc),
@@ -12,6 +14,7 @@ pub enum Token {
     Font(FontCommand),
 }
 
+#[derive(Debug, Clone)]
 pub enum Operation {
     Plus,
     Minus,
@@ -41,6 +44,7 @@ pub enum Operation {
     BigCup,
 }
 
+#[derive(Debug, Clone)]
 pub enum Misc {
     Frac,
     Pow,
@@ -78,6 +82,7 @@ pub enum Misc {
     Text,
 }
 
+#[derive(Debug, Clone)]
 pub enum Relation {
     Eq,
     Ne,
@@ -101,6 +106,7 @@ pub enum Relation {
     PropTo,
 }
 
+#[derive(Debug, Clone)]
 pub enum Logical {
     And,
     Or,
@@ -115,6 +121,7 @@ pub enum Logical {
     Models,
 }
 
+#[derive(Debug, Clone)]
 pub enum Grouping {
     RParen,
     LParen,
@@ -132,6 +139,7 @@ pub enum Grouping {
     Norm,
 }
 
+#[derive(Debug, Clone)]
 pub enum Arrow {
     UpArrow,
     DownArrow,
@@ -148,6 +156,7 @@ pub enum Arrow {
     BigLeftRightArrow,
 }
 
+#[derive(Debug, Clone)]
 pub enum Accent {
     Hat,
     Overline,
@@ -163,6 +172,7 @@ pub enum Accent {
     Cancel,
 }
 
+#[derive(Debug, Clone)]
 pub enum Greek {
     Alpha,
     Beta,
@@ -200,6 +210,7 @@ pub enum Greek {
     Omega,
 }
 
+#[derive(Debug, Clone)]
 pub enum FontCommand {
     Big,
     BigOutline,