From 317878dd3b367a46f1b8bb2811d2efea76eb2526 Mon Sep 17 00:00:00 2001 From: 0x4261756D <38735823+0x4261756D@users.noreply.github.com> Date: Tue, 6 Jun 2023 23:56:38 +0200 Subject: [PATCH] deeduplicate code --- src/main.rs | 2251 ++++++--------------------------------------------- 1 file changed, 234 insertions(+), 2017 deletions(-) diff --git a/src/main.rs b/src/main.rs index fe9b7f2..a1af587 100644 --- a/src/main.rs +++ b/src/main.rs @@ -69,6 +69,66 @@ enum TokenizerState Function, } +fn tokenize_terminal_no_str(last_index: &mut i32, index: usize, token: &mut Option, state: &mut TokenizerState, new_token: Option, new_state: TokenizerState) +{ + *last_index = index as i32; + *token = new_token; + *state = new_state; +} +fn tokenize_terminal(last_index: &mut i32, index: usize, token: &mut Option, state: &mut TokenizerState, new_token: Option, new_state: TokenizerState, token_str: &mut String, ch: char) +{ + tokenize_terminal_no_str(last_index, index, token, state, new_token, new_state); + token_str.push(ch); +} +fn tokenize_backtrack(last_index: &mut i32, index: &mut usize, tokens: &mut Vec, token: &mut Option, token_str: &mut String, state: &mut TokenizerState) -> Result<(), &'static str> +{ + return tokenize_backtrack_custom_token(last_index, index, tokens, token, token_str, state, token.clone().unwrap()); +} +fn tokenize_backtrack_name(last_index: &mut i32, index: &mut usize, tokens: &mut Vec, token: &mut Option, token_str: &mut String, state: &mut TokenizerState) -> Result<(), &'static str> +{ + if *last_index == -1 || token.is_none() + { + println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); + return Err("Lexerr"); + } + *index = *last_index as usize; + *last_index = -1; + tokens.push(Token::Name(token_str.clone())); + *token = None; + token_str.clear(); + *state = TokenizerState::Start; + return Ok(()); +} +fn tokenize_backtrack_custom_token(last_index: &mut i32, index: &mut usize, tokens: &mut Vec, token: &mut Option, token_str: &mut String, state: &mut TokenizerState, new_token: Token) -> Result<(), &'static str> +{ + if *last_index == -1 || token.is_none() + { + println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); + return Err("Lexerr"); + } + *index = *last_index as usize; + *last_index = -1; + tokens.push(new_token); + *token = None; + token_str.clear(); + *state = TokenizerState::Start; + return Ok(()); +} +fn tokenize_alphanumeric_nonterminal(last_index: &mut i32, index: &mut usize, tokens: &mut Vec, token: &mut Option, token_str: &mut String, state: &mut TokenizerState, ch: char) -> Result<(), &'static str> +{ + if ch.is_ascii_alphanumeric() || ch == '_' + { + *last_index = *index as i32; + token_str.push(ch); + *state = TokenizerState::Name; + } + else + { + tokenize_backtrack_name(last_index, index, tokens, token, token_str, state)?; + } + return Ok(()); +} + fn tokenize(file_content: &String) -> Result, &'static str> { let mut tokens: Vec = Vec::new(); @@ -90,249 +150,44 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - '-' => - { - last_index = index as i32; - token = Some(Token::Minus); - state = TokenizerState::Minus; - } - 'a' => - { - last_index = index as i32; - token = Some(Token::Name("a".to_string())); - token_str.push(ch); - state = TokenizerState::A; - } - 'b' => - { - last_index = index as i32; - token = Some(Token::Name("b".to_string())); - token_str.push(ch); - state = TokenizerState::B; - } - 'd' => - { - last_index = index as i32; - token = Some(Token::Name("d".to_string())); - token_str.push(ch); - state = TokenizerState::D; - } - 'e' => - { - last_index = index as i32; - token = Some(Token::Name("e".to_string())); - token_str.push(ch); - state = TokenizerState::E; - } - 'f' => - { - last_index = index as i32; - token = Some(Token::Name("f".to_string())); - token_str.push(ch); - state = TokenizerState::F; - } - 'i' => - { - last_index = index as i32; - token = Some(Token::Name("i".to_string())); - token_str.push(ch); - state = TokenizerState::I; - } - 'g' => - { - last_index = index as i32; - token = Some(Token::Name("g".to_string())); - token_str.push(ch); - state = TokenizerState::G; - } - 'l' => - { - last_index = index as i32; - token = Some(Token::Name("l".to_string())); - token_str.push(ch); - state = TokenizerState::L; - } - 'n' => - { - last_index = index as i32; - token = Some(Token::Name("n".to_string())); - token_str.push(ch); - state = TokenizerState::N; - } - 'o' => - { - last_index = index as i32; - token = Some(Token::Name("o".to_string())); - token_str.push(ch); - state = TokenizerState::O; - } - 'r' => - { - last_index = index as i32; - token = Some(Token::Name("r".to_string())); - token_str.push(ch); - state = TokenizerState::R; - } - 't' => - { - last_index = index as i32; - token = Some(Token::Name("t".to_string())); - token_str.push(ch); - state = TokenizerState::T; - } - 'u' => - { - last_index = index as i32; - token = Some(Token::Name("u".to_string())); - token_str.push(ch); - state = TokenizerState::U; - } - 'w' => - { - last_index = index as i32; - token = Some(Token::Name("w".to_string())); - token_str.push(ch); - state = TokenizerState::W; - } - ',' => - { - last_index = index as i32; - token = Some(Token::Comma); - state = TokenizerState::Comma; - } - '=' => - { - last_index = index as i32; - token = Some(Token::Equals); - state = TokenizerState::Equals; - } - '(' => - { - last_index = index as i32; - token = Some(Token::RoundOpen); - state = TokenizerState::RoundOpen; - } - ')' => - { - last_index = index as i32; - token = Some(Token::RoundClosed); - state = TokenizerState::RoundClosed; - } - '.' => - { - last_index = index as i32; - token = Some(Token::Dot); - state = TokenizerState::Dot; - } - ':' => - { - last_index = index as i32; - token = Some(Token::Colon); - state = TokenizerState::Colon; - } - '{' => - { - last_index = index as i32; - token = Some(Token::CurlyOpen); - state = TokenizerState::CurlyOpen; - } - '}' => - { - last_index = index as i32; - token = Some(Token::CurlyClosed); - state = TokenizerState::CurlyClosed; - } - '[' => - { - last_index = index as i32; - token = Some(Token::SquareOpen); - state = TokenizerState::SquareOpen; - } - ']' => - { - last_index = index as i32; - token = Some(Token::SquareClosed); - state = TokenizerState::SquareClosed; - } - '+' => - { - last_index = index as i32; - token = Some(Token::Plus); - state = TokenizerState::Plus; - } - '~' => - { - last_index = index as i32; - token = Some(Token::Tilde); - state = TokenizerState::Tilde; - } - '>' => - { - last_index = index as i32; - token = Some(Token::Gt); - state = TokenizerState::Gt; - } - '<' => - { - last_index = index as i32; - token = Some(Token::Lt); - state = TokenizerState::Lt; - } - '#' => - { - last_index = index as i32; - token = Some(Token::Hash); - state = TokenizerState::Hash; - } - '|' => - { - last_index = index as i32; - token = Some(Token::Pipe); - state = TokenizerState::Pipe; - } - '&' => - { - last_index = index as i32; - token = Some(Token::Ampersand); - state = TokenizerState::Ampersand; - } - '%' => - { - last_index = index as i32; - token = Some(Token::Percent); - state = TokenizerState::Percent; - } - '*' => - { - last_index = index as i32; - token = Some(Token::Star); - state = TokenizerState::Star; - } - '/' => - { - last_index = index as i32; - token = Some(Token::Slash); - state = TokenizerState::Slash; - } - ';' => - { - last_index = index as i32; - token = Some(Token::Semicolon); - state = TokenizerState::Semicolon; - } - '^' => - { - last_index = index as i32; - token = Some(Token::Caret); - state = TokenizerState::Caret; - } - '0' => - { - last_index = index as i32; - token = Some(Token::IntLiteral("0".to_string())); - token_str.push(ch); - state = TokenizerState::Zero; - } + '-' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Minus), TokenizerState::Minus), + 'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("a".to_string())), TokenizerState::A, &mut token_str, ch), + 'b' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("b".to_string())), TokenizerState::B, &mut token_str, ch), + 'd' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("d".to_string())), TokenizerState::D, &mut token_str, ch), + 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("e".to_string())), TokenizerState::E, &mut token_str, ch), + 'f' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("f".to_string())), TokenizerState::F, &mut token_str, ch), + 'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("i".to_string())), TokenizerState::I, &mut token_str, ch), + 'g' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("g".to_string())), TokenizerState::G, &mut token_str, ch), + 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("l".to_string())), TokenizerState::L, &mut token_str, ch), + 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("n".to_string())), TokenizerState::N, &mut token_str, ch), + 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("o".to_string())), TokenizerState::O, &mut token_str, ch), + 'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("r".to_string())), TokenizerState::R, &mut token_str, ch), + 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("t".to_string())), TokenizerState::T, &mut token_str, ch), + 'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("u".to_string())), TokenizerState::U, &mut token_str, ch), + 'w' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("w".to_string())), TokenizerState::W, &mut token_str, ch), + ',' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Comma), TokenizerState::Comma), + '=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Equals), TokenizerState::Equals), + '(' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::RoundOpen), TokenizerState::RoundOpen), + ')' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::RoundClosed), TokenizerState::RoundClosed), + '.' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Dot), TokenizerState::Dot), + ':' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Colon), TokenizerState::Colon), + '{' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::CurlyOpen), TokenizerState::CurlyOpen), + '}' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::CurlyClosed), TokenizerState::CurlyClosed), + '[' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::SquareOpen), TokenizerState::SquareOpen), + ']' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::SquareClosed), TokenizerState::SquareClosed), + '+' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Plus), TokenizerState::Plus), + '~' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Tilde), TokenizerState::Tilde), + '>' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Gt), TokenizerState::Gt), + '<' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Lt), TokenizerState::Lt), + '#' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Hash), TokenizerState::Hash), + '|' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Pipe), TokenizerState::Pipe), + '&' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Ampersand), TokenizerState::Ampersand), + '%' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Percent), TokenizerState::Percent), + '*' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Star), TokenizerState::Star), + '/' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Slash), TokenizerState::Slash), + ';' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Semicolon), TokenizerState::Semicolon), + '^' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Caret), TokenizerState::Caret), + '0' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::IntLiteral("0".to_string())), TokenizerState::Zero, &mut token_str, ch), '"' => { token = None; @@ -348,17 +203,11 @@ fn tokenize(file_content: &String) -> Result, &'static str> if ch.is_whitespace() { } else if ch.is_ascii_alphabetic() || ch == '_' { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; + tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name(token_str.clone())), TokenizerState::Name, &mut token_str, ch); } else if ch.is_numeric() && ch.is_ascii() { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::IntLiteral(token_str.clone())); - state = TokenizerState::Number; + tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::IntLiteral(token_str.clone())), TokenizerState::Number, &mut token_str, ch); } else { @@ -375,12 +224,7 @@ fn tokenize(file_content: &String) -> Result, &'static str> { state = TokenizerState::QuoteBackslash; } - '"' => - { - last_index = index as i32; - token = Some(Token::StringLiteral(token_str.clone())); - state = TokenizerState::String; - } + '"' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::StringLiteral(token_str.clone())), TokenizerState::String), _ => { token_str.push(ch); @@ -456,12 +300,7 @@ fn tokenize(file_content: &String) -> Result, &'static str> { state = TokenizerState::QuoteBackslash; } - '"' => - { - last_index = index as i32; - token = Some(Token::StringLiteral(token_str.clone())); - state = TokenizerState::String; - } + '"' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::StringLiteral(token_str.clone())), TokenizerState::String), _ => { if !ch.is_whitespace() @@ -579,17 +418,8 @@ fn tokenize(file_content: &String) -> Result, &'static str> } TokenizerState::String => { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - token = None; - tokens.push(Token::StringLiteral(token_str.clone())); - token_str.clear(); - state = TokenizerState::Start; + let content = token_str.clone(); + tokenize_backtrack_custom_token(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, Token::StringLiteral(content))?; } TokenizerState::Name => { @@ -757,320 +587,80 @@ fn tokenize(file_content: &String) -> Result, &'static str> TokenizerState::GtEquals | TokenizerState::LtEquals | TokenizerState::SquareOpen | TokenizerState::SquareClosed | TokenizerState::Pipe | TokenizerState::Ampersand | TokenizerState::Percent | TokenizerState::Star | TokenizerState::Semicolon | - TokenizerState::Caret => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.clone().unwrap()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } + TokenizerState::Caret | TokenizerState::DotDotDot | TokenizerState::GtGt | + TokenizerState::LtLt | TokenizerState::SlashSlash => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?, TokenizerState::Tilde => { match ch { - '=' => - { - last_index = index as i32; - token = Some(Token::TildeEquals); - state = TokenizerState::TildeEquals; - } - _ => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.clone().unwrap()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } + '=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::TildeEquals), TokenizerState::TildeEquals), + _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?, } } TokenizerState::Gt => { match ch { - '>' => - { - last_index = index as i32; - token = Some(Token::GtGt); - state = TokenizerState::GtGt; - } - '=' => - { - last_index = index as i32; - token = Some(Token::GtEquals); - state = TokenizerState::GtEquals; - } - _ => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.clone().unwrap()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } + '>' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::GtGt), TokenizerState::GtGt), + '=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::GtEquals), TokenizerState::GtEquals), + _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?, } } TokenizerState::Lt => { match ch { - '>' => - { - last_index = index as i32; - token = Some(Token::LtLt); - state = TokenizerState::LtLt; - } - '=' => - { - last_index = index as i32; - token = Some(Token::LtEquals); - state = TokenizerState::LtEquals; - } - _ => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.clone().unwrap()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } + '>' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::LtLt), TokenizerState::LtLt), + '=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::LtEquals), TokenizerState::LtEquals), + _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?, } } - TokenizerState::GtGt => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - token = None; - token_str.clear(); - state = TokenizerState::Start; - tokens.push(Token::GtGt); - } - TokenizerState::LtLt => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - token = None; - token_str.clear(); - state = TokenizerState::Start; - tokens.push(Token::LtLt); - } TokenizerState::Slash => { match ch { - '/' => - { - last_index = index as i32; - token = Some(Token::SlashSlash); - state = TokenizerState::SlashSlash; - } - _ => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.clone().unwrap()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } + '/' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::SlashSlash), TokenizerState::SlashSlash), + _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?, } } - TokenizerState::SlashSlash => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - token = None; - token_str.clear(); - state = TokenizerState::Start; - tokens.push(Token::SlashSlash); - } TokenizerState::Dot => { match ch { - '.' => - { - last_index = index as i32; - token = Some(Token::DotDot); - state = TokenizerState::DotDot; - } - _ => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.clone().unwrap()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } + '.' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::DotDot), TokenizerState::DotDot), + _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?, } } TokenizerState::DotDot => { match ch { - '.' => - { - last_index = index as i32; - token = Some(Token::DotDotDot); - state = TokenizerState::DotDotDot; - } - _ => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.clone().unwrap()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } + '.' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::DotDotDot), TokenizerState::DotDotDot), + _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?, } } - TokenizerState::DotDotDot => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - token = None; - token_str.clear(); - state = TokenizerState::Start; - tokens.push(Token::And); - } TokenizerState::Colon => { match ch { - ':' => - { - last_index = index as i32; - token = Some(Token::ColonColon); - state = TokenizerState::ColonColon; - } - _ => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.clone().unwrap()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } + ':' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::ColonColon), TokenizerState::ColonColon), + _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?, } } TokenizerState::Equals => { match ch { - '=' => - { - last_index = index as i32; - token = Some(Token::EqualsEquals); - state = TokenizerState::EqualsEquals; - } - _ => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.clone().unwrap()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } + '=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::EqualsEquals), TokenizerState::EqualsEquals), + _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?, } } TokenizerState::Minus => { match ch { - '-' => - { - last_index = index as i32; - token = None; - state = TokenizerState::SmallCommentStart; - } - _ => - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.clone().unwrap()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } + '-' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, None, TokenizerState::SmallCommentStart), + _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?, } } TokenizerState::SmallCommentStart => @@ -1158,13 +748,7 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'n' => - { - last_index = index as i32; - token = Some(Token::Name("an".to_string())); - token_str.push(ch); - state = TokenizerState::An; - } + 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("an".to_string())), TokenizerState::An, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -1195,37 +779,8 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'd' => - { - last_index = index as i32; - token = Some(Token::Name("and".to_string())); - token_str.push(ch); - state = TokenizerState::And; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'd' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("and".to_string())), TokenizerState::And, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::And => @@ -1256,13 +811,7 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'h' => - { - last_index = index as i32; - token = Some(Token::Name("wh".to_string())); - token_str.push(ch); - state = TokenizerState::Wh; - } + 'h' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("wh".to_string())), TokenizerState::Wh, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -1293,111 +842,24 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'i' => - { - last_index = index as i32; - token = Some(Token::Name("whi".to_string())); - token_str.push(ch); - state = TokenizerState::Whi; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("whi".to_string())), TokenizerState::Whi, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Whi => { match ch { - 'l' => - { - last_index = index as i32; - token = Some(Token::Name("whil".to_string())); - token_str.push(ch); - state = TokenizerState::Whil; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("whil".to_string())), TokenizerState::Whil, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Whil => { match ch { - 'e' => - { - last_index = index as i32; - token = Some(Token::Name("while".to_string())); - token_str.push(ch); - state = TokenizerState::While; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("while".to_string())), TokenizerState::While, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::While => @@ -1428,13 +890,7 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'r' => - { - last_index = index as i32; - token = Some(Token::Name("br".to_string())); - token_str.push(ch); - state = TokenizerState::Br; - } + 'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("br".to_string())), TokenizerState::Br, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -1465,111 +921,24 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'e' => - { - last_index = index as i32; - token = Some(Token::Name("bre".to_string())); - token_str.push(ch); - state = TokenizerState::Bre; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("bre".to_string())), TokenizerState::Bre, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Bre => { match ch { - 'a' => - { - last_index = index as i32; - token = Some(Token::Name("brea".to_string())); - token_str.push(ch); - state = TokenizerState::Brea; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("brea".to_string())), TokenizerState::Brea, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Brea => { match ch { - 'k' => - { - last_index = index as i32; - token = Some(Token::Name("break".to_string())); - token_str.push(ch); - state = TokenizerState::Break; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'k' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("break".to_string())), TokenizerState::Break, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Break => @@ -1600,111 +969,24 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'o' => - { - last_index = index as i32; - token = Some(Token::Name("go".to_string())); - token_str.push(ch); - state = TokenizerState::Go; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.clone().unwrap()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("go".to_string())), TokenizerState::Go, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Go => { match ch { - 't' => - { - last_index = index as i32; - token = Some(Token::Name("got".to_string())); - token_str.push(ch); - state = TokenizerState::Got; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("got".to_string())), TokenizerState::Got, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Got => { match ch { - 'o' => - { - last_index = index as i32; - token = Some(Token::Name("goto".to_string())); - token_str.push(ch); - state = TokenizerState::Goto; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("goto".to_string())), TokenizerState::Goto, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Goto => @@ -1735,13 +1017,7 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'e' => - { - last_index = index as i32; - token = Some(Token::Name("re".to_string())); - token_str.push(ch); - state = TokenizerState::Re; - } + 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("re".to_string())), TokenizerState::Re, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -1772,155 +1048,33 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 't' => - { - last_index = index as i32; - token = Some(Token::Name("ret".to_string())); - token_str.push(ch); - state = TokenizerState::Ret; - } - 'p' => - { - last_index = index as i32; - token = Some(Token::Name("rep".to_string())); - token_str.push(ch); - state = TokenizerState::Rep; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("ret".to_string())), TokenizerState::Ret, &mut token_str, ch), + 'p' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("rep".to_string())), TokenizerState::Rep, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Ret => { match ch { - 'u' => - { - last_index = index as i32; - token = Some(Token::Name("retu".to_string())); - token_str.push(ch); - state = TokenizerState::Retu; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("retu".to_string())), TokenizerState::Retu, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Retu => { match ch { - 'r' => - { - last_index = index as i32; - token = Some(Token::Name("retur".to_string())); - token_str.push(ch); - state = TokenizerState::Retur; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("retur".to_string())), TokenizerState::Retur, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Retur => { match ch { - 'n' => - { - last_index = index as i32; - token = Some(Token::Name("return".to_string())); - token_str.push(ch); - state = TokenizerState::Return; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("return".to_string())), TokenizerState::Return, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Return => @@ -1951,111 +1105,24 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'e' => - { - last_index = index as i32; - token = Some(Token::Name("repe".to_string())); - token_str.push(ch); - state = TokenizerState::Repe; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("repe".to_string())), TokenizerState::Repe, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Repe => { match ch { - 'a' => - { - last_index = index as i32; - token = Some(Token::Name("repea".to_string())); - token_str.push(ch); - state = TokenizerState::Repea; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("repea".to_string())), TokenizerState::Repea, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Repea => { match ch { - 't' => - { - last_index = index as i32; - token = Some(Token::Name("repeat".to_string())); - token_str.push(ch); - state = TokenizerState::Repeat; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("repeat".to_string())), TokenizerState::Repeat, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Repeat => @@ -2086,20 +1153,8 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'i' => - { - last_index = index as i32; - token = Some(Token::Name("ni".to_string())); - token_str.push(ch); - state = TokenizerState::Ni; - } - 'o' => - { - last_index = index as i32; - token = Some(Token::Name("no".to_string())); - token_str.push(ch); - state = TokenizerState::No; - } + 'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("ni".to_string())), TokenizerState::Ni, &mut token_str, ch), + 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("no".to_string())), TokenizerState::No, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -2130,37 +1185,8 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 't' => - { - last_index = index as i32; - token = Some(Token::Name("not".to_string())); - token_str.push(ch); - state = TokenizerState::Not; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("not".to_string())), TokenizerState::Not, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Not => @@ -2191,37 +1217,8 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'l' => - { - last_index = index as i32; - token = Some(Token::Name("nil".to_string())); - token_str.push(ch); - state = TokenizerState::Nil; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("nil".to_string())), TokenizerState::Nil, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Nil => @@ -2252,20 +1249,8 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'h' => - { - last_index = index as i32; - token = Some(Token::Name("th".to_string())); - token_str.push(ch); - state = TokenizerState::Th; - } - 'r' => - { - last_index = index as i32; - token = Some(Token::Name("tr".to_string())); - token_str.push(ch); - state = TokenizerState::Tr; - } + 'h' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("th".to_string())), TokenizerState::Th, &mut token_str, ch), + 'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("tr".to_string())), TokenizerState::Tr, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -2296,74 +1281,16 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'e' => - { - last_index = index as i32; - token = Some(Token::Name("the".to_string())); - token_str.push(ch); - state = TokenizerState::The; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("the".to_string())), TokenizerState::The, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::The => { match ch { - 'n' => - { - last_index = index as i32; - token = Some(Token::Name("then".to_string())); - token_str.push(ch); - state = TokenizerState::Then; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("then".to_string())), TokenizerState::Then, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Then => @@ -2394,74 +1321,16 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'u' => - { - last_index = index as i32; - token = Some(Token::Name("tru".to_string())); - token_str.push(ch); - state = TokenizerState::Tru; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("tru".to_string())), TokenizerState::Tru, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Tru => { match ch { - 'e' => - { - last_index = index as i32; - token = Some(Token::Name("true".to_string())); - token_str.push(ch); - state = TokenizerState::True; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("true".to_string())), TokenizerState::True, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::True => @@ -2492,20 +1361,8 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'l' => - { - last_index = index as i32; - token = Some(Token::Name("el".to_string())); - token_str.push(ch); - state = TokenizerState::El; - } - 'n' => - { - last_index = index as i32; - token = Some(Token::Name("en".to_string())); - token_str.push(ch); - state = TokenizerState::En; - } + 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("el".to_string())), TokenizerState::El, &mut token_str, ch), + 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("en".to_string())), TokenizerState::En, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -2536,37 +1393,8 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'd' => - { - last_index = index as i32; - token = Some(Token::Name("end".to_string())); - token_str.push(ch); - state = TokenizerState::End; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'd' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("end".to_string())), TokenizerState::End, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::End => @@ -2597,87 +1425,23 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 's' => - { - last_index = index as i32; - token = Some(Token::Name("els".to_string())); - token_str.push(ch); - state = TokenizerState::Els; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 's' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("els".to_string())), TokenizerState::Els, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Els => { match ch { - 'e' => - { - last_index = index as i32; - token = Some(Token::Name("else".to_string())); - token_str.push(ch); - state = TokenizerState::Else; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("else".to_string())), TokenizerState::Else, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Else => { match ch { - 'i' => - { - last_index = index as i32; - token = Some(Token::Name("elsei".to_string())); - token_str.push(ch); - state = TokenizerState::Elsei; - } + 'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("elsei".to_string())), TokenizerState::Elsei, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -2708,37 +1472,8 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'f' => - { - last_index = index as i32; - token = Some(Token::Name("elseif".to_string())); - token_str.push(ch); - state = TokenizerState::Elseif; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'f' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("elseif".to_string())), TokenizerState::Elseif, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Elseif => @@ -2769,13 +1504,7 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'r' => - { - last_index = index as i32; - token = Some(Token::Name("or".to_string())); - token_str.push(ch); - state = TokenizerState::Or; - } + 'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("or".to_string())), TokenizerState::Or, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -2830,13 +1559,7 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'o' => - { - last_index = index as i32; - token = Some(Token::Name("do".to_string())); - token_str.push(ch); - state = TokenizerState::Do; - } + 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("do".to_string())), TokenizerState::Do, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -2891,20 +1614,8 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'f' => - { - last_index = index as i32; - token = Some(Token::Name("if".to_string())); - token_str.push(ch); - state = TokenizerState::If; - } - 'n' => - { - last_index = index as i32; - token = Some(Token::Name("in".to_string())); - token_str.push(ch); - state = TokenizerState::In; - } + 'f' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("if".to_string())), TokenizerState::If, &mut token_str, ch), + 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("in".to_string())), TokenizerState::In, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -2983,27 +1694,9 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'a' => - { - last_index = index as i32; - token = Some(Token::Name("fa".to_string())); - token_str.push(ch); - state = TokenizerState::Fa; - } - 'o' => - { - last_index = index as i32; - token = Some(Token::Name("fo".to_string())); - token_str.push(ch); - state = TokenizerState::Fo; - } - 'u' => - { - last_index = index as i32; - token = Some(Token::Name("fu".to_string())); - token_str.push(ch); - state = TokenizerState::Fu; - } + 'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fa".to_string())), TokenizerState::Fa, &mut token_str, ch), + 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fo".to_string())), TokenizerState::Fo, &mut token_str, ch), + 'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fu".to_string())), TokenizerState::Fu, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -3034,222 +1727,48 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'n' => - { - last_index = index as i32; - token = Some(Token::Name("fun".to_string())); - token_str.push(ch); - state = TokenizerState::Fun; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fun".to_string())), TokenizerState::Fun, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Fun => { match ch { - 'c' => - { - last_index = index as i32; - token = Some(Token::Name("func".to_string())); - token_str.push(ch); - state = TokenizerState::Func; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'c' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("func".to_string())), TokenizerState::Func, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Func => { match ch { - 't' => - { - last_index = index as i32; - token = Some(Token::Name("funct".to_string())); - token_str.push(ch); - state = TokenizerState::Funct; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("funct".to_string())), TokenizerState::Funct, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Funct => { match ch { - 'i' => - { - last_index = index as i32; - token = Some(Token::Name("functi".to_string())); - token_str.push(ch); - state = TokenizerState::Functi; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("functi".to_string())), TokenizerState::Functi, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Functi => { match ch { - 'o' => - { - last_index = index as i32; - token = Some(Token::Name("functio".to_string())); - token_str.push(ch); - state = TokenizerState::Functio; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("functio".to_string())), TokenizerState::Functio, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Functio => { match ch { - 'n' => - { - last_index = index as i32; - token = Some(Token::Name("function".to_string())); - token_str.push(ch); - state = TokenizerState::Function; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("function".to_string())), TokenizerState::Function, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Function => @@ -3280,111 +1799,24 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'l' => - { - last_index = index as i32; - token = Some(Token::Name("fal".to_string())); - token_str.push(ch); - state = TokenizerState::Fal; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fal".to_string())), TokenizerState::Fal, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Fal => { match ch { - 's' => - { - last_index = index as i32; - token = Some(Token::Name("fals".to_string())); - token_str.push(ch); - state = TokenizerState::Fals; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 's' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fals".to_string())), TokenizerState::Fals, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Fals => { match ch { - 'e' => - { - last_index = index as i32; - token = Some(Token::Name("false".to_string())); - token_str.push(ch); - state = TokenizerState::False; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("false".to_string())), TokenizerState::False, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::False => @@ -3415,37 +1847,8 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'r' => - { - last_index = index as i32; - token = Some(Token::Name("for".to_string())); - token_str.push(ch); - state = TokenizerState::For; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("for".to_string())), TokenizerState::For, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::For => @@ -3476,13 +1879,7 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'o' => - { - last_index = index as i32; - token = Some(Token::Name("lo".to_string())); - token_str.push(ch); - state = TokenizerState::Lo; - } + 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("lo".to_string())), TokenizerState::Lo, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -3513,111 +1910,24 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'c' => - { - last_index = index as i32; - token = Some(Token::Name("loc".to_string())); - token_str.push(ch); - state = TokenizerState::Loc; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'c' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("loc".to_string())), TokenizerState::Loc, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Loc => { match ch { - 'a' => - { - last_index = index as i32; - token = Some(Token::Name("loca".to_string())); - token_str.push(ch); - state = TokenizerState::Loca; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("loca".to_string())), TokenizerState::Loca, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Loca => { match ch { - 'l' => - { - last_index = index as i32; - token = Some(Token::Name("local".to_string())); - token_str.push(ch); - state = TokenizerState::Local; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("local".to_string())), TokenizerState::Local, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Local => @@ -3648,13 +1958,7 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 'n' => - { - last_index = index as i32; - token = Some(Token::Name("un".to_string())); - token_str.push(ch); - state = TokenizerState::Un; - } + 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("un".to_string())), TokenizerState::Un, &mut token_str, ch), _ => { if ch.is_ascii_alphanumeric() || ch == '_' @@ -3685,111 +1989,24 @@ fn tokenize(file_content: &String) -> Result, &'static str> { match ch { - 't' => - { - last_index = index as i32; - token = Some(Token::Name("unt".to_string())); - token_str.push(ch); - state = TokenizerState::Unt; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("unt".to_string())), TokenizerState::Unt, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Unt => { match ch { - 'i' => - { - last_index = index as i32; - token = Some(Token::Name("unti".to_string())); - token_str.push(ch); - state = TokenizerState::Unti; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("unti".to_string())), TokenizerState::Unti, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Unti => { match ch { - 'l' => - { - last_index = index as i32; - token = Some(Token::Name("until".to_string())); - token_str.push(ch); - state = TokenizerState::Until; - } - _ => - { - if ch.is_ascii_alphanumeric() || ch == '_' - { - last_index = index as i32; - token_str.push(ch); - token = Some(Token::Name(token_str.clone())); - state = TokenizerState::Name; - } - else - { - if last_index == -1 || token.is_none() - { - println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens); - return Err("Lexerr"); - } - index = last_index as usize; - last_index = -1; - tokens.push(token.unwrap().clone()); - token = None; - token_str.clear(); - state = TokenizerState::Start; - } - } + 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("until".to_string())), TokenizerState::Until, &mut token_str, ch), + _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?, } } TokenizerState::Until => @@ -3816,7 +2033,7 @@ fn tokenize(file_content: &String) -> Result, &'static str> tokens.push(Token::Until); } } - _ => todo!("State {:?}", state) + _ => todo!("State: {:?}", state), } index += 1; }