0x4261756D/luaaaaah
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

deeduplicate code

Browse Source
This commit is contained in:
0x4261756D 2023-06-06 23:56:38 +02:00
parent a149609f03
commit 317878dd3b
1 changed files with 234 additions and 2017 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2251
src/main.rs
View File

@ -69,6 +69,66 @@ enum TokenizerState
Function, Function,
} }
fn tokenize_terminal_no_str(last_index: &mut i32, index: usize, token: &mut Option<Token>, state: &mut TokenizerState, new_token: Option<Token>, new_state: TokenizerState)
{
*last_index = index as i32;
*token = new_token;
*state = new_state;
}
fn tokenize_terminal(last_index: &mut i32, index: usize, token: &mut Option<Token>, state: &mut TokenizerState, new_token: Option<Token>, new_state: TokenizerState, token_str: &mut String, ch: char)
{
tokenize_terminal_no_str(last_index, index, token, state, new_token, new_state);
token_str.push(ch);
}
fn tokenize_backtrack(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState) -> Result<(), &'static str>
{
return tokenize_backtrack_custom_token(last_index, index, tokens, token, token_str, state, token.clone().unwrap());
}
fn tokenize_backtrack_name(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState) -> Result<(), &'static str>
{
if *last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
*index = *last_index as usize;
*last_index = -1;
tokens.push(Token::Name(token_str.clone()));
*token = None;
token_str.clear();
*state = TokenizerState::Start;
return Ok(());
}
fn tokenize_backtrack_custom_token(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState, new_token: Token) -> Result<(), &'static str>
{
if *last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
*index = *last_index as usize;
*last_index = -1;
tokens.push(new_token);
*token = None;
token_str.clear();
*state = TokenizerState::Start;
return Ok(());
}
fn tokenize_alphanumeric_nonterminal(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState, ch: char) -> Result<(), &'static str>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
*last_index = *index as i32;
token_str.push(ch);
*state = TokenizerState::Name;
}
else
{
tokenize_backtrack_name(last_index, index, tokens, token, token_str, state)?;
}
return Ok(());
}
fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str> fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
let mut tokens: Vec<Token> = Vec::new(); let mut tokens: Vec<Token> = Vec::new();
@ -90,249 +150,44 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'-' => '-' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Minus), TokenizerState::Minus),
{ 'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("a".to_string())), TokenizerState::A, &mut token_str, ch),
last_index = index as i32; 'b' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("b".to_string())), TokenizerState::B, &mut token_str, ch),
token = Some(Token::Minus); 'd' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("d".to_string())), TokenizerState::D, &mut token_str, ch),
state = TokenizerState::Minus; 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("e".to_string())), TokenizerState::E, &mut token_str, ch),
} 'f' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("f".to_string())), TokenizerState::F, &mut token_str, ch),
'a' => 'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("i".to_string())), TokenizerState::I, &mut token_str, ch),
{ 'g' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("g".to_string())), TokenizerState::G, &mut token_str, ch),
last_index = index as i32; 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("l".to_string())), TokenizerState::L, &mut token_str, ch),
token = Some(Token::Name("a".to_string())); 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("n".to_string())), TokenizerState::N, &mut token_str, ch),
token_str.push(ch); 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("o".to_string())), TokenizerState::O, &mut token_str, ch),
state = TokenizerState::A; 'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("r".to_string())), TokenizerState::R, &mut token_str, ch),
} 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("t".to_string())), TokenizerState::T, &mut token_str, ch),
'b' => 'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("u".to_string())), TokenizerState::U, &mut token_str, ch),
{ 'w' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("w".to_string())), TokenizerState::W, &mut token_str, ch),
last_index = index as i32; ',' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Comma), TokenizerState::Comma),
token = Some(Token::Name("b".to_string())); '=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Equals), TokenizerState::Equals),
token_str.push(ch); '(' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::RoundOpen), TokenizerState::RoundOpen),
state = TokenizerState::B; ')' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::RoundClosed), TokenizerState::RoundClosed),
} '.' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Dot), TokenizerState::Dot),
'd' => ':' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Colon), TokenizerState::Colon),
{ '{' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::CurlyOpen), TokenizerState::CurlyOpen),
last_index = index as i32; '}' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::CurlyClosed), TokenizerState::CurlyClosed),
token = Some(Token::Name("d".to_string())); '[' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::SquareOpen), TokenizerState::SquareOpen),
token_str.push(ch); ']' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::SquareClosed), TokenizerState::SquareClosed),
state = TokenizerState::D; '+' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Plus), TokenizerState::Plus),
} '~' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Tilde), TokenizerState::Tilde),
'e' => '>' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Gt), TokenizerState::Gt),
{ '<' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Lt), TokenizerState::Lt),
last_index = index as i32; '#' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Hash), TokenizerState::Hash),
token = Some(Token::Name("e".to_string())); '|' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Pipe), TokenizerState::Pipe),
token_str.push(ch); '&' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Ampersand), TokenizerState::Ampersand),
state = TokenizerState::E; '%' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Percent), TokenizerState::Percent),
} '*' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Star), TokenizerState::Star),
'f' => '/' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Slash), TokenizerState::Slash),
{ ';' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Semicolon), TokenizerState::Semicolon),
last_index = index as i32; '^' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Caret), TokenizerState::Caret),
token = Some(Token::Name("f".to_string())); '0' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::IntLiteral("0".to_string())), TokenizerState::Zero, &mut token_str, ch),
token_str.push(ch);
state = TokenizerState::F;
}
'i' =>
{
last_index = index as i32;
token = Some(Token::Name("i".to_string()));
token_str.push(ch);
state = TokenizerState::I;
}
'g' =>
{
last_index = index as i32;
token = Some(Token::Name("g".to_string()));
token_str.push(ch);
state = TokenizerState::G;
}
'l' =>
{
last_index = index as i32;
token = Some(Token::Name("l".to_string()));
token_str.push(ch);
state = TokenizerState::L;
}
'n' =>
{
last_index = index as i32;
token = Some(Token::Name("n".to_string()));
token_str.push(ch);
state = TokenizerState::N;
}
'o' =>
{
last_index = index as i32;
token = Some(Token::Name("o".to_string()));
token_str.push(ch);
state = TokenizerState::O;
}
'r' =>
{
last_index = index as i32;
token = Some(Token::Name("r".to_string()));
token_str.push(ch);
state = TokenizerState::R;
}
't' =>
{
last_index = index as i32;
token = Some(Token::Name("t".to_string()));
token_str.push(ch);
state = TokenizerState::T;
}
'u' =>
{
last_index = index as i32;
token = Some(Token::Name("u".to_string()));
token_str.push(ch);
state = TokenizerState::U;
}
'w' =>
{
last_index = index as i32;
token = Some(Token::Name("w".to_string()));
token_str.push(ch);
state = TokenizerState::W;
}
',' =>
{
last_index = index as i32;
token = Some(Token::Comma);
state = TokenizerState::Comma;
}
'=' =>
{
last_index = index as i32;
token = Some(Token::Equals);
state = TokenizerState::Equals;
}
'(' =>
{
last_index = index as i32;
token = Some(Token::RoundOpen);
state = TokenizerState::RoundOpen;
}
')' =>
{
last_index = index as i32;
token = Some(Token::RoundClosed);
state = TokenizerState::RoundClosed;
}
'.' =>
{
last_index = index as i32;
token = Some(Token::Dot);
state = TokenizerState::Dot;
}
':' =>
{
last_index = index as i32;
token = Some(Token::Colon);
state = TokenizerState::Colon;
}
'{' =>
{
last_index = index as i32;
token = Some(Token::CurlyOpen);
state = TokenizerState::CurlyOpen;
}
'}' =>
{
last_index = index as i32;
token = Some(Token::CurlyClosed);
state = TokenizerState::CurlyClosed;
}
'[' =>
{
last_index = index as i32;
token = Some(Token::SquareOpen);
state = TokenizerState::SquareOpen;
}
']' =>
{
last_index = index as i32;
token = Some(Token::SquareClosed);
state = TokenizerState::SquareClosed;
}
'+' =>
{
last_index = index as i32;
token = Some(Token::Plus);
state = TokenizerState::Plus;
}
'~' =>
{
last_index = index as i32;
token = Some(Token::Tilde);
state = TokenizerState::Tilde;
}
'>' =>
{
last_index = index as i32;
token = Some(Token::Gt);
state = TokenizerState::Gt;
}
'<' =>
{
last_index = index as i32;
token = Some(Token::Lt);
state = TokenizerState::Lt;
}
'#' =>
{
last_index = index as i32;
token = Some(Token::Hash);
state = TokenizerState::Hash;
}
'|' =>
{
last_index = index as i32;
token = Some(Token::Pipe);
state = TokenizerState::Pipe;
}
'&' =>
{
last_index = index as i32;
token = Some(Token::Ampersand);
state = TokenizerState::Ampersand;
}
'%' =>
{
last_index = index as i32;
token = Some(Token::Percent);
state = TokenizerState::Percent;
}
'*' =>
{
last_index = index as i32;
token = Some(Token::Star);
state = TokenizerState::Star;
}
'/' =>
{
last_index = index as i32;
token = Some(Token::Slash);
state = TokenizerState::Slash;
}
';' =>
{
last_index = index as i32;
token = Some(Token::Semicolon);
state = TokenizerState::Semicolon;
}
'^' =>
{
last_index = index as i32;
token = Some(Token::Caret);
state = TokenizerState::Caret;
}
'0' =>
{
last_index = index as i32;
token = Some(Token::IntLiteral("0".to_string()));
token_str.push(ch);
state = TokenizerState::Zero;
}
'"' => '"' =>
{ {
token = None; token = None;
@ -348,17 +203,11 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
if ch.is_whitespace() { } if ch.is_whitespace() { }
else if ch.is_ascii_alphabetic() || ch == '_' else if ch.is_ascii_alphabetic() || ch == '_'
{ {
last_index = index as i32; tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name(token_str.clone())), TokenizerState::Name, &mut token_str, ch);
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
} }
else if ch.is_numeric() && ch.is_ascii() else if ch.is_numeric() && ch.is_ascii()
{ {
last_index = index as i32; tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::IntLiteral(token_str.clone())), TokenizerState::Number, &mut token_str, ch);
token_str.push(ch);
token = Some(Token::IntLiteral(token_str.clone()));
state = TokenizerState::Number;
} }
else else
{ {
@ -375,12 +224,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
state = TokenizerState::QuoteBackslash; state = TokenizerState::QuoteBackslash;
} }
'"' => '"' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::StringLiteral(token_str.clone())), TokenizerState::String),
{
last_index = index as i32;
token = Some(Token::StringLiteral(token_str.clone()));
state = TokenizerState::String;
}
_ => _ =>
{ {
token_str.push(ch); token_str.push(ch);
@ -456,12 +300,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
state = TokenizerState::QuoteBackslash; state = TokenizerState::QuoteBackslash;
} }
'"' => '"' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::StringLiteral(token_str.clone())), TokenizerState::String),
{
last_index = index as i32;
token = Some(Token::StringLiteral(token_str.clone()));
state = TokenizerState::String;
}
_ => _ =>
{ {
if !ch.is_whitespace() if !ch.is_whitespace()
@ -579,17 +418,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
} }
TokenizerState::String => TokenizerState::String =>
{ {
if last_index == -1 || token.is_none() let content = token_str.clone();
{ tokenize_backtrack_custom_token(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, Token::StringLiteral(content))?;
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
token = None;
tokens.push(Token::StringLiteral(token_str.clone()));
token_str.clear();
state = TokenizerState::Start;
} }
TokenizerState::Name => TokenizerState::Name =>
{ {
@ -757,320 +587,80 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
TokenizerState::GtEquals | TokenizerState::LtEquals | TokenizerState::SquareOpen | TokenizerState::GtEquals | TokenizerState::LtEquals | TokenizerState::SquareOpen |
TokenizerState::SquareClosed | TokenizerState::Pipe | TokenizerState::Ampersand | TokenizerState::SquareClosed | TokenizerState::Pipe | TokenizerState::Ampersand |
TokenizerState::Percent | TokenizerState::Star | TokenizerState::Semicolon | TokenizerState::Percent | TokenizerState::Star | TokenizerState::Semicolon |
TokenizerState::Caret => TokenizerState::Caret | TokenizerState::DotDotDot | TokenizerState::GtGt |
{ TokenizerState::LtLt | TokenizerState::SlashSlash => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
TokenizerState::Tilde => TokenizerState::Tilde =>
{ {
match ch match ch
{ {
'=' => '=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::TildeEquals), TokenizerState::TildeEquals),
{ _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
last_index = index as i32;
token = Some(Token::TildeEquals);
state = TokenizerState::TildeEquals;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
} }
} }
TokenizerState::Gt => TokenizerState::Gt =>
{ {
match ch match ch
{ {
'>' => '>' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::GtGt), TokenizerState::GtGt),
{ '=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::GtEquals), TokenizerState::GtEquals),
last_index = index as i32; _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
token = Some(Token::GtGt);
state = TokenizerState::GtGt;
}
'=' =>
{
last_index = index as i32;
token = Some(Token::GtEquals);
state = TokenizerState::GtEquals;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
} }
} }
TokenizerState::Lt => TokenizerState::Lt =>
{ {
match ch match ch
{ {
'>' => '>' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::LtLt), TokenizerState::LtLt),
{ '=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::LtEquals), TokenizerState::LtEquals),
last_index = index as i32; _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
token = Some(Token::LtLt);
state = TokenizerState::LtLt;
}
'=' =>
{
last_index = index as i32;
token = Some(Token::LtEquals);
state = TokenizerState::LtEquals;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
} }
} }
TokenizerState::GtGt =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
token = None;
token_str.clear();
state = TokenizerState::Start;
tokens.push(Token::GtGt);
}
TokenizerState::LtLt =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
token = None;
token_str.clear();
state = TokenizerState::Start;
tokens.push(Token::LtLt);
}
TokenizerState::Slash => TokenizerState::Slash =>
{ {
match ch match ch
{ {
'/' => '/' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::SlashSlash), TokenizerState::SlashSlash),
{ _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
last_index = index as i32;
token = Some(Token::SlashSlash);
state = TokenizerState::SlashSlash;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
} }
} }
TokenizerState::SlashSlash =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
token = None;
token_str.clear();
state = TokenizerState::Start;
tokens.push(Token::SlashSlash);
}
TokenizerState::Dot => TokenizerState::Dot =>
{ {
match ch match ch
{ {
'.' => '.' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::DotDot), TokenizerState::DotDot),
{ _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
last_index = index as i32;
token = Some(Token::DotDot);
state = TokenizerState::DotDot;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
} }
} }
TokenizerState::DotDot => TokenizerState::DotDot =>
{ {
match ch match ch
{ {
'.' => '.' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::DotDotDot), TokenizerState::DotDotDot),
{ _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
last_index = index as i32;
token = Some(Token::DotDotDot);
state = TokenizerState::DotDotDot;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
} }
} }
TokenizerState::DotDotDot =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
token = None;
token_str.clear();
state = TokenizerState::Start;
tokens.push(Token::And);
}
TokenizerState::Colon => TokenizerState::Colon =>
{ {
match ch match ch
{ {
':' => ':' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::ColonColon), TokenizerState::ColonColon),
{ _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
last_index = index as i32;
token = Some(Token::ColonColon);
state = TokenizerState::ColonColon;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
} }
} }
TokenizerState::Equals => TokenizerState::Equals =>
{ {
match ch match ch
{ {
'=' => '=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::EqualsEquals), TokenizerState::EqualsEquals),
{ _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
last_index = index as i32;
token = Some(Token::EqualsEquals);
state = TokenizerState::EqualsEquals;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
} }
} }
TokenizerState::Minus => TokenizerState::Minus =>
{ {
match ch match ch
{ {
'-' => '-' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, None, TokenizerState::SmallCommentStart),
{ _ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
last_index = index as i32;
token = None;
state = TokenizerState::SmallCommentStart;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
} }
} }
TokenizerState::SmallCommentStart => TokenizerState::SmallCommentStart =>
@ -1158,13 +748,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'n' => 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("an".to_string())), TokenizerState::An, &mut token_str, ch),
{
last_index = index as i32;
token = Some(Token::Name("an".to_string()));
token_str.push(ch);
state = TokenizerState::An;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -1195,37 +779,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'd' => 'd' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("and".to_string())), TokenizerState::And, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("and".to_string()));
token_str.push(ch);
state = TokenizerState::And;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::And => TokenizerState::And =>
@ -1256,13 +811,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'h' => 'h' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("wh".to_string())), TokenizerState::Wh, &mut token_str, ch),
{
last_index = index as i32;
token = Some(Token::Name("wh".to_string()));
token_str.push(ch);
state = TokenizerState::Wh;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -1293,111 +842,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'i' => 'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("whi".to_string())), TokenizerState::Whi, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("whi".to_string()));
token_str.push(ch);
state = TokenizerState::Whi;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Whi => TokenizerState::Whi =>
{ {
match ch match ch
{ {
'l' => 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("whil".to_string())), TokenizerState::Whil, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("whil".to_string()));
token_str.push(ch);
state = TokenizerState::Whil;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Whil => TokenizerState::Whil =>
{ {
match ch match ch
{ {
'e' => 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("while".to_string())), TokenizerState::While, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("while".to_string()));
token_str.push(ch);
state = TokenizerState::While;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::While => TokenizerState::While =>
@ -1428,13 +890,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'r' => 'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("br".to_string())), TokenizerState::Br, &mut token_str, ch),
{
last_index = index as i32;
token = Some(Token::Name("br".to_string()));
token_str.push(ch);
state = TokenizerState::Br;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -1465,111 +921,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'e' => 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("bre".to_string())), TokenizerState::Bre, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("bre".to_string()));
token_str.push(ch);
state = TokenizerState::Bre;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Bre => TokenizerState::Bre =>
{ {
match ch match ch
{ {
'a' => 'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("brea".to_string())), TokenizerState::Brea, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("brea".to_string()));
token_str.push(ch);
state = TokenizerState::Brea;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Brea => TokenizerState::Brea =>
{ {
match ch match ch
{ {
'k' => 'k' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("break".to_string())), TokenizerState::Break, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("break".to_string()));
token_str.push(ch);
state = TokenizerState::Break;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Break => TokenizerState::Break =>
@ -1600,111 +969,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'o' => 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("go".to_string())), TokenizerState::Go, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("go".to_string()));
token_str.push(ch);
state = TokenizerState::Go;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Go => TokenizerState::Go =>
{ {
match ch match ch
{ {
't' => 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("got".to_string())), TokenizerState::Got, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("got".to_string()));
token_str.push(ch);
state = TokenizerState::Got;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Got => TokenizerState::Got =>
{ {
match ch match ch
{ {
'o' => 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("goto".to_string())), TokenizerState::Goto, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("goto".to_string()));
token_str.push(ch);
state = TokenizerState::Goto;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Goto => TokenizerState::Goto =>
@ -1735,13 +1017,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'e' => 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("re".to_string())), TokenizerState::Re, &mut token_str, ch),
{
last_index = index as i32;
token = Some(Token::Name("re".to_string()));
token_str.push(ch);
state = TokenizerState::Re;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -1772,155 +1048,33 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
't' => 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("ret".to_string())), TokenizerState::Ret, &mut token_str, ch),
{ 'p' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("rep".to_string())), TokenizerState::Rep, &mut token_str, ch),
last_index = index as i32; _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
token = Some(Token::Name("ret".to_string()));
token_str.push(ch);
state = TokenizerState::Ret;
}
'p' =>
{
last_index = index as i32;
token = Some(Token::Name("rep".to_string()));
token_str.push(ch);
state = TokenizerState::Rep;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Ret => TokenizerState::Ret =>
{ {
match ch match ch
{ {
'u' => 'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("retu".to_string())), TokenizerState::Retu, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("retu".to_string()));
token_str.push(ch);
state = TokenizerState::Retu;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Retu => TokenizerState::Retu =>
{ {
match ch match ch
{ {
'r' => 'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("retur".to_string())), TokenizerState::Retur, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("retur".to_string()));
token_str.push(ch);
state = TokenizerState::Retur;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Retur => TokenizerState::Retur =>
{ {
match ch match ch
{ {
'n' => 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("return".to_string())), TokenizerState::Return, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("return".to_string()));
token_str.push(ch);
state = TokenizerState::Return;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Return => TokenizerState::Return =>
@ -1951,111 +1105,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'e' => 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("repe".to_string())), TokenizerState::Repe, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("repe".to_string()));
token_str.push(ch);
state = TokenizerState::Repe;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Repe => TokenizerState::Repe =>
{ {
match ch match ch
{ {
'a' => 'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("repea".to_string())), TokenizerState::Repea, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("repea".to_string()));
token_str.push(ch);
state = TokenizerState::Repea;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Repea => TokenizerState::Repea =>
{ {
match ch match ch
{ {
't' => 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("repeat".to_string())), TokenizerState::Repeat, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("repeat".to_string()));
token_str.push(ch);
state = TokenizerState::Repeat;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Repeat => TokenizerState::Repeat =>
@ -2086,20 +1153,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'i' => 'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("ni".to_string())), TokenizerState::Ni, &mut token_str, ch),
{ 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("no".to_string())), TokenizerState::No, &mut token_str, ch),
last_index = index as i32;
token = Some(Token::Name("ni".to_string()));
token_str.push(ch);
state = TokenizerState::Ni;
}
'o' =>
{
last_index = index as i32;
token = Some(Token::Name("no".to_string()));
token_str.push(ch);
state = TokenizerState::No;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -2130,37 +1185,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
't' => 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("not".to_string())), TokenizerState::Not, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("not".to_string()));
token_str.push(ch);
state = TokenizerState::Not;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Not => TokenizerState::Not =>
@ -2191,37 +1217,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'l' => 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("nil".to_string())), TokenizerState::Nil, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("nil".to_string()));
token_str.push(ch);
state = TokenizerState::Nil;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Nil => TokenizerState::Nil =>
@ -2252,20 +1249,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'h' => 'h' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("th".to_string())), TokenizerState::Th, &mut token_str, ch),
{ 'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("tr".to_string())), TokenizerState::Tr, &mut token_str, ch),
last_index = index as i32;
token = Some(Token::Name("th".to_string()));
token_str.push(ch);
state = TokenizerState::Th;
}
'r' =>
{
last_index = index as i32;
token = Some(Token::Name("tr".to_string()));
token_str.push(ch);
state = TokenizerState::Tr;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -2296,74 +1281,16 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'e' => 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("the".to_string())), TokenizerState::The, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("the".to_string()));
token_str.push(ch);
state = TokenizerState::The;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::The => TokenizerState::The =>
{ {
match ch match ch
{ {
'n' => 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("then".to_string())), TokenizerState::Then, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("then".to_string()));
token_str.push(ch);
state = TokenizerState::Then;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Then => TokenizerState::Then =>
@ -2394,74 +1321,16 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'u' => 'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("tru".to_string())), TokenizerState::Tru, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("tru".to_string()));
token_str.push(ch);
state = TokenizerState::Tru;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Tru => TokenizerState::Tru =>
{ {
match ch match ch
{ {
'e' => 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("true".to_string())), TokenizerState::True, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("true".to_string()));
token_str.push(ch);
state = TokenizerState::True;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::True => TokenizerState::True =>
@ -2492,20 +1361,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'l' => 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("el".to_string())), TokenizerState::El, &mut token_str, ch),
{ 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("en".to_string())), TokenizerState::En, &mut token_str, ch),
last_index = index as i32;
token = Some(Token::Name("el".to_string()));
token_str.push(ch);
state = TokenizerState::El;
}
'n' =>
{
last_index = index as i32;
token = Some(Token::Name("en".to_string()));
token_str.push(ch);
state = TokenizerState::En;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -2536,37 +1393,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'd' => 'd' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("end".to_string())), TokenizerState::End, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("end".to_string()));
token_str.push(ch);
state = TokenizerState::End;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::End => TokenizerState::End =>
@ -2597,87 +1425,23 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
's' => 's' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("els".to_string())), TokenizerState::Els, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("els".to_string()));
token_str.push(ch);
state = TokenizerState::Els;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Els => TokenizerState::Els =>
{ {
match ch match ch
{ {
'e' => 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("else".to_string())), TokenizerState::Else, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("else".to_string()));
token_str.push(ch);
state = TokenizerState::Else;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Else => TokenizerState::Else =>
{ {
match ch match ch
{ {
'i' => 'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("elsei".to_string())), TokenizerState::Elsei, &mut token_str, ch),
{
last_index = index as i32;
token = Some(Token::Name("elsei".to_string()));
token_str.push(ch);
state = TokenizerState::Elsei;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -2708,37 +1472,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'f' => 'f' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("elseif".to_string())), TokenizerState::Elseif, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("elseif".to_string()));
token_str.push(ch);
state = TokenizerState::Elseif;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Elseif => TokenizerState::Elseif =>
@ -2769,13 +1504,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'r' => 'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("or".to_string())), TokenizerState::Or, &mut token_str, ch),
{
last_index = index as i32;
token = Some(Token::Name("or".to_string()));
token_str.push(ch);
state = TokenizerState::Or;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -2830,13 +1559,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'o' => 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("do".to_string())), TokenizerState::Do, &mut token_str, ch),
{
last_index = index as i32;
token = Some(Token::Name("do".to_string()));
token_str.push(ch);
state = TokenizerState::Do;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -2891,20 +1614,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'f' => 'f' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("if".to_string())), TokenizerState::If, &mut token_str, ch),
{ 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("in".to_string())), TokenizerState::In, &mut token_str, ch),
last_index = index as i32;
token = Some(Token::Name("if".to_string()));
token_str.push(ch);
state = TokenizerState::If;
}
'n' =>
{
last_index = index as i32;
token = Some(Token::Name("in".to_string()));
token_str.push(ch);
state = TokenizerState::In;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -2983,27 +1694,9 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'a' => 'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fa".to_string())), TokenizerState::Fa, &mut token_str, ch),
{ 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fo".to_string())), TokenizerState::Fo, &mut token_str, ch),
last_index = index as i32; 'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fu".to_string())), TokenizerState::Fu, &mut token_str, ch),
token = Some(Token::Name("fa".to_string()));
token_str.push(ch);
state = TokenizerState::Fa;
}
'o' =>
{
last_index = index as i32;
token = Some(Token::Name("fo".to_string()));
token_str.push(ch);
state = TokenizerState::Fo;
}
'u' =>
{
last_index = index as i32;
token = Some(Token::Name("fu".to_string()));
token_str.push(ch);
state = TokenizerState::Fu;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -3034,222 +1727,48 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'n' => 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fun".to_string())), TokenizerState::Fun, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("fun".to_string()));
token_str.push(ch);
state = TokenizerState::Fun;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Fun => TokenizerState::Fun =>
{ {
match ch match ch
{ {
'c' => 'c' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("func".to_string())), TokenizerState::Func, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("func".to_string()));
token_str.push(ch);
state = TokenizerState::Func;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Func => TokenizerState::Func =>
{ {
match ch match ch
{ {
't' => 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("funct".to_string())), TokenizerState::Funct, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("funct".to_string()));
token_str.push(ch);
state = TokenizerState::Funct;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Funct => TokenizerState::Funct =>
{ {
match ch match ch
{ {
'i' => 'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("functi".to_string())), TokenizerState::Functi, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("functi".to_string()));
token_str.push(ch);
state = TokenizerState::Functi;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Functi => TokenizerState::Functi =>
{ {
match ch match ch
{ {
'o' => 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("functio".to_string())), TokenizerState::Functio, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("functio".to_string()));
token_str.push(ch);
state = TokenizerState::Functio;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Functio => TokenizerState::Functio =>
{ {
match ch match ch
{ {
'n' => 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("function".to_string())), TokenizerState::Function, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("function".to_string()));
token_str.push(ch);
state = TokenizerState::Function;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Function => TokenizerState::Function =>
@ -3280,111 +1799,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'l' => 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fal".to_string())), TokenizerState::Fal, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("fal".to_string()));
token_str.push(ch);
state = TokenizerState::Fal;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Fal => TokenizerState::Fal =>
{ {
match ch match ch
{ {
's' => 's' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fals".to_string())), TokenizerState::Fals, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("fals".to_string()));
token_str.push(ch);
state = TokenizerState::Fals;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Fals => TokenizerState::Fals =>
{ {
match ch match ch
{ {
'e' => 'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("false".to_string())), TokenizerState::False, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("false".to_string()));
token_str.push(ch);
state = TokenizerState::False;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::False => TokenizerState::False =>
@ -3415,37 +1847,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'r' => 'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("for".to_string())), TokenizerState::For, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("for".to_string()));
token_str.push(ch);
state = TokenizerState::For;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::For => TokenizerState::For =>
@ -3476,13 +1879,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'o' => 'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("lo".to_string())), TokenizerState::Lo, &mut token_str, ch),
{
last_index = index as i32;
token = Some(Token::Name("lo".to_string()));
token_str.push(ch);
state = TokenizerState::Lo;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -3513,111 +1910,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'c' => 'c' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("loc".to_string())), TokenizerState::Loc, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("loc".to_string()));
token_str.push(ch);
state = TokenizerState::Loc;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Loc => TokenizerState::Loc =>
{ {
match ch match ch
{ {
'a' => 'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("loca".to_string())), TokenizerState::Loca, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("loca".to_string()));
token_str.push(ch);
state = TokenizerState::Loca;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Loca => TokenizerState::Loca =>
{ {
match ch match ch
{ {
'l' => 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("local".to_string())), TokenizerState::Local, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("local".to_string()));
token_str.push(ch);
state = TokenizerState::Local;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Local => TokenizerState::Local =>
@ -3648,13 +1958,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
'n' => 'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("un".to_string())), TokenizerState::Un, &mut token_str, ch),
{
last_index = index as i32;
token = Some(Token::Name("un".to_string()));
token_str.push(ch);
state = TokenizerState::Un;
}
_ => _ =>
{ {
if ch.is_ascii_alphanumeric() || ch == '_' if ch.is_ascii_alphanumeric() || ch == '_'
@ -3685,111 +1989,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{ {
match ch match ch
{ {
't' => 't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("unt".to_string())), TokenizerState::Unt, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("unt".to_string()));
token_str.push(ch);
state = TokenizerState::Unt;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Unt => TokenizerState::Unt =>
{ {
match ch match ch
{ {
'i' => 'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("unti".to_string())), TokenizerState::Unti, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("unti".to_string()));
token_str.push(ch);
state = TokenizerState::Unti;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Unti => TokenizerState::Unti =>
{ {
match ch match ch
{ {
'l' => 'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("until".to_string())), TokenizerState::Until, &mut token_str, ch),
{ _ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
last_index = index as i32;
token = Some(Token::Name("until".to_string()));
token_str.push(ch);
state = TokenizerState::Until;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
} }
} }
TokenizerState::Until => TokenizerState::Until =>
@ -3816,7 +2033,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
tokens.push(Token::Until); tokens.push(Token::Until);
} }
} }
_ => todo!("State {:?}", state) _ => todo!("State: {:?}", state),
} }
index += 1; index += 1;
} }