0x4261756D
/
luaaaaah
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

deeduplicate code

This commit is contained in:
0x4261756D 2023-06-06 23:56:38 +02:00
parent a149609f03
commit 317878dd3b
1 changed files with 234 additions and 2017 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2251
src/main.rs
View File

@ -69,6 +69,66 @@ enum TokenizerState
Function,
}
fn tokenize_terminal_no_str(last_index: &mut i32, index: usize, token: &mut Option<Token>, state: &mut TokenizerState, new_token: Option<Token>, new_state: TokenizerState)
{
*last_index = index as i32;
*token = new_token;
*state = new_state;
}
fn tokenize_terminal(last_index: &mut i32, index: usize, token: &mut Option<Token>, state: &mut TokenizerState, new_token: Option<Token>, new_state: TokenizerState, token_str: &mut String, ch: char)
{
tokenize_terminal_no_str(last_index, index, token, state, new_token, new_state);
token_str.push(ch);
}
fn tokenize_backtrack(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState) -> Result<(), &'static str>
{
return tokenize_backtrack_custom_token(last_index, index, tokens, token, token_str, state, token.clone().unwrap());
}
fn tokenize_backtrack_name(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState) -> Result<(), &'static str>
{
if *last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
*index = *last_index as usize;
*last_index = -1;
tokens.push(Token::Name(token_str.clone()));
*token = None;
token_str.clear();
*state = TokenizerState::Start;
return Ok(());
}
fn tokenize_backtrack_custom_token(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState, new_token: Token) -> Result<(), &'static str>
{
if *last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
*index = *last_index as usize;
*last_index = -1;
tokens.push(new_token);
*token = None;
token_str.clear();
*state = TokenizerState::Start;
return Ok(());
}
fn tokenize_alphanumeric_nonterminal(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState, ch: char) -> Result<(), &'static str>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
*last_index = *index as i32;
token_str.push(ch);
*state = TokenizerState::Name;
}
else
{
tokenize_backtrack_name(last_index, index, tokens, token, token_str, state)?;
}
return Ok(());
}
fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
let mut tokens: Vec<Token> = Vec::new();
@ -90,249 +150,44 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'-' =>
{
last_index = index as i32;
token = Some(Token::Minus);
state = TokenizerState::Minus;
}
'a' =>
{
last_index = index as i32;
token = Some(Token::Name("a".to_string()));
token_str.push(ch);
state = TokenizerState::A;
}
'b' =>
{
last_index = index as i32;
token = Some(Token::Name("b".to_string()));
token_str.push(ch);
state = TokenizerState::B;
}
'd' =>
{
last_index = index as i32;
token = Some(Token::Name("d".to_string()));
token_str.push(ch);
state = TokenizerState::D;
}
'e' =>
{
last_index = index as i32;
token = Some(Token::Name("e".to_string()));
token_str.push(ch);
state = TokenizerState::E;
}
'f' =>
{
last_index = index as i32;
token = Some(Token::Name("f".to_string()));
token_str.push(ch);
state = TokenizerState::F;
}
'i' =>
{
last_index = index as i32;
token = Some(Token::Name("i".to_string()));
token_str.push(ch);
state = TokenizerState::I;
}
'g' =>
{
last_index = index as i32;
token = Some(Token::Name("g".to_string()));
token_str.push(ch);
state = TokenizerState::G;
}
'l' =>
{
last_index = index as i32;
token = Some(Token::Name("l".to_string()));
token_str.push(ch);
state = TokenizerState::L;
}
'n' =>
{
last_index = index as i32;
token = Some(Token::Name("n".to_string()));
token_str.push(ch);
state = TokenizerState::N;
}
'o' =>
{
last_index = index as i32;
token = Some(Token::Name("o".to_string()));
token_str.push(ch);
state = TokenizerState::O;
}
'r' =>
{
last_index = index as i32;
token = Some(Token::Name("r".to_string()));
token_str.push(ch);
state = TokenizerState::R;
}
't' =>
{
last_index = index as i32;
token = Some(Token::Name("t".to_string()));
token_str.push(ch);
state = TokenizerState::T;
}
'u' =>
{
last_index = index as i32;
token = Some(Token::Name("u".to_string()));
token_str.push(ch);
state = TokenizerState::U;
}
'w' =>
{
last_index = index as i32;
token = Some(Token::Name("w".to_string()));
token_str.push(ch);
state = TokenizerState::W;
}
',' =>
{
last_index = index as i32;
token = Some(Token::Comma);
state = TokenizerState::Comma;
}
'=' =>
{
last_index = index as i32;
token = Some(Token::Equals);
state = TokenizerState::Equals;
}
'(' =>
{
last_index = index as i32;
token = Some(Token::RoundOpen);
state = TokenizerState::RoundOpen;
}
')' =>
{
last_index = index as i32;
token = Some(Token::RoundClosed);
state = TokenizerState::RoundClosed;
}
'.' =>
{
last_index = index as i32;
token = Some(Token::Dot);
state = TokenizerState::Dot;
}
':' =>
{
last_index = index as i32;
token = Some(Token::Colon);
state = TokenizerState::Colon;
}
'{' =>
{
last_index = index as i32;
token = Some(Token::CurlyOpen);
state = TokenizerState::CurlyOpen;
}
'}' =>
{
last_index = index as i32;
token = Some(Token::CurlyClosed);
state = TokenizerState::CurlyClosed;
}
'[' =>
{
last_index = index as i32;
token = Some(Token::SquareOpen);
state = TokenizerState::SquareOpen;
}
']' =>
{
last_index = index as i32;
token = Some(Token::SquareClosed);
state = TokenizerState::SquareClosed;
}
'+' =>
{
last_index = index as i32;
token = Some(Token::Plus);
state = TokenizerState::Plus;
}
'~' =>
{
last_index = index as i32;
token = Some(Token::Tilde);
state = TokenizerState::Tilde;
}
'>' =>
{
last_index = index as i32;
token = Some(Token::Gt);
state = TokenizerState::Gt;
}
'<' =>
{
last_index = index as i32;
token = Some(Token::Lt);
state = TokenizerState::Lt;
}
'#' =>
{
last_index = index as i32;
token = Some(Token::Hash);
state = TokenizerState::Hash;
}
'|' =>
{
last_index = index as i32;
token = Some(Token::Pipe);
state = TokenizerState::Pipe;
}
'&' =>
{
last_index = index as i32;
token = Some(Token::Ampersand);
state = TokenizerState::Ampersand;
}
'%' =>
{
last_index = index as i32;
token = Some(Token::Percent);
state = TokenizerState::Percent;
}
'*' =>
{
last_index = index as i32;
token = Some(Token::Star);
state = TokenizerState::Star;
}
'/' =>
{
last_index = index as i32;
token = Some(Token::Slash);
state = TokenizerState::Slash;
}
';' =>
{
last_index = index as i32;
token = Some(Token::Semicolon);
state = TokenizerState::Semicolon;
}
'^' =>
{
last_index = index as i32;
token = Some(Token::Caret);
state = TokenizerState::Caret;
}
'0' =>
{
last_index = index as i32;
token = Some(Token::IntLiteral("0".to_string()));
token_str.push(ch);
state = TokenizerState::Zero;
}
'-' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Minus), TokenizerState::Minus),
'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("a".to_string())), TokenizerState::A, &mut token_str, ch),
'b' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("b".to_string())), TokenizerState::B, &mut token_str, ch),
'd' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("d".to_string())), TokenizerState::D, &mut token_str, ch),
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("e".to_string())), TokenizerState::E, &mut token_str, ch),
'f' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("f".to_string())), TokenizerState::F, &mut token_str, ch),
'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("i".to_string())), TokenizerState::I, &mut token_str, ch),
'g' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("g".to_string())), TokenizerState::G, &mut token_str, ch),
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("l".to_string())), TokenizerState::L, &mut token_str, ch),
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("n".to_string())), TokenizerState::N, &mut token_str, ch),
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("o".to_string())), TokenizerState::O, &mut token_str, ch),
'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("r".to_string())), TokenizerState::R, &mut token_str, ch),
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("t".to_string())), TokenizerState::T, &mut token_str, ch),
'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("u".to_string())), TokenizerState::U, &mut token_str, ch),
'w' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("w".to_string())), TokenizerState::W, &mut token_str, ch),
',' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Comma), TokenizerState::Comma),
'=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Equals), TokenizerState::Equals),
'(' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::RoundOpen), TokenizerState::RoundOpen),
')' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::RoundClosed), TokenizerState::RoundClosed),
'.' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Dot), TokenizerState::Dot),
':' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Colon), TokenizerState::Colon),
'{' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::CurlyOpen), TokenizerState::CurlyOpen),
'}' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::CurlyClosed), TokenizerState::CurlyClosed),
'[' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::SquareOpen), TokenizerState::SquareOpen),
']' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::SquareClosed), TokenizerState::SquareClosed),
'+' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Plus), TokenizerState::Plus),
'~' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Tilde), TokenizerState::Tilde),
'>' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Gt), TokenizerState::Gt),
'<' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Lt), TokenizerState::Lt),
'#' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Hash), TokenizerState::Hash),
'|' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Pipe), TokenizerState::Pipe),
'&' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Ampersand), TokenizerState::Ampersand),
'%' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Percent), TokenizerState::Percent),
'*' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Star), TokenizerState::Star),
'/' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Slash), TokenizerState::Slash),
';' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Semicolon), TokenizerState::Semicolon),
'^' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Caret), TokenizerState::Caret),
'0' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::IntLiteral("0".to_string())), TokenizerState::Zero, &mut token_str, ch),
'"' =>
{
token = None;
@ -348,17 +203,11 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
if ch.is_whitespace() { }
else if ch.is_ascii_alphabetic() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name(token_str.clone())), TokenizerState::Name, &mut token_str, ch);
}
else if ch.is_numeric() && ch.is_ascii()
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::IntLiteral(token_str.clone()));
state = TokenizerState::Number;
tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::IntLiteral(token_str.clone())), TokenizerState::Number, &mut token_str, ch);
}
else
{
@ -375,12 +224,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
state = TokenizerState::QuoteBackslash;
}
'"' =>
{
last_index = index as i32;
token = Some(Token::StringLiteral(token_str.clone()));
state = TokenizerState::String;
}
'"' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::StringLiteral(token_str.clone())), TokenizerState::String),
_ =>
{
token_str.push(ch);
@ -456,12 +300,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
state = TokenizerState::QuoteBackslash;
}
'"' =>
{
last_index = index as i32;
token = Some(Token::StringLiteral(token_str.clone()));
state = TokenizerState::String;
}
'"' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::StringLiteral(token_str.clone())), TokenizerState::String),
_ =>
{
if !ch.is_whitespace()
@ -579,17 +418,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
}
TokenizerState::String =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
token = None;
tokens.push(Token::StringLiteral(token_str.clone()));
token_str.clear();
state = TokenizerState::Start;
let content = token_str.clone();
tokenize_backtrack_custom_token(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, Token::StringLiteral(content))?;
}
TokenizerState::Name =>
{
@ -757,320 +587,80 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
TokenizerState::GtEquals | TokenizerState::LtEquals | TokenizerState::SquareOpen |
TokenizerState::SquareClosed | TokenizerState::Pipe | TokenizerState::Ampersand |
TokenizerState::Percent | TokenizerState::Star | TokenizerState::Semicolon |
TokenizerState::Caret =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
TokenizerState::Caret | TokenizerState::DotDotDot | TokenizerState::GtGt |
TokenizerState::LtLt | TokenizerState::SlashSlash => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
TokenizerState::Tilde =>
{
match ch
{
'=' =>
{
last_index = index as i32;
token = Some(Token::TildeEquals);
state = TokenizerState::TildeEquals;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
'=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::TildeEquals), TokenizerState::TildeEquals),
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
}
}
TokenizerState::Gt =>
{
match ch
{
'>' =>
{
last_index = index as i32;
token = Some(Token::GtGt);
state = TokenizerState::GtGt;
}
'=' =>
{
last_index = index as i32;
token = Some(Token::GtEquals);
state = TokenizerState::GtEquals;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
'>' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::GtGt), TokenizerState::GtGt),
'=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::GtEquals), TokenizerState::GtEquals),
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
}
}
TokenizerState::Lt =>
{
match ch
{
'>' =>
{
last_index = index as i32;
token = Some(Token::LtLt);
state = TokenizerState::LtLt;
}
'=' =>
{
last_index = index as i32;
token = Some(Token::LtEquals);
state = TokenizerState::LtEquals;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
'>' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::LtLt), TokenizerState::LtLt),
'=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::LtEquals), TokenizerState::LtEquals),
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
}
}
TokenizerState::GtGt =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
token = None;
token_str.clear();
state = TokenizerState::Start;
tokens.push(Token::GtGt);
}
TokenizerState::LtLt =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
token = None;
token_str.clear();
state = TokenizerState::Start;
tokens.push(Token::LtLt);
}
TokenizerState::Slash =>
{
match ch
{
'/' =>
{
last_index = index as i32;
token = Some(Token::SlashSlash);
state = TokenizerState::SlashSlash;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
'/' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::SlashSlash), TokenizerState::SlashSlash),
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
}
}
TokenizerState::SlashSlash =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
token = None;
token_str.clear();
state = TokenizerState::Start;
tokens.push(Token::SlashSlash);
}
TokenizerState::Dot =>
{
match ch
{
'.' =>
{
last_index = index as i32;
token = Some(Token::DotDot);
state = TokenizerState::DotDot;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
'.' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::DotDot), TokenizerState::DotDot),
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
}
}
TokenizerState::DotDot =>
{
match ch
{
'.' =>
{
last_index = index as i32;
token = Some(Token::DotDotDot);
state = TokenizerState::DotDotDot;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
'.' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::DotDotDot), TokenizerState::DotDotDot),
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
}
}
TokenizerState::DotDotDot =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
token = None;
token_str.clear();
state = TokenizerState::Start;
tokens.push(Token::And);
}
TokenizerState::Colon =>
{
match ch
{
':' =>
{
last_index = index as i32;
token = Some(Token::ColonColon);
state = TokenizerState::ColonColon;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
':' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::ColonColon), TokenizerState::ColonColon),
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
}
}
TokenizerState::Equals =>
{
match ch
{
'=' =>
{
last_index = index as i32;
token = Some(Token::EqualsEquals);
state = TokenizerState::EqualsEquals;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
'=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::EqualsEquals), TokenizerState::EqualsEquals),
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
}
}
TokenizerState::Minus =>
{
match ch
{
'-' =>
{
last_index = index as i32;
token = None;
state = TokenizerState::SmallCommentStart;
}
_ =>
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
'-' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, None, TokenizerState::SmallCommentStart),
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
}
}
TokenizerState::SmallCommentStart =>
@ -1158,13 +748,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'n' =>
{
last_index = index as i32;
token = Some(Token::Name("an".to_string()));
token_str.push(ch);
state = TokenizerState::An;
}
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("an".to_string())), TokenizerState::An, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -1195,37 +779,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'd' =>
{
last_index = index as i32;
token = Some(Token::Name("and".to_string()));
token_str.push(ch);
state = TokenizerState::And;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'd' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("and".to_string())), TokenizerState::And, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::And =>
@ -1256,13 +811,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'h' =>
{
last_index = index as i32;
token = Some(Token::Name("wh".to_string()));
token_str.push(ch);
state = TokenizerState::Wh;
}
'h' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("wh".to_string())), TokenizerState::Wh, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -1293,111 +842,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'i' =>
{
last_index = index as i32;
token = Some(Token::Name("whi".to_string()));
token_str.push(ch);
state = TokenizerState::Whi;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("whi".to_string())), TokenizerState::Whi, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Whi =>
{
match ch
{
'l' =>
{
last_index = index as i32;
token = Some(Token::Name("whil".to_string()));
token_str.push(ch);
state = TokenizerState::Whil;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("whil".to_string())), TokenizerState::Whil, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Whil =>
{
match ch
{
'e' =>
{
last_index = index as i32;
token = Some(Token::Name("while".to_string()));
token_str.push(ch);
state = TokenizerState::While;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("while".to_string())), TokenizerState::While, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::While =>
@ -1428,13 +890,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'r' =>
{
last_index = index as i32;
token = Some(Token::Name("br".to_string()));
token_str.push(ch);
state = TokenizerState::Br;
}
'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("br".to_string())), TokenizerState::Br, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -1465,111 +921,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'e' =>
{
last_index = index as i32;
token = Some(Token::Name("bre".to_string()));
token_str.push(ch);
state = TokenizerState::Bre;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("bre".to_string())), TokenizerState::Bre, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Bre =>
{
match ch
{
'a' =>
{
last_index = index as i32;
token = Some(Token::Name("brea".to_string()));
token_str.push(ch);
state = TokenizerState::Brea;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("brea".to_string())), TokenizerState::Brea, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Brea =>
{
match ch
{
'k' =>
{
last_index = index as i32;
token = Some(Token::Name("break".to_string()));
token_str.push(ch);
state = TokenizerState::Break;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'k' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("break".to_string())), TokenizerState::Break, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Break =>
@ -1600,111 +969,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'o' =>
{
last_index = index as i32;
token = Some(Token::Name("go".to_string()));
token_str.push(ch);
state = TokenizerState::Go;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.clone().unwrap());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("go".to_string())), TokenizerState::Go, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Go =>
{
match ch
{
't' =>
{
last_index = index as i32;
token = Some(Token::Name("got".to_string()));
token_str.push(ch);
state = TokenizerState::Got;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("got".to_string())), TokenizerState::Got, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Got =>
{
match ch
{
'o' =>
{
last_index = index as i32;
token = Some(Token::Name("goto".to_string()));
token_str.push(ch);
state = TokenizerState::Goto;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("goto".to_string())), TokenizerState::Goto, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Goto =>
@ -1735,13 +1017,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'e' =>
{
last_index = index as i32;
token = Some(Token::Name("re".to_string()));
token_str.push(ch);
state = TokenizerState::Re;
}
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("re".to_string())), TokenizerState::Re, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -1772,155 +1048,33 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
't' =>
{
last_index = index as i32;
token = Some(Token::Name("ret".to_string()));
token_str.push(ch);
state = TokenizerState::Ret;
}
'p' =>
{
last_index = index as i32;
token = Some(Token::Name("rep".to_string()));
token_str.push(ch);
state = TokenizerState::Rep;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("ret".to_string())), TokenizerState::Ret, &mut token_str, ch),
'p' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("rep".to_string())), TokenizerState::Rep, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Ret =>
{
match ch
{
'u' =>
{
last_index = index as i32;
token = Some(Token::Name("retu".to_string()));
token_str.push(ch);
state = TokenizerState::Retu;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("retu".to_string())), TokenizerState::Retu, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Retu =>
{
match ch
{
'r' =>
{
last_index = index as i32;
token = Some(Token::Name("retur".to_string()));
token_str.push(ch);
state = TokenizerState::Retur;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("retur".to_string())), TokenizerState::Retur, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Retur =>
{
match ch
{
'n' =>
{
last_index = index as i32;
token = Some(Token::Name("return".to_string()));
token_str.push(ch);
state = TokenizerState::Return;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("return".to_string())), TokenizerState::Return, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Return =>
@ -1951,111 +1105,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'e' =>
{
last_index = index as i32;
token = Some(Token::Name("repe".to_string()));
token_str.push(ch);
state = TokenizerState::Repe;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("repe".to_string())), TokenizerState::Repe, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Repe =>
{
match ch
{
'a' =>
{
last_index = index as i32;
token = Some(Token::Name("repea".to_string()));
token_str.push(ch);
state = TokenizerState::Repea;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("repea".to_string())), TokenizerState::Repea, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Repea =>
{
match ch
{
't' =>
{
last_index = index as i32;
token = Some(Token::Name("repeat".to_string()));
token_str.push(ch);
state = TokenizerState::Repeat;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("repeat".to_string())), TokenizerState::Repeat, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Repeat =>
@ -2086,20 +1153,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'i' =>
{
last_index = index as i32;
token = Some(Token::Name("ni".to_string()));
token_str.push(ch);
state = TokenizerState::Ni;
}
'o' =>
{
last_index = index as i32;
token = Some(Token::Name("no".to_string()));
token_str.push(ch);
state = TokenizerState::No;
}
'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("ni".to_string())), TokenizerState::Ni, &mut token_str, ch),
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("no".to_string())), TokenizerState::No, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -2130,37 +1185,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
't' =>
{
last_index = index as i32;
token = Some(Token::Name("not".to_string()));
token_str.push(ch);
state = TokenizerState::Not;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("not".to_string())), TokenizerState::Not, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Not =>
@ -2191,37 +1217,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'l' =>
{
last_index = index as i32;
token = Some(Token::Name("nil".to_string()));
token_str.push(ch);
state = TokenizerState::Nil;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("nil".to_string())), TokenizerState::Nil, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Nil =>
@ -2252,20 +1249,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'h' =>
{
last_index = index as i32;
token = Some(Token::Name("th".to_string()));
token_str.push(ch);
state = TokenizerState::Th;
}
'r' =>
{
last_index = index as i32;
token = Some(Token::Name("tr".to_string()));
token_str.push(ch);
state = TokenizerState::Tr;
}
'h' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("th".to_string())), TokenizerState::Th, &mut token_str, ch),
'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("tr".to_string())), TokenizerState::Tr, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -2296,74 +1281,16 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'e' =>
{
last_index = index as i32;
token = Some(Token::Name("the".to_string()));
token_str.push(ch);
state = TokenizerState::The;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("the".to_string())), TokenizerState::The, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::The =>
{
match ch
{
'n' =>
{
last_index = index as i32;
token = Some(Token::Name("then".to_string()));
token_str.push(ch);
state = TokenizerState::Then;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("then".to_string())), TokenizerState::Then, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Then =>
@ -2394,74 +1321,16 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'u' =>
{
last_index = index as i32;
token = Some(Token::Name("tru".to_string()));
token_str.push(ch);
state = TokenizerState::Tru;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("tru".to_string())), TokenizerState::Tru, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Tru =>
{
match ch
{
'e' =>
{
last_index = index as i32;
token = Some(Token::Name("true".to_string()));
token_str.push(ch);
state = TokenizerState::True;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("true".to_string())), TokenizerState::True, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::True =>
@ -2492,20 +1361,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'l' =>
{
last_index = index as i32;
token = Some(Token::Name("el".to_string()));
token_str.push(ch);
state = TokenizerState::El;
}
'n' =>
{
last_index = index as i32;
token = Some(Token::Name("en".to_string()));
token_str.push(ch);
state = TokenizerState::En;
}
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("el".to_string())), TokenizerState::El, &mut token_str, ch),
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("en".to_string())), TokenizerState::En, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -2536,37 +1393,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'd' =>
{
last_index = index as i32;
token = Some(Token::Name("end".to_string()));
token_str.push(ch);
state = TokenizerState::End;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'd' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("end".to_string())), TokenizerState::End, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::End =>
@ -2597,87 +1425,23 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
's' =>
{
last_index = index as i32;
token = Some(Token::Name("els".to_string()));
token_str.push(ch);
state = TokenizerState::Els;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
's' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("els".to_string())), TokenizerState::Els, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Els =>
{
match ch
{
'e' =>
{
last_index = index as i32;
token = Some(Token::Name("else".to_string()));
token_str.push(ch);
state = TokenizerState::Else;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("else".to_string())), TokenizerState::Else, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Else =>
{
match ch
{
'i' =>
{
last_index = index as i32;
token = Some(Token::Name("elsei".to_string()));
token_str.push(ch);
state = TokenizerState::Elsei;
}
'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("elsei".to_string())), TokenizerState::Elsei, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -2708,37 +1472,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'f' =>
{
last_index = index as i32;
token = Some(Token::Name("elseif".to_string()));
token_str.push(ch);
state = TokenizerState::Elseif;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'f' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("elseif".to_string())), TokenizerState::Elseif, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Elseif =>
@ -2769,13 +1504,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'r' =>
{
last_index = index as i32;
token = Some(Token::Name("or".to_string()));
token_str.push(ch);
state = TokenizerState::Or;
}
'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("or".to_string())), TokenizerState::Or, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -2830,13 +1559,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'o' =>
{
last_index = index as i32;
token = Some(Token::Name("do".to_string()));
token_str.push(ch);
state = TokenizerState::Do;
}
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("do".to_string())), TokenizerState::Do, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -2891,20 +1614,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'f' =>
{
last_index = index as i32;
token = Some(Token::Name("if".to_string()));
token_str.push(ch);
state = TokenizerState::If;
}
'n' =>
{
last_index = index as i32;
token = Some(Token::Name("in".to_string()));
token_str.push(ch);
state = TokenizerState::In;
}
'f' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("if".to_string())), TokenizerState::If, &mut token_str, ch),
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("in".to_string())), TokenizerState::In, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -2983,27 +1694,9 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'a' =>
{
last_index = index as i32;
token = Some(Token::Name("fa".to_string()));
token_str.push(ch);
state = TokenizerState::Fa;
}
'o' =>
{
last_index = index as i32;
token = Some(Token::Name("fo".to_string()));
token_str.push(ch);
state = TokenizerState::Fo;
}
'u' =>
{
last_index = index as i32;
token = Some(Token::Name("fu".to_string()));
token_str.push(ch);
state = TokenizerState::Fu;
}
'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fa".to_string())), TokenizerState::Fa, &mut token_str, ch),
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fo".to_string())), TokenizerState::Fo, &mut token_str, ch),
'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fu".to_string())), TokenizerState::Fu, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -3034,222 +1727,48 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'n' =>
{
last_index = index as i32;
token = Some(Token::Name("fun".to_string()));
token_str.push(ch);
state = TokenizerState::Fun;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fun".to_string())), TokenizerState::Fun, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Fun =>
{
match ch
{
'c' =>
{
last_index = index as i32;
token = Some(Token::Name("func".to_string()));
token_str.push(ch);
state = TokenizerState::Func;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'c' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("func".to_string())), TokenizerState::Func, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Func =>
{
match ch
{
't' =>
{
last_index = index as i32;
token = Some(Token::Name("funct".to_string()));
token_str.push(ch);
state = TokenizerState::Funct;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("funct".to_string())), TokenizerState::Funct, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Funct =>
{
match ch
{
'i' =>
{
last_index = index as i32;
token = Some(Token::Name("functi".to_string()));
token_str.push(ch);
state = TokenizerState::Functi;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("functi".to_string())), TokenizerState::Functi, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Functi =>
{
match ch
{
'o' =>
{
last_index = index as i32;
token = Some(Token::Name("functio".to_string()));
token_str.push(ch);
state = TokenizerState::Functio;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("functio".to_string())), TokenizerState::Functio, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Functio =>
{
match ch
{
'n' =>
{
last_index = index as i32;
token = Some(Token::Name("function".to_string()));
token_str.push(ch);
state = TokenizerState::Function;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("function".to_string())), TokenizerState::Function, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Function =>
@ -3280,111 +1799,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'l' =>
{
last_index = index as i32;
token = Some(Token::Name("fal".to_string()));
token_str.push(ch);
state = TokenizerState::Fal;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fal".to_string())), TokenizerState::Fal, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Fal =>
{
match ch
{
's' =>
{
last_index = index as i32;
token = Some(Token::Name("fals".to_string()));
token_str.push(ch);
state = TokenizerState::Fals;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
's' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fals".to_string())), TokenizerState::Fals, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Fals =>
{
match ch
{
'e' =>
{
last_index = index as i32;
token = Some(Token::Name("false".to_string()));
token_str.push(ch);
state = TokenizerState::False;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("false".to_string())), TokenizerState::False, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::False =>
@ -3415,37 +1847,8 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'r' =>
{
last_index = index as i32;
token = Some(Token::Name("for".to_string()));
token_str.push(ch);
state = TokenizerState::For;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("for".to_string())), TokenizerState::For, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::For =>
@ -3476,13 +1879,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'o' =>
{
last_index = index as i32;
token = Some(Token::Name("lo".to_string()));
token_str.push(ch);
state = TokenizerState::Lo;
}
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("lo".to_string())), TokenizerState::Lo, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -3513,111 +1910,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'c' =>
{
last_index = index as i32;
token = Some(Token::Name("loc".to_string()));
token_str.push(ch);
state = TokenizerState::Loc;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'c' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("loc".to_string())), TokenizerState::Loc, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Loc =>
{
match ch
{
'a' =>
{
last_index = index as i32;
token = Some(Token::Name("loca".to_string()));
token_str.push(ch);
state = TokenizerState::Loca;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("loca".to_string())), TokenizerState::Loca, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Loca =>
{
match ch
{
'l' =>
{
last_index = index as i32;
token = Some(Token::Name("local".to_string()));
token_str.push(ch);
state = TokenizerState::Local;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("local".to_string())), TokenizerState::Local, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Local =>
@ -3648,13 +1958,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
'n' =>
{
last_index = index as i32;
token = Some(Token::Name("un".to_string()));
token_str.push(ch);
state = TokenizerState::Un;
}
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("un".to_string())), TokenizerState::Un, &mut token_str, ch),
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
@ -3685,111 +1989,24 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
{
match ch
{
't' =>
{
last_index = index as i32;
token = Some(Token::Name("unt".to_string()));
token_str.push(ch);
state = TokenizerState::Unt;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("unt".to_string())), TokenizerState::Unt, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Unt =>
{
match ch
{
'i' =>
{
last_index = index as i32;
token = Some(Token::Name("unti".to_string()));
token_str.push(ch);
state = TokenizerState::Unti;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("unti".to_string())), TokenizerState::Unti, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Unti =>
{
match ch
{
'l' =>
{
last_index = index as i32;
token = Some(Token::Name("until".to_string()));
token_str.push(ch);
state = TokenizerState::Until;
}
_ =>
{
if ch.is_ascii_alphanumeric() || ch == '_'
{
last_index = index as i32;
token_str.push(ch);
token = Some(Token::Name(token_str.clone()));
state = TokenizerState::Name;
}
else
{
if last_index == -1 || token.is_none()
{
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
return Err("Lexerr");
}
index = last_index as usize;
last_index = -1;
tokens.push(token.unwrap().clone());
token = None;
token_str.clear();
state = TokenizerState::Start;
}
}
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("until".to_string())), TokenizerState::Until, &mut token_str, ch),
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
}
}
TokenizerState::Until =>
@ -3816,7 +2033,7 @@ fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
tokens.push(Token::Until);
}
}
_ => todo!("State {:?}", state)
_ => todo!("State: {:?}", state),
}
index += 1;
}