2042 lines
63 KiB
Rust
2042 lines
63 KiB
Rust
use std::{env, fs};
|
|
|
|
fn main()
|
|
{
|
|
let args: Vec<String> = env::args().collect();
|
|
let file_content = fs::read_to_string(&args[1]).expect("Could not read source file");
|
|
|
|
match compile(&file_content)
|
|
{
|
|
Ok(()) =>
|
|
{
|
|
println!("Done compiling");
|
|
}
|
|
Err(msg) => println!("ERROR: {}", msg)
|
|
}
|
|
}
|
|
|
|
fn compile(file_content: &String) -> Result<(), &'static str>
|
|
{
|
|
let tokens: Vec<Token> = tokenize(&file_content)?;
|
|
println!("{:?}", tokens);
|
|
return Ok(());
|
|
}
|
|
#[derive(Debug, Clone)]
|
|
enum Token
|
|
{
|
|
Name(String),
|
|
And, Break, Do, Else, Elseif, End,
|
|
False, For, Function, Goto, If, In,
|
|
Local, Nil, Not, Or, Repeat, Return,
|
|
Then, True, Until, While,
|
|
Plus, Minus, Star, Slash, Percent, Caret, Hash,
|
|
Ampersand, Tilde, Pipe, LtLt, GtGt, SlashSlash,
|
|
EqualsEquals, TildeEquals, LtEquals, GtEquals, Lt, Gt, Equals,
|
|
RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, ColonColon,
|
|
Semicolon, Colon, Comma, Dot, DotDot, DotDotDot,
|
|
IntLiteral(String),
|
|
HexLiteral(String),
|
|
StringLiteral(String),
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq)]
|
|
enum TokenizerState
|
|
{
|
|
Start,
|
|
Quote, SingleQuote, Name, Number, Zero,
|
|
A, B, D, E, F, G, I, L, N, O, R, T, U, W,
|
|
Plus, Minus, Star, Slash, Percent, Caret, Hash,
|
|
Ampersand, Tilde, Pipe, Lt, Gt, Equals, RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed,
|
|
Colon, Semicolon, Comma, Dot,
|
|
|
|
An, Br, Do, El, En, Fa, Fo, Fu, Go, If, In, Lo, Ni, No, Or, Re, Th, Tr, Un, Wh,
|
|
LtLt, GtGt, SlashSlash, EqualsEquals, TildeEquals, LtEquals, GtEquals, ColonColon, DotDot,
|
|
SmallCommentStart, QuoteBackslash, SingleQuoteBackslash, String, HexNumberX, ExpNumber,
|
|
|
|
And, Bre, Els, End, Fal, For, Fun, Got, Loc, Nil, Not, Rep, Ret, The, Tru, Unt, Whi,
|
|
DotDotDot, HexNumber, QuoteBackslashZ, SingleQuoteBackslashZ,
|
|
BigCommentLongBracketStart, SmallComment,
|
|
|
|
Brea, Else, Fals, Func, Goto, Loca, Repe, Retu, Then, True, Unti, Whil, HexExpNumber,
|
|
BigComment, BigCommentLongBracketEnd,
|
|
|
|
Break, Elsei, False, Funct, Local, Repea, Retur, Until, While,
|
|
|
|
Elseif, Functi, Repeat, Return,
|
|
|
|
Functio,
|
|
|
|
Function,
|
|
}
|
|
|
|
fn tokenize_terminal_no_str(last_index: &mut i32, index: usize, token: &mut Option<Token>, state: &mut TokenizerState, new_token: Option<Token>, new_state: TokenizerState)
|
|
{
|
|
*last_index = index as i32;
|
|
*token = new_token;
|
|
*state = new_state;
|
|
}
|
|
fn tokenize_terminal(last_index: &mut i32, index: usize, token: &mut Option<Token>, state: &mut TokenizerState, new_token: Option<Token>, new_state: TokenizerState, token_str: &mut String, ch: char)
|
|
{
|
|
tokenize_terminal_no_str(last_index, index, token, state, new_token, new_state);
|
|
token_str.push(ch);
|
|
}
|
|
fn tokenize_backtrack(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState) -> Result<(), &'static str>
|
|
{
|
|
return tokenize_backtrack_custom_token(last_index, index, tokens, token, token_str, state, token.clone().unwrap());
|
|
}
|
|
fn tokenize_backtrack_name(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState) -> Result<(), &'static str>
|
|
{
|
|
if *last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
*index = *last_index as usize;
|
|
*last_index = -1;
|
|
tokens.push(Token::Name(token_str.clone()));
|
|
*token = None;
|
|
token_str.clear();
|
|
*state = TokenizerState::Start;
|
|
return Ok(());
|
|
}
|
|
fn tokenize_backtrack_custom_token(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState, new_token: Token) -> Result<(), &'static str>
|
|
{
|
|
if *last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
*index = *last_index as usize;
|
|
*last_index = -1;
|
|
tokens.push(new_token);
|
|
*token = None;
|
|
token_str.clear();
|
|
*state = TokenizerState::Start;
|
|
return Ok(());
|
|
}
|
|
fn tokenize_alphanumeric_nonterminal(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState, ch: char) -> Result<(), &'static str>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
*last_index = *index as i32;
|
|
token_str.push(ch);
|
|
*state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
tokenize_backtrack_name(last_index, index, tokens, token, token_str, state)?;
|
|
}
|
|
return Ok(());
|
|
}
|
|
|
|
fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
|
|
{
|
|
let mut tokens: Vec<Token> = Vec::new();
|
|
let mut state = TokenizerState::Start;
|
|
let char_vec: Vec<char> = file_content.chars().collect();
|
|
|
|
let mut last_index: i32 = -1;
|
|
let mut index = 0;
|
|
let mut token: Option<Token> = None;
|
|
let mut token_str: String = String::new();
|
|
let mut long_bracket_level = 0;
|
|
|
|
while index < char_vec.len()
|
|
{
|
|
let ch = char_vec[index];
|
|
match state
|
|
{
|
|
TokenizerState::Start =>
|
|
{
|
|
match ch
|
|
{
|
|
'-' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Minus), TokenizerState::Minus),
|
|
'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("a".to_string())), TokenizerState::A, &mut token_str, ch),
|
|
'b' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("b".to_string())), TokenizerState::B, &mut token_str, ch),
|
|
'd' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("d".to_string())), TokenizerState::D, &mut token_str, ch),
|
|
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("e".to_string())), TokenizerState::E, &mut token_str, ch),
|
|
'f' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("f".to_string())), TokenizerState::F, &mut token_str, ch),
|
|
'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("i".to_string())), TokenizerState::I, &mut token_str, ch),
|
|
'g' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("g".to_string())), TokenizerState::G, &mut token_str, ch),
|
|
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("l".to_string())), TokenizerState::L, &mut token_str, ch),
|
|
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("n".to_string())), TokenizerState::N, &mut token_str, ch),
|
|
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("o".to_string())), TokenizerState::O, &mut token_str, ch),
|
|
'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("r".to_string())), TokenizerState::R, &mut token_str, ch),
|
|
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("t".to_string())), TokenizerState::T, &mut token_str, ch),
|
|
'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("u".to_string())), TokenizerState::U, &mut token_str, ch),
|
|
'w' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("w".to_string())), TokenizerState::W, &mut token_str, ch),
|
|
',' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Comma), TokenizerState::Comma),
|
|
'=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Equals), TokenizerState::Equals),
|
|
'(' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::RoundOpen), TokenizerState::RoundOpen),
|
|
')' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::RoundClosed), TokenizerState::RoundClosed),
|
|
'.' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Dot), TokenizerState::Dot),
|
|
':' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Colon), TokenizerState::Colon),
|
|
'{' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::CurlyOpen), TokenizerState::CurlyOpen),
|
|
'}' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::CurlyClosed), TokenizerState::CurlyClosed),
|
|
'[' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::SquareOpen), TokenizerState::SquareOpen),
|
|
']' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::SquareClosed), TokenizerState::SquareClosed),
|
|
'+' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Plus), TokenizerState::Plus),
|
|
'~' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Tilde), TokenizerState::Tilde),
|
|
'>' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Gt), TokenizerState::Gt),
|
|
'<' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Lt), TokenizerState::Lt),
|
|
'#' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Hash), TokenizerState::Hash),
|
|
'|' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Pipe), TokenizerState::Pipe),
|
|
'&' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Ampersand), TokenizerState::Ampersand),
|
|
'%' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Percent), TokenizerState::Percent),
|
|
'*' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Star), TokenizerState::Star),
|
|
'/' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Slash), TokenizerState::Slash),
|
|
';' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Semicolon), TokenizerState::Semicolon),
|
|
'^' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::Caret), TokenizerState::Caret),
|
|
'0' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::IntLiteral("0".to_string())), TokenizerState::Zero, &mut token_str, ch),
|
|
'"' =>
|
|
{
|
|
token = None;
|
|
state = TokenizerState::Quote;
|
|
}
|
|
'\'' =>
|
|
{
|
|
token = None;
|
|
state = TokenizerState::SingleQuote;
|
|
}
|
|
_ =>
|
|
{
|
|
if ch.is_whitespace() { }
|
|
else if ch.is_ascii_alphabetic() || ch == '_'
|
|
{
|
|
tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name(token_str.clone())), TokenizerState::Name, &mut token_str, ch);
|
|
}
|
|
else if ch.is_numeric() && ch.is_ascii()
|
|
{
|
|
tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::IntLiteral(token_str.clone())), TokenizerState::Number, &mut token_str, ch);
|
|
}
|
|
else
|
|
{
|
|
todo!("State {:?}, Char {}", state, ch);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::Quote =>
|
|
{
|
|
match ch
|
|
{
|
|
'\\' =>
|
|
{
|
|
state = TokenizerState::QuoteBackslash;
|
|
}
|
|
'"' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::StringLiteral(token_str.clone())), TokenizerState::String),
|
|
_ =>
|
|
{
|
|
token_str.push(ch);
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::QuoteBackslash =>
|
|
{
|
|
match ch
|
|
{
|
|
'a' =>
|
|
{
|
|
token_str.push('\u{0007}');
|
|
state = TokenizerState::Quote;
|
|
}
|
|
'b' =>
|
|
{
|
|
token_str.push('\u{0008}');
|
|
state = TokenizerState::Quote;
|
|
}
|
|
't' =>
|
|
{
|
|
token_str.push('\t');
|
|
state = TokenizerState::Quote;
|
|
}
|
|
'n' | '\n' =>
|
|
{
|
|
token_str.push('\n');
|
|
state = TokenizerState::Quote;
|
|
}
|
|
'v' =>
|
|
{
|
|
token_str.push('\u{000b}');
|
|
state = TokenizerState::Quote;
|
|
}
|
|
'f' =>
|
|
{
|
|
token_str.push('\u{000c}');
|
|
state = TokenizerState::Quote;
|
|
}
|
|
'r' =>
|
|
{
|
|
token_str.push('\r');
|
|
state = TokenizerState::Quote;
|
|
}
|
|
'\\' =>
|
|
{
|
|
token_str.push('\\');
|
|
state = TokenizerState::Quote;
|
|
}
|
|
'"' =>
|
|
{
|
|
token_str.push('\"');
|
|
state = TokenizerState::Quote;
|
|
}
|
|
'\'' =>
|
|
{
|
|
token_str.push('\'');
|
|
state = TokenizerState::Quote;
|
|
}
|
|
'z' =>
|
|
{
|
|
state = TokenizerState::QuoteBackslashZ;
|
|
}
|
|
_ => return Err("Unknown escape sequence"),
|
|
}
|
|
}
|
|
TokenizerState::QuoteBackslashZ =>
|
|
{
|
|
match ch
|
|
{
|
|
'\\' =>
|
|
{
|
|
state = TokenizerState::QuoteBackslash;
|
|
}
|
|
'"' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::StringLiteral(token_str.clone())), TokenizerState::String),
|
|
_ =>
|
|
{
|
|
if !ch.is_whitespace()
|
|
{
|
|
token_str.push(ch);
|
|
state = TokenizerState::Quote;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::SingleQuote =>
|
|
{
|
|
match ch
|
|
{
|
|
'\\' =>
|
|
{
|
|
state = TokenizerState::SingleQuoteBackslash;
|
|
}
|
|
'\'' =>
|
|
{
|
|
last_index = index as i32;
|
|
token = Some(Token::StringLiteral(token_str.clone()));
|
|
state = TokenizerState::String;
|
|
}
|
|
_ =>
|
|
{
|
|
token_str.push(ch);
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::SingleQuoteBackslash =>
|
|
{
|
|
match ch
|
|
{
|
|
'a' =>
|
|
{
|
|
token_str.push('\u{0007}');
|
|
state = TokenizerState::SingleQuote;
|
|
}
|
|
'b' =>
|
|
{
|
|
token_str.push('\u{0008}');
|
|
state = TokenizerState::SingleQuote;
|
|
}
|
|
't' =>
|
|
{
|
|
token_str.push('\t');
|
|
state = TokenizerState::SingleQuote;
|
|
}
|
|
'n' | '\n' =>
|
|
{
|
|
token_str.push('\n');
|
|
state = TokenizerState::SingleQuote;
|
|
}
|
|
'v' =>
|
|
{
|
|
token_str.push('\u{000b}');
|
|
state = TokenizerState::SingleQuote;
|
|
}
|
|
'f' =>
|
|
{
|
|
token_str.push('\u{000c}');
|
|
state = TokenizerState::SingleQuote;
|
|
}
|
|
'r' =>
|
|
{
|
|
token_str.push('\r');
|
|
state = TokenizerState::SingleQuote;
|
|
}
|
|
'\\' =>
|
|
{
|
|
token_str.push('\\');
|
|
state = TokenizerState::SingleQuote;
|
|
}
|
|
'"' =>
|
|
{
|
|
token_str.push('\"');
|
|
state = TokenizerState::SingleQuote;
|
|
}
|
|
'\'' =>
|
|
{
|
|
token_str.push('\'');
|
|
state = TokenizerState::SingleQuote;
|
|
}
|
|
'z' =>
|
|
{
|
|
state = TokenizerState::SingleQuoteBackslashZ;
|
|
}
|
|
_ => return Err("Unknown escape sequence"),
|
|
}
|
|
}
|
|
TokenizerState::SingleQuoteBackslashZ =>
|
|
{
|
|
match ch
|
|
{
|
|
'\\' =>
|
|
{
|
|
state = TokenizerState::SingleQuoteBackslash;
|
|
}
|
|
'\'' =>
|
|
{
|
|
last_index = index as i32;
|
|
token = Some(Token::StringLiteral(token_str.clone()));
|
|
state = TokenizerState::String;
|
|
}
|
|
_ =>
|
|
{
|
|
if !ch.is_whitespace()
|
|
{
|
|
token_str.push(ch);
|
|
state = TokenizerState::SingleQuote;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::String =>
|
|
{
|
|
let content = token_str.clone();
|
|
tokenize_backtrack_custom_token(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, Token::StringLiteral(content))?;
|
|
}
|
|
TokenizerState::Name =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
TokenizerState::Zero =>
|
|
{
|
|
match ch
|
|
{
|
|
'x' =>
|
|
{
|
|
token_str.push(ch);
|
|
token = None;
|
|
state = TokenizerState::HexNumberX;
|
|
}
|
|
_ =>
|
|
{
|
|
if ch.is_numeric() && ch.is_ascii()
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::IntLiteral(token_str.clone()));
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::HexNumberX =>
|
|
{
|
|
if ch.is_ascii() && ch.is_numeric() || match ch
|
|
{
|
|
'A'..='F' | 'a'..='f' => true,
|
|
_ => false,
|
|
}
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::HexLiteral(token_str.clone()));
|
|
state = TokenizerState::HexNumber;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
TokenizerState::HexNumber =>
|
|
{
|
|
match ch
|
|
{
|
|
'p' =>
|
|
{
|
|
token_str.push(ch);
|
|
token = None;
|
|
state = TokenizerState::HexExpNumber;
|
|
}
|
|
_ =>
|
|
{
|
|
if ch.is_ascii() && ch.is_numeric() || match ch
|
|
{
|
|
'A'..='F' | 'a'..='f' => true,
|
|
_ => false,
|
|
}
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::HexLiteral(token_str.clone()));
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::Number =>
|
|
{
|
|
match ch
|
|
{
|
|
'e' =>
|
|
{
|
|
token_str.push(ch);
|
|
token = None;
|
|
state = TokenizerState::ExpNumber;
|
|
}
|
|
_ =>
|
|
{
|
|
if ch.is_numeric() && ch.is_ascii()
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::IntLiteral(token_str.clone()));
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::Comma | TokenizerState::RoundOpen | TokenizerState::RoundClosed |
|
|
TokenizerState::CurlyOpen | TokenizerState::CurlyClosed | TokenizerState::Plus |
|
|
TokenizerState::TildeEquals | TokenizerState::EqualsEquals | TokenizerState::Hash |
|
|
TokenizerState::GtEquals | TokenizerState::LtEquals | TokenizerState::SquareOpen |
|
|
TokenizerState::SquareClosed | TokenizerState::Pipe | TokenizerState::Ampersand |
|
|
TokenizerState::Percent | TokenizerState::Star | TokenizerState::Semicolon |
|
|
TokenizerState::Caret | TokenizerState::DotDotDot | TokenizerState::GtGt |
|
|
TokenizerState::LtLt | TokenizerState::SlashSlash => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
|
|
TokenizerState::Tilde =>
|
|
{
|
|
match ch
|
|
{
|
|
'=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::TildeEquals), TokenizerState::TildeEquals),
|
|
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
|
|
}
|
|
}
|
|
TokenizerState::Gt =>
|
|
{
|
|
match ch
|
|
{
|
|
'>' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::GtGt), TokenizerState::GtGt),
|
|
'=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::GtEquals), TokenizerState::GtEquals),
|
|
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
|
|
}
|
|
}
|
|
TokenizerState::Lt =>
|
|
{
|
|
match ch
|
|
{
|
|
'>' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::LtLt), TokenizerState::LtLt),
|
|
'=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::LtEquals), TokenizerState::LtEquals),
|
|
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
|
|
}
|
|
}
|
|
TokenizerState::Slash =>
|
|
{
|
|
match ch
|
|
{
|
|
'/' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::SlashSlash), TokenizerState::SlashSlash),
|
|
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
|
|
}
|
|
}
|
|
TokenizerState::Dot =>
|
|
{
|
|
match ch
|
|
{
|
|
'.' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::DotDot), TokenizerState::DotDot),
|
|
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
|
|
}
|
|
}
|
|
TokenizerState::DotDot =>
|
|
{
|
|
match ch
|
|
{
|
|
'.' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::DotDotDot), TokenizerState::DotDotDot),
|
|
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
|
|
}
|
|
}
|
|
TokenizerState::Colon =>
|
|
{
|
|
match ch
|
|
{
|
|
':' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::ColonColon), TokenizerState::ColonColon),
|
|
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
|
|
}
|
|
}
|
|
TokenizerState::Equals =>
|
|
{
|
|
match ch
|
|
{
|
|
'=' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, Some(Token::EqualsEquals), TokenizerState::EqualsEquals),
|
|
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
|
|
}
|
|
}
|
|
TokenizerState::Minus =>
|
|
{
|
|
match ch
|
|
{
|
|
'-' => tokenize_terminal_no_str(&mut last_index, index, &mut token, &mut state, None, TokenizerState::SmallCommentStart),
|
|
_ => tokenize_backtrack(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
|
|
}
|
|
}
|
|
TokenizerState::SmallCommentStart =>
|
|
{
|
|
match ch
|
|
{
|
|
'[' =>
|
|
{
|
|
token = None;
|
|
state = TokenizerState::BigCommentLongBracketStart;
|
|
}
|
|
'\n' =>
|
|
{
|
|
state = TokenizerState::Start;
|
|
last_index = -1;
|
|
}
|
|
_ =>
|
|
{
|
|
state = TokenizerState::SmallComment;
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::SmallComment =>
|
|
{
|
|
match ch
|
|
{
|
|
'\n' =>
|
|
{
|
|
state = TokenizerState::Start;
|
|
last_index = -1;
|
|
}
|
|
_ => { }
|
|
}
|
|
}
|
|
TokenizerState::BigCommentLongBracketStart =>
|
|
{
|
|
match ch
|
|
{
|
|
'=' =>
|
|
{
|
|
long_bracket_level += 1;
|
|
}
|
|
'[' =>
|
|
{
|
|
state = TokenizerState::BigComment;
|
|
}
|
|
_ => return Err("Malformed long bracket at the beginning of a big comment"),
|
|
}
|
|
}
|
|
TokenizerState::BigComment =>
|
|
{
|
|
match ch
|
|
{
|
|
']' =>
|
|
{
|
|
state = TokenizerState::BigCommentLongBracketEnd;
|
|
}
|
|
_ => { }
|
|
}
|
|
}
|
|
TokenizerState::BigCommentLongBracketEnd =>
|
|
{
|
|
match ch
|
|
{
|
|
'=' =>
|
|
{
|
|
if long_bracket_level == 0
|
|
{
|
|
return Err("Long bracket level too big when ending big comment");
|
|
}
|
|
long_bracket_level -= 1;
|
|
}
|
|
']' =>
|
|
{
|
|
if long_bracket_level != 0
|
|
{
|
|
return Err("Long bracket level too small when ending big comment");
|
|
}
|
|
state = TokenizerState::Start;
|
|
}
|
|
_ => return Err("Malformed long bracket when ending big comment"),
|
|
}
|
|
}
|
|
TokenizerState::A =>
|
|
{
|
|
match ch
|
|
{
|
|
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("an".to_string())), TokenizerState::An, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::An =>
|
|
{
|
|
match ch
|
|
{
|
|
'd' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("and".to_string())), TokenizerState::And, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::And =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::And);
|
|
}
|
|
}
|
|
TokenizerState::W =>
|
|
{
|
|
match ch
|
|
{
|
|
'h' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("wh".to_string())), TokenizerState::Wh, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::Wh =>
|
|
{
|
|
match ch
|
|
{
|
|
'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("whi".to_string())), TokenizerState::Whi, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Whi =>
|
|
{
|
|
match ch
|
|
{
|
|
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("whil".to_string())), TokenizerState::Whil, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Whil =>
|
|
{
|
|
match ch
|
|
{
|
|
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("while".to_string())), TokenizerState::While, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::While =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::While);
|
|
}
|
|
}
|
|
TokenizerState::B =>
|
|
{
|
|
match ch
|
|
{
|
|
'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("br".to_string())), TokenizerState::Br, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::Br =>
|
|
{
|
|
match ch
|
|
{
|
|
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("bre".to_string())), TokenizerState::Bre, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Bre =>
|
|
{
|
|
match ch
|
|
{
|
|
'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("brea".to_string())), TokenizerState::Brea, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Brea =>
|
|
{
|
|
match ch
|
|
{
|
|
'k' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("break".to_string())), TokenizerState::Break, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Break =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Break);
|
|
}
|
|
}
|
|
TokenizerState::G =>
|
|
{
|
|
match ch
|
|
{
|
|
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("go".to_string())), TokenizerState::Go, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Go =>
|
|
{
|
|
match ch
|
|
{
|
|
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("got".to_string())), TokenizerState::Got, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Got =>
|
|
{
|
|
match ch
|
|
{
|
|
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("goto".to_string())), TokenizerState::Goto, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Goto =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Goto);
|
|
}
|
|
}
|
|
TokenizerState::R =>
|
|
{
|
|
match ch
|
|
{
|
|
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("re".to_string())), TokenizerState::Re, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::Re =>
|
|
{
|
|
match ch
|
|
{
|
|
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("ret".to_string())), TokenizerState::Ret, &mut token_str, ch),
|
|
'p' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("rep".to_string())), TokenizerState::Rep, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Ret =>
|
|
{
|
|
match ch
|
|
{
|
|
'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("retu".to_string())), TokenizerState::Retu, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Retu =>
|
|
{
|
|
match ch
|
|
{
|
|
'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("retur".to_string())), TokenizerState::Retur, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Retur =>
|
|
{
|
|
match ch
|
|
{
|
|
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("return".to_string())), TokenizerState::Return, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Return =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Return);
|
|
}
|
|
}
|
|
TokenizerState::Rep =>
|
|
{
|
|
match ch
|
|
{
|
|
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("repe".to_string())), TokenizerState::Repe, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Repe =>
|
|
{
|
|
match ch
|
|
{
|
|
'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("repea".to_string())), TokenizerState::Repea, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Repea =>
|
|
{
|
|
match ch
|
|
{
|
|
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("repeat".to_string())), TokenizerState::Repeat, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Repeat =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Repeat);
|
|
}
|
|
}
|
|
TokenizerState::N =>
|
|
{
|
|
match ch
|
|
{
|
|
'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("ni".to_string())), TokenizerState::Ni, &mut token_str, ch),
|
|
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("no".to_string())), TokenizerState::No, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::No =>
|
|
{
|
|
match ch
|
|
{
|
|
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("not".to_string())), TokenizerState::Not, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Not =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Not);
|
|
}
|
|
}
|
|
TokenizerState::Ni =>
|
|
{
|
|
match ch
|
|
{
|
|
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("nil".to_string())), TokenizerState::Nil, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Nil =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Nil);
|
|
}
|
|
}
|
|
TokenizerState::T =>
|
|
{
|
|
match ch
|
|
{
|
|
'h' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("th".to_string())), TokenizerState::Th, &mut token_str, ch),
|
|
'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("tr".to_string())), TokenizerState::Tr, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::Th =>
|
|
{
|
|
match ch
|
|
{
|
|
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("the".to_string())), TokenizerState::The, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::The =>
|
|
{
|
|
match ch
|
|
{
|
|
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("then".to_string())), TokenizerState::Then, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Then =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Then);
|
|
}
|
|
}
|
|
TokenizerState::Tr =>
|
|
{
|
|
match ch
|
|
{
|
|
'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("tru".to_string())), TokenizerState::Tru, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Tru =>
|
|
{
|
|
match ch
|
|
{
|
|
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("true".to_string())), TokenizerState::True, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::True =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::True);
|
|
}
|
|
}
|
|
TokenizerState::E =>
|
|
{
|
|
match ch
|
|
{
|
|
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("el".to_string())), TokenizerState::El, &mut token_str, ch),
|
|
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("en".to_string())), TokenizerState::En, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::En =>
|
|
{
|
|
match ch
|
|
{
|
|
'd' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("end".to_string())), TokenizerState::End, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::End =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::End);
|
|
}
|
|
}
|
|
TokenizerState::El =>
|
|
{
|
|
match ch
|
|
{
|
|
's' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("els".to_string())), TokenizerState::Els, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Els =>
|
|
{
|
|
match ch
|
|
{
|
|
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("else".to_string())), TokenizerState::Else, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Else =>
|
|
{
|
|
match ch
|
|
{
|
|
'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("elsei".to_string())), TokenizerState::Elsei, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Else);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::Elsei =>
|
|
{
|
|
match ch
|
|
{
|
|
'f' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("elseif".to_string())), TokenizerState::Elseif, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Elseif =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Elseif);
|
|
}
|
|
}
|
|
TokenizerState::O =>
|
|
{
|
|
match ch
|
|
{
|
|
'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("or".to_string())), TokenizerState::Or, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::Or =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Or);
|
|
}
|
|
}
|
|
TokenizerState::D =>
|
|
{
|
|
match ch
|
|
{
|
|
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("do".to_string())), TokenizerState::Do, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::Do =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Do);
|
|
}
|
|
}
|
|
TokenizerState::I =>
|
|
{
|
|
match ch
|
|
{
|
|
'f' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("if".to_string())), TokenizerState::If, &mut token_str, ch),
|
|
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("in".to_string())), TokenizerState::In, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::In =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::In);
|
|
}
|
|
}
|
|
TokenizerState::If =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::If);
|
|
}
|
|
}
|
|
TokenizerState::F =>
|
|
{
|
|
match ch
|
|
{
|
|
'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fa".to_string())), TokenizerState::Fa, &mut token_str, ch),
|
|
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fo".to_string())), TokenizerState::Fo, &mut token_str, ch),
|
|
'u' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fu".to_string())), TokenizerState::Fu, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::Fu =>
|
|
{
|
|
match ch
|
|
{
|
|
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fun".to_string())), TokenizerState::Fun, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Fun =>
|
|
{
|
|
match ch
|
|
{
|
|
'c' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("func".to_string())), TokenizerState::Func, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Func =>
|
|
{
|
|
match ch
|
|
{
|
|
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("funct".to_string())), TokenizerState::Funct, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Funct =>
|
|
{
|
|
match ch
|
|
{
|
|
'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("functi".to_string())), TokenizerState::Functi, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Functi =>
|
|
{
|
|
match ch
|
|
{
|
|
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("functio".to_string())), TokenizerState::Functio, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Functio =>
|
|
{
|
|
match ch
|
|
{
|
|
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("function".to_string())), TokenizerState::Function, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Function =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Function);
|
|
}
|
|
}
|
|
TokenizerState::Fa =>
|
|
{
|
|
match ch
|
|
{
|
|
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fal".to_string())), TokenizerState::Fal, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Fal =>
|
|
{
|
|
match ch
|
|
{
|
|
's' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("fals".to_string())), TokenizerState::Fals, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Fals =>
|
|
{
|
|
match ch
|
|
{
|
|
'e' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("false".to_string())), TokenizerState::False, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::False =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::False);
|
|
}
|
|
}
|
|
TokenizerState::Fo =>
|
|
{
|
|
match ch
|
|
{
|
|
'r' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("for".to_string())), TokenizerState::For, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::For =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::For);
|
|
}
|
|
}
|
|
TokenizerState::L =>
|
|
{
|
|
match ch
|
|
{
|
|
'o' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("lo".to_string())), TokenizerState::Lo, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::Lo =>
|
|
{
|
|
match ch
|
|
{
|
|
'c' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("loc".to_string())), TokenizerState::Loc, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Loc =>
|
|
{
|
|
match ch
|
|
{
|
|
'a' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("loca".to_string())), TokenizerState::Loca, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Loca =>
|
|
{
|
|
match ch
|
|
{
|
|
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("local".to_string())), TokenizerState::Local, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Local =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Local);
|
|
}
|
|
}
|
|
TokenizerState::U =>
|
|
{
|
|
match ch
|
|
{
|
|
'n' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("un".to_string())), TokenizerState::Un, &mut token_str, ch),
|
|
_ =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
tokens.push(token.clone().unwrap());
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
TokenizerState::Un =>
|
|
{
|
|
match ch
|
|
{
|
|
't' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("unt".to_string())), TokenizerState::Unt, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Unt =>
|
|
{
|
|
match ch
|
|
{
|
|
'i' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("unti".to_string())), TokenizerState::Unti, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Unti =>
|
|
{
|
|
match ch
|
|
{
|
|
'l' => tokenize_terminal(&mut last_index, index, &mut token, &mut state, Some(Token::Name("until".to_string())), TokenizerState::Until, &mut token_str, ch),
|
|
_ => tokenize_alphanumeric_nonterminal(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, ch)?,
|
|
}
|
|
}
|
|
TokenizerState::Until =>
|
|
{
|
|
if ch.is_ascii_alphanumeric() || ch == '_'
|
|
{
|
|
last_index = index as i32;
|
|
token_str.push(ch);
|
|
token = Some(Token::Name(token_str.clone()));
|
|
state = TokenizerState::Name;
|
|
}
|
|
else
|
|
{
|
|
if last_index == -1 || token.is_none()
|
|
{
|
|
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
|
return Err("Lexerr");
|
|
}
|
|
index = last_index as usize;
|
|
last_index = -1;
|
|
token = None;
|
|
token_str.clear();
|
|
state = TokenizerState::Start;
|
|
tokens.push(Token::Until);
|
|
}
|
|
}
|
|
_ => todo!("State: {:?}", state),
|
|
}
|
|
index += 1;
|
|
}
|
|
|
|
return Ok(tokens);
|
|
} |