From 0ca8a731a9ea652cd7a053b3cb221c5ef77452d4 Mon Sep 17 00:00:00 2001 From: 0x4261756D <–38735823+0x4261756D@users.noreply.github.com> Date: Thu, 20 Jul 2023 15:20:28 +0200 Subject: [PATCH] Implement LL-like parser --- src/parser.rs | 666 +++++++++++++++++++++++++++++++++++++++++++++-- src/tokenizer.rs | 33 ++- 2 files changed, 667 insertions(+), 32 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 31e605e..79e5a69 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -44,12 +44,42 @@ pub enum StatNode #[derive(Debug)] pub struct RetstatNode { - values: ExplistNode + values: Option } #[derive(Debug)] -pub struct ExpNode +pub enum ExpNode { - + Nil, + False, + True, + Numeral(f64), + LiteralString(String), + Varargs, + Functiondef(FuncbodyNode), + Suffixexp(Box), + Tableconstructor(TableconstructorNode), + Unop(UnopType, Box), + Binop { lhs: Box, op: BinopType, rhs: Box } +} +#[derive(Debug)] +pub enum UnopType +{ + Minus, LogicalNot, Length, BinaryNot, +} +#[derive(Debug)] +pub enum BinopType +{ + LogicalOr, + LocicalAnd, + Lt, Gt, LtEquals, GtEquals, NotEquals, Equals, + BinaryOr, + BinaryNot, + BinaryAnd, + Shl, Shr, + Concat, + Add, Sub, + Mul, Div, IntDiv, Mod, + Exp, } #[derive(Debug)] pub struct ExplistNode @@ -57,6 +87,23 @@ pub struct ExplistNode exps: Vec } #[derive(Debug)] +pub struct TableconstructorNode +{ + exps: Option +} +#[derive(Debug)] +pub struct FieldlistNode +{ + exps: Vec +} +#[derive(Debug)] +pub enum FieldNode +{ + IndexedAssignment { index: ExpNode, rhs: ExpNode }, + Assignment { lhs: String, rhs: ExpNode }, + Exp(ExpNode), +} +#[derive(Debug)] pub struct VarlistNode { vars: Vec @@ -69,9 +116,11 @@ pub struct FunctioncallNode args: ArgsNode, } #[derive(Debug)] -pub struct ArgsNode +pub enum ArgsNode { - + Bracketed(Option), + Tableconstructor(TableconstructorNode), + Literal(String), } #[derive(Debug)] pub struct ElseifNode @@ -82,28 +131,59 @@ pub struct ElseifNode #[derive(Debug)] pub struct FuncnameNode { - -}#[derive(Debug)] + name: String, + dotted_names: Vec, + first_arg: Option, +} +#[derive(Debug)] +pub struct ParlistNode +{ + names: Vec, + has_varargs: bool, +} +#[derive(Debug)] pub struct FuncbodyNode { - + pars: Option, + body: BlockNode, } #[derive(Debug)] pub struct AttnamelistNode { - + attnames: Vec +} +#[derive(Debug)] +pub struct AttnameNode +{ + name: String, + attribute: Option, } #[derive(Debug)] pub enum VarNode { Name(String), - Indexed { value: PrefixexpNode, index: ExpNode }, - Member { value: PrefixexpNode, name: String } + Indexed { value: SuffixexpNode, index: ExpNode }, + Member { value: SuffixexpNode, name: String } } #[derive(Debug)] -pub enum PrefixexpNode +pub struct SuffixexpNode { - + first_part: SuffixexpFirstPart, + suffixes: Vec, +} +#[derive(Debug)] +pub enum SuffixexpFirstPart // a:b:test() => a:b.test(b) => a.b.test(a, b) +{ + Name(String), + BracketedExpr(ExpNode), +} +#[derive(Debug)] +pub enum SuffixexpSuffix +{ + Dot(String), + Indexed(ExpNode), + Args(ArgsNode), + ArgsFirstArg(String, ArgsNode), } fn parse_chunk(tokens: &Vec, i: &mut usize) -> Result { @@ -112,7 +192,8 @@ fn parse_chunk(tokens: &Vec, i: &mut usize) -> Result, i: &mut usize) -> Result { let mut stats: Vec = Vec::new(); - while *i < tokens.len() && tokens[*i] != Token::Return + while *i < tokens.len() && tokens[*i] != Token::Return && tokens[*i] != Token::End && tokens[*i] != Token::Elseif && + tokens[*i] != Token::Else { stats.push(parse_stat(tokens, i)?); } @@ -416,11 +497,11 @@ fn parse_stat(tokens: &Vec, i: &mut usize) -> Result { *i += 1; - return Ok(StatNode::Assignment { lhs: VarlistNode { vars: Vec::from([suffix_expression_to_var(suffix_expression)]) }, rhs: parse_explist(tokens, i)? }); + return Ok(StatNode::Assignment { lhs: VarlistNode { vars: Vec::from([suffix_expression_to_var(suffix_expression)?]) }, rhs: parse_explist(tokens, i)? }); } Token::Comma => { - let mut vars = Vec::from([suffix_expression_to_var(suffix_expression)]); + let mut vars = Vec::from([suffix_expression_to_var(suffix_expression)?]); while tokens[*i] == Token::Comma { *i += 1; @@ -435,16 +516,72 @@ fn parse_stat(tokens: &Vec, i: &mut usize) -> Result { - if let SuffixexpNode() + if suffix_expression.suffixes.is_empty() + { + println!("{:?} {} {:?}", tokens[*i], i, suffix_expression); + return Err("Expected function call but suffix is empty"); + } + if let Some(SuffixexpSuffix::Args(_)) = suffix_expression.suffixes.last() + { + return Ok(StatNode::Functioncall(suffix_expression_to_functioncall(suffix_expression)?)); + } + if let Some(SuffixexpSuffix::ArgsFirstArg(_, _)) = suffix_expression.suffixes.last() + { + return Ok(StatNode::Functioncall(suffix_expression_to_functioncall(suffix_expression)?)); + } + else + { + println!("{:?} {} {:?}", tokens[*i], i, suffix_expression.suffixes.last()); + return Err("Expected function call"); + } } } } - _ => Err("Unexpected token while parsing stat") + _ => + { + println!("{:?} {:?} {:?}", tokens[*i - 2], tokens[*i - 1], tokens[*i]); + Err("Unexpected token while parsing stat") + } } } -fn suffix_expression_to_var(suffixexp: SuffixexpNode) -> VarNode +fn suffix_expression_to_functioncall(suffixexp: SuffixexpNode) -> Result { - todo!() + let mut new_suffixexp = suffixexp; + let last = new_suffixexp.suffixes.pop(); + if let Some(SuffixexpSuffix::Args(args)) = last + { + return Ok(FunctioncallNode { function: new_suffixexp, object_arg: None, args }); + } + if let Some(SuffixexpSuffix::ArgsFirstArg(first_arg, args)) = last + { + return Ok(FunctioncallNode { function: new_suffixexp, object_arg: Some(first_arg.clone()), args }); + } + return Err("Cannot convert suffixexp to functioncall"); +} +fn suffix_expression_to_var(suffixexp: SuffixexpNode) -> Result +{ + if suffixexp.suffixes.is_empty() + { + return if let SuffixexpFirstPart::Name(name) = suffixexp.first_part + { + Ok(VarNode::Name(name.clone())) + } + else + { + Err("Can only convert suffix exp without suffix to var if its first part is a name") + }; + } + let mut new_suffixexp = suffixexp; + let last = new_suffixexp.suffixes.pop(); + if let Some(SuffixexpSuffix::Dot(name)) = last + { + return Ok(VarNode::Member { value: new_suffixexp, name: name.clone() }); + } + if let Some(SuffixexpSuffix::Indexed(index)) = last + { + return Ok(VarNode::Indexed { value: new_suffixexp, index: index }); + } + return Err("Cannot convert suffixexp to var"); } fn parse_var(tokens: &Vec, i: &mut usize) -> Result { @@ -452,36 +589,509 @@ fn parse_var(tokens: &Vec, i: &mut usize) -> Result, i: &mut usize) -> Result { - todo!() + if *i > tokens.len() + { + return Err("Reached end of tokens while parsing args"); + } + match &tokens[*i] + { + Token::RoundOpen => + { + *i += 1; + if *i >= tokens.len() + { + return Err("Reached end of tokens while paring bracketed args"); + } + if tokens[*i] == Token::RoundClosed + { + *i += 1; + return Ok(ArgsNode::Bracketed(None)); + } + let exps = parse_explist(tokens, i)?; + if *i >= tokens.len() || tokens[*i] != Token::RoundClosed + { + println!("|{:?}|{}|{:?}|", tokens[*i], i, exps); + return Err("Expected ')' to close bracketed args"); + } + *i += 1; + return Ok(ArgsNode::Bracketed(Some(exps))); + } + Token::CurlyOpen => + { + return Ok(ArgsNode::Tableconstructor(parse_tableconstructor(tokens, i)?)); + } + Token::StringLiteral(name) => + { + *i += 1; + return Ok(ArgsNode::Literal(name.clone())); + } + _ => return Err("Unexpected token while parsing args") + } } fn parse_suffixexp(tokens: &Vec, i: &mut usize) -> Result { // primaryexp { '.' 'Name' | '[' exp']' | ':' 'Name' args | args } - todo!() + if *i >= tokens.len() + { + return Err("Reached end of tokens while parsing suffixexp"); + } + let first_part = match &tokens[*i] + { + Token::Name(name) => + { + *i += 1; + SuffixexpFirstPart::Name(name.clone()) + }, + Token::RoundOpen => + { + *i += 1; + let ret = SuffixexpFirstPart::BracketedExpr(parse_exp(tokens, i)?); + if *i >= tokens.len() || tokens[*i] != Token::RoundClosed + { + return Err("Expected ')' to close bracketed primary expression"); + } + *i += 1; + ret + } + _ => return Err("Unexpected token as first part of suffixexp") + }; + let mut suffixes = Vec::new(); + while *i < tokens.len() + { + match tokens[*i] + { + Token::Dot => + { + *i += 1; + if *i >= tokens.len() + { + return Err("Reached end of tokens but expected name for dotted suffix expression"); + } + if let Token::Name(name) = &tokens[*i] + { + *i += 1; + suffixes.push(SuffixexpSuffix::Dot(name.clone())); + } + else + { + return Err("Expected name for dotted suffix expression"); + } + } + Token::SquareOpen => + { + *i += 1; + suffixes.push(SuffixexpSuffix::Indexed(parse_exp(tokens, i)?)); + if *i >= tokens.len() || tokens[*i] != Token::SquareClosed + { + return Err("Expected ']' to close indexed suffix expression"); + } + *i += 1; + } + Token::Colon => + { + *i += 1; + if *i >= tokens.len() + { + return Err("Reached end of tokens but expected name for dotted suffix expression"); + } + if let Token::Name(name) = &tokens[*i] + { + *i += 1; + suffixes.push(SuffixexpSuffix::ArgsFirstArg(name.clone(), parse_args(tokens, i)?)); + } + else + { + return Err("Expected name for dotted suffix expression"); + } + } + Token::RoundOpen | Token::CurlyOpen | Token::StringLiteral(_) => + { + suffixes.push(SuffixexpSuffix::Args(parse_args(tokens, i)?)); + } + _ => break, + } + } + return Ok(SuffixexpNode { first_part, suffixes }); } fn parse_retstat(tokens: &Vec, i: &mut usize) -> Result { - todo!("{:?}", tokens[*i]) + if *i >= tokens.len() || tokens[*i] != Token::Return + { + return Err("Expected 'return' to start retstat"); + } + *i += 1; + if *i >= tokens.len() || tokens[*i] == Token::Semicolon || tokens[*i] == Token::Else || tokens[*i] == Token::Elseif || + tokens[*i] == Token::End + { + if *i < tokens.len() && tokens[*i] == Token::Semicolon + { + *i += 1; + } + return Ok(RetstatNode { values: None }); + } + let values = parse_explist(tokens, i)?; + if *i < tokens.len() && tokens[*i] == Token::Semicolon + { + *i += 1; + } + return Ok(RetstatNode { values: Some(values) }); } fn parse_exp(tokens: &Vec, i: &mut usize) -> Result { - todo!("{:?}", tokens[*i]) + let lhs = parse_exp_primary(tokens, i)?; + return parse_exp_precedence(tokens, i, lhs, 0); +} +fn get_precedence(token: &Token) -> Result +{ + match token + { + Token::Or => Ok(2), + Token::And => Ok(4), + Token::Lt | Token::Gt | Token::LtEquals | Token::GtEquals | Token::TildeEquals | Token::EqualsEquals => Ok(6), + Token::Pipe => Ok(8), + Token::Tilde => Ok(10), + Token::Ampersand => Ok(12), + Token::LtLt | Token::GtGt => Ok(14), + Token::DotDot => Ok(16), + Token::Plus | Token::Minus => Ok(18), + Token::Star | Token::Slash | Token::SlashSlash | Token::Percent => Ok(20), + Token::Caret => Ok(22), + _ => Err("Tried to get precedence for unknown operator"), + } +} +fn get_binop(token: &Token) -> Result +{ + match token + { + Token::Or => Ok(BinopType::LogicalOr), + Token::And => Ok(BinopType::LocicalAnd), + Token::Lt => Ok(BinopType::Lt), + Token::Gt => Ok(BinopType::Lt), + Token::LtEquals => Ok(BinopType::LtEquals), + Token::GtEquals => Ok(BinopType::GtEquals), + Token::TildeEquals => Ok(BinopType::NotEquals), + Token::EqualsEquals => Ok(BinopType::Equals), + Token::Pipe => Ok(BinopType::BinaryOr), + Token::Tilde => Ok(BinopType::BinaryNot), + Token::Ampersand => Ok(BinopType::BinaryAnd), + Token::DotDot => Ok(BinopType::Concat), + Token::Plus => Ok(BinopType::Add), + Token::Minus => Ok(BinopType::Sub), + Token::Star => Ok(BinopType::Mul), + Token::Slash => Ok(BinopType::Div), + Token::SlashSlash => Ok(BinopType::IntDiv), + Token::Percent => Ok(BinopType::Mod), + Token::Caret => Ok(BinopType::Exp), + _ => + { + println!("{:?}", token); + Err("Tried to get binop type for unknown operator") + } + } +} +fn is_binop(token: &Token) -> bool +{ + match token + { + Token::Or | Token::And | Token::Lt | Token::Gt | Token::LtEquals | Token::GtEquals | Token::TildeEquals | Token::EqualsEquals | + Token::Pipe | Token::Tilde | Token::Ampersand | Token::LtLt | Token::GtGt | Token::DotDot | Token::Plus | Token::Minus | + Token::Star | Token::Slash | Token::SlashSlash | Token::Percent | Token::Caret => + { + true + } + _ => false + } +} +fn is_right_associative(token: &Token) -> bool +{ + return token == &Token::DotDot || token == &Token::Caret; +} +fn parse_exp_precedence(tokens: &Vec, i: &mut usize, lhs: ExpNode, min_precedence: u8) -> Result +{ + let mut lhs = lhs; + while *i < tokens.len() && is_binop(&tokens[*i]) + { + let precedence = get_precedence(&tokens[*i])?; + if precedence < min_precedence + { + break; + } + let op = get_binop(&tokens[*i])?; + *i += 1; + let mut rhs = parse_exp_primary(tokens, i)?; + while *i < tokens.len() && is_binop(&tokens[*i]) && (get_precedence(&tokens[*i])? > precedence || + (get_precedence(&tokens[*i])? == precedence && is_right_associative(&tokens[*i]))) + { + rhs = parse_exp_precedence(tokens, i, rhs, precedence + if precedence == get_precedence(&tokens[*i])? {0} else {1})?; + } + lhs = ExpNode::Binop { lhs: Box::new(lhs), op, rhs: Box::new(rhs) }; + } + return Ok(lhs); +} +fn parse_exp_primary(tokens: &Vec, i: &mut usize) -> Result +{ + if *i >= tokens.len() + { + return Err("Reached end of tokens but expected primary expression"); + } + match &tokens[*i] + { + Token::Nil => + { + *i += 1; + Ok(ExpNode::Nil) + }, + Token::True => + { + *i += 1; + Ok(ExpNode::True) + }, + Token::False => + { + *i += 1; + Ok(ExpNode::False) + }, + Token::Numeral(number_str) => + { + *i += 1; + Ok(ExpNode::Numeral(number_str.parse::().map_err(|_| "Could not parse number")?)) + }, + Token::StringLiteral(string) => + { + *i += 1; + Ok(ExpNode::LiteralString(string.clone())) + }, + Token::DotDotDot => + { + *i += 1; + Ok(ExpNode::Varargs) + }, + Token::Function => + { + *i += 1; + Ok(ExpNode::Functiondef(parse_funcbody(tokens, i)?)) + } + Token::CurlyOpen => Ok(ExpNode::Tableconstructor(parse_tableconstructor(tokens, i)?)), + Token::Minus => + { + Ok(ExpNode::Unop(UnopType::Minus, Box::new(parse_exp(tokens, i)?))) + } + Token::Hash => + { + Ok(ExpNode::Unop(UnopType::Length, Box::new(parse_exp(tokens, i)?))) + } + Token::Not => + { + Ok(ExpNode::Unop(UnopType::LogicalNot, Box::new(parse_exp(tokens, i)?))) + } + Token::Tilde => + { + Ok(ExpNode::Unop(UnopType::BinaryNot, Box::new(parse_exp(tokens, i)?))) + } + _ => Ok(ExpNode::Suffixexp(Box::new(parse_suffixexp(tokens, i)?))), + } +} +fn parse_tableconstructor(tokens: &Vec, i: &mut usize) -> Result +{ + todo!() } fn parse_explist(tokens: &Vec, i: &mut usize) -> Result { - todo!("{:?}", tokens[*i]) + let mut exps: Vec = Vec::from([parse_exp(tokens, i)?]); + while *i < tokens.len() && tokens[*i] == Token::Comma + { + *i += 1; + exps.push(parse_exp(tokens, i)?); + } + return Ok(ExplistNode { exps }); } fn parse_funcname(tokens: &Vec, i: &mut usize) -> Result { - todo!("{:?}", tokens[*i]) + if *i >= tokens.len() + { + return Err("Reached end of tokens while parsing funcname"); + } + if let Token::Name(name) = &tokens[*i] + { + *i += 1; + let mut dotted_names = Vec::new(); + while *i < tokens.len() && tokens[*i] == Token::Dot + { + *i += 1; + if *i >= tokens.len() + { + return Err("Reached end of tokens while parsing dotted part of funcname"); + } + if let Token::Name(dotted_name) = &tokens[*i] + { + *i += 1; + dotted_names.push(dotted_name.clone()); + } + else + { + return Err("Expected name in dotted funcname"); + } + } + let first_arg = if *i < tokens.len() && tokens[*i] == Token::Colon + { + *i += 1; + if *i >= tokens.len() + { + return Err("Reached end of tokens while parsing funcname first arg"); + } + if let Token::Name(arg_name) = &tokens[*i] + { + *i += 1; + Some(arg_name.clone()) + } + else + { + return Err("Expected name of first arg in funcname"); + } + } + else + { + None + }; + return Ok(FuncnameNode { name: name.clone(), dotted_names, first_arg }); + } + else + { + return Err("Expected func name"); + } } fn parse_funcbody(tokens: &Vec, i: &mut usize) -> Result { - todo!("{:?}", tokens[*i]) + if *i >= tokens.len() || tokens[*i] != Token::RoundOpen + { + return Err("Expected '(' to start funcbody"); + } + *i += 1; + if *i >= tokens.len() + { + return Err("Reached end of tokens while parsing funcbody parlist"); + } + let pars = if tokens[*i] == Token::RoundClosed + { + *i += 1; + None + } + else + { + let ret = Some(parse_parlist(tokens, i)?); + if *i >= tokens.len() || tokens[*i] != Token::RoundClosed + { + return Err("Expected ')' to close funcbody parlist"); + } + *i += 1; + ret + }; + let block = parse_block(tokens, i)?; + if *i >= tokens.len() || tokens[*i] != Token::End + { + println!("{:?}", &tokens[(*i - 10)..(*i + 10)]); + return Err("Expected 'end' to close funcbody"); + } + *i += 1; + return Ok(FuncbodyNode { pars, body: block }); +} +fn parse_parlist(tokens: &Vec, i: &mut usize) -> Result +{ + if *i >= tokens.len() + { + return Err("Reached end of tokens while parsing parlist"); + } + if tokens[*i] == Token::DotDotDot + { + *i += 1; + return Ok(ParlistNode { names: Vec::new(), has_varargs: true }); + } + let first_name = if let Token::Name(name) = &tokens[*i] + { + *i += 1; + name.clone() + } + else + { + return Err("Expected name to start parlist"); + }; + let mut names = Vec::from([first_name]); + let mut has_varargs = false; + while *i < tokens.len() && tokens[*i] == Token::Comma + { + *i += 1; + if *i >= tokens.len() + { + return Err("Reached end of tokens while parsing parlist name list"); + } + match &tokens[*i] + { + Token::Name(name) => + { + *i += 1; + names.push(name.clone()); + } + Token::DotDotDot => + { + *i += 1; + has_varargs = true; + break; + } + _ => return Err("Unexpected token while parsing parlist name list"), + } + } + return Ok(ParlistNode { names, has_varargs }); } fn parse_attnamelist(tokens: &Vec, i: &mut usize) -> Result { - todo!("{:?}", tokens[*i]) + let mut attnames: Vec = Vec::from([parse_attname(tokens, i)?]); + while *i < tokens.len() && tokens[*i] == Token::Comma + { + *i += 1; + attnames.push(parse_attname(tokens, i)?); + } + return Ok(AttnamelistNode { attnames }); +} +fn parse_attname(tokens: &Vec, i: &mut usize) -> Result +{ + if *i >= tokens.len() + { + return Err("Reached end of tokens but expected name for attrib name"); + } + if let Token::Name(name) = &tokens[*i] + { + *i += 1; + let attribute = if *i < tokens.len() && tokens[*i] == Token::Lt + { + *i += 1; + if *i >= tokens.len() + { + return Err("Reached end of tokens but expected attribute"); + } + if let Token::Name(attrib) = &tokens[*i] + { + *i += 1; + if *i >= tokens.len() || tokens[*i] != Token::Gt + { + return Err("Exptected '>' to close attribute name"); + } + Some(attrib.clone()) + } + else + { + return Err("Expected attribute in attrib name"); + } + } + else + { + None + }; + return Ok(AttnameNode { name: name.clone(), attribute }); + } + else + { + return Err("Expected name for attrib name"); + } } //=============================================================================================================================================== //=============================================================================================================================================== diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 8fee3e8..a60f3ff 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1,4 +1,4 @@ -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum Token { Name(String), @@ -186,11 +186,11 @@ fn tokenize_char(state: &mut TokenizerState, ch: char, last_index: &mut i32, ind if ch.is_whitespace() { } else if ch.is_ascii_alphabetic() || ch == '_' { - tokenize_terminal(last_index, *index, token, state, Some(Token::Name(token_str.clone())), TokenizerState::Name, token_str, ch); + tokenize_terminal(last_index, *index, token, state, Some(Token::Name(ch.to_string())), TokenizerState::Name, token_str, ch); } else if ch.is_numeric() && ch.is_ascii() { - tokenize_terminal(last_index, *index, token, state, Some(Token::Numeral(token_str.clone())), TokenizerState::Number, token_str, ch); + tokenize_terminal(last_index, *index, token, state, Some(Token::Numeral(ch.to_string())), TokenizerState::Number, token_str, ch); } else { @@ -1178,7 +1178,32 @@ pub fn tokenize(file_content: &String) -> Result, &'static str> TokenizerState::End => tokenize_backtrack_custom_token(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, Token::End)?, TokenizerState::And => tokenize_backtrack_custom_token(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, Token::And)?, TokenizerState::Semicolon => tokenize_backtrack_custom_token(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, Token::Semicolon)?, - _ => todo!("state: {:?}", state), + TokenizerState::Number => + { + if let Some(numeral_token) = token + { + if let Token::Numeral(_) = numeral_token + { + tokens.push(numeral_token); + } + else + { + return Err("In number state but current token is not a numeral") + } + } + else + { + return Err("In number state but no current token") + } + } + TokenizerState::Start => + { + if token.is_some() + { + return Err("Finished tokenizing in the start state but the token was non-empty"); + } + } + _ => todo!("state: {:?} {:?}", state, token), } return Ok(tokens);