Simplifications curtesy of clippy

This commit is contained in:
0x4261756D 2023-12-22 22:13:17 +01:00
parent dcfb3147ba
commit 8478b10210
1 changed files with 71 additions and 67 deletions

View File

@ -141,7 +141,7 @@ fn main()
{
print!("---Output---\n'{}'\n", msg);
}
let expected = &format!("//valid,{}:END:", maybe_msg.unwrap_or(String::new()).replace("\n", "\n//"));
let expected = &format!("//valid,{}:END:", maybe_msg.unwrap_or(String::new()).replace('\n', "\n//"));
if file_content.starts_with(expected)
{
println!("===PASSED===");
@ -160,7 +160,7 @@ fn main()
Err(msg) =>
{
println!("ERROR: {}", msg);
if file_content.starts_with(&format!("//invalid,{}:END:", msg.replace("\n", "\n//")))
if file_content.starts_with(&format!("//invalid,{}:END:", msg.replace('\n', "\n//")))
{
println!("===PASSED===");
count += 1;
@ -169,7 +169,7 @@ fn main()
{
if let Some(index) = file_content.find(":END:")
{
let expected = &format!("//invalid,{}:END:", msg.replace("\n", "\n//"));
let expected = &format!("//invalid,{}:END:", msg.replace('\n', "\n//"));
let expected_output = file_content[10..index].replace("\n//", "\n");
println!("\n===FAILED===\nExpected the output to be\n'{}'\n({})", expected_output, expected);
}
@ -206,28 +206,28 @@ fn main()
}
}
fn compile(file_content: &String, file_path: &str, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, interpret: bool, run: bool, debug: bool) -> Result<Option<String>, String>
fn compile(file_content: &str, file_path: &str, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, interpret: bool, run: bool, debug: bool) -> Result<Option<String>, String>
{
let mut tokens: Vec<Token> = tokenize(&file_content)?;
let mut tokens: Vec<Token> = tokenize(file_content)?;
println!("---Done tokenizing, got {} tokens---", tokens.len());
let mut functions: Vec<Function> = Vec::new();
extract_functions(&mut tokens, &mut functions, &intrinsics, debug)?;
extract_functions(&mut tokens, &mut functions, intrinsics, debug)?;
println!("---Done extracting functions, got {} functions and reduced the token count to {}---", functions.len(), tokens.len());
resolve_imports(&mut tokens, &mut functions, file_path, &mut Vec::from([fs::canonicalize(file_path).unwrap()]), intrinsics, debug)?;
println!("---Done importing files---");
let mut arrays: Vec<Arr> = extract_arrays(&mut tokens, &intrinsics, &functions, debug)?;
let mut arrays: Vec<Arr> = extract_arrays(&mut tokens, intrinsics, &functions, debug)?;
println!("---Done extracting arrays, got {} arrays and reduced the token count to {}---", arrays.len(), tokens.len());
let operations = parse_until_delimiter(&mut tokens.iter().peekable(), &intrinsics, None, debug)?;
let operations = parse_until_delimiter(&mut tokens.iter().peekable(), intrinsics, None, debug)?;
println!("---Done parsing tokens into {} operations---", operations.len());
validate_function_calls(&operations, &functions, &arrays, debug)?;
validate_function_calls(&operations, &functions, debug)?;
println!("---Done validating function calls---");
typecheck(&operations, &functions, &intrinsics, &arrays, debug)?;
typecheck(&operations, &functions, intrinsics, &arrays, debug)?;
println!("---Done typechecking---");
let start = Instant::now();
let output = if interpret
{
println!("---Starting to interpret the program---");
Some(interpret_program(&operations, &mut Vec::new(), &functions, &mut arrays, &intrinsics, debug)?)
Some(interpret_program(&operations, &mut Vec::new(), &functions, &mut arrays, debug)?)
}
else
{
@ -235,14 +235,14 @@ fn compile(file_content: &String, file_path: &str, intrinsics: &HashMap<&str, (V
};
if !interpret
{
if let Err(err) = generate_assembly_linux_x64(&operations, &functions, &intrinsics, &arrays, debug)
if let Err(err) = generate_assembly_linux_x64(&operations, &functions, &arrays)
{
return Err(err.to_string());
}
let mut fasm_process = match Command::new("fasm").arg("out.asm").spawn()
{
Ok(process) => process,
Err(err) => return Err(format!("Fasm process error: {}", err.to_string())),
Err(err) => return Err(format!("Fasm process error: {}", err)),
};
match fasm_process.wait()
{
@ -292,7 +292,7 @@ fn compile(file_content: &String, file_path: &str, intrinsics: &HashMap<&str, (V
};
}
println!("---Done after {:?}---", start.elapsed());
return Ok(output);
Ok(output)
}
fn resolve_imports(tokens: &mut Vec<Token>, functions: &mut Vec<Function>, file_path: &str, visited_paths: &mut Vec<PathBuf>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, debug: bool) -> Result<(), String>
@ -305,6 +305,7 @@ fn resolve_imports(tokens: &mut Vec<Token>, functions: &mut Vec<Function>, file_
{
if let Some(Token::StringLit(import_path, _, _)) = tokens_iter.next()
{
println!("Resolving {}", import_path);
let full_import_path = fs::canonicalize(file_path).map_err(|e| e.to_string())?.parent().unwrap().join(import_path);
if visited_paths.contains(&full_import_path)
{
@ -320,7 +321,7 @@ fn resolve_imports(tokens: &mut Vec<Token>, functions: &mut Vec<Function>, file_
{
let mut import_tokens: Vec<Token> = tokenize(&file_content)?;
println!("--Done tokenizing the imported file at {}:{}, got {} tokens--", line, col, tokens.len());
extract_functions(&mut import_tokens, functions, &intrinsics, debug)?;
extract_functions(&mut import_tokens, functions, intrinsics, debug)?;
resolve_imports(&mut import_tokens, functions, file_path, visited_paths, intrinsics, debug)?;
println!("--Now totalling {} functions--", functions.len());
}
@ -340,7 +341,7 @@ fn resolve_imports(tokens: &mut Vec<Token>, functions: &mut Vec<Function>, file_
}
tokens.clear();
tokens.extend_from_slice(&new_tokens);
return Ok(());
Ok(())
}
struct AssemblyData
@ -368,7 +369,7 @@ const ASSEMBLY_LINUX_X64_DYNAMIC_DATA_LENGTH: u32 = 16384;
// r14: base
// r15: dynamic end
fn generate_assembly_linux_x64(operations: &Vec<Operation>, functions: &Vec<Function>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, arrays: &Vec<Arr>, debug: bool) -> Result<(), std::io::Error>
fn generate_assembly_linux_x64(operations: &Vec<Operation>, functions: &Vec<Function>, arrays: &Vec<Arr>) -> Result<(), std::io::Error>
{
let mut data = AssemblyData
{
@ -381,11 +382,11 @@ fn generate_assembly_linux_x64(operations: &Vec<Operation>, functions: &Vec<Func
data.arrays += format!("\tarr_{}: rq {}\n", array.name, array.length).as_str();
}
data.code += "_start:\n";
merge_assemblies(&mut data, generate_assembly_linux_x64_block(operations, functions, intrinsics, arrays, debug));
merge_assemblies(&mut data, generate_assembly_linux_x64_block(operations, functions, arrays));
data.code += ASSEMBLY_LINUX_X64_EXIT;
for function in functions
{
merge_assemblies(&mut data, generate_assembly_linux_x64_function(function.name.as_str(), &function.content, functions, intrinsics, arrays, debug));
merge_assemblies(&mut data, generate_assembly_linux_x64_function(function.name.as_str(), &function.content, functions, arrays));
}
if data.code.contains("call intToStr")
@ -453,7 +454,7 @@ fn generate_assembly_linux_x64(operations: &Vec<Operation>, functions: &Vec<Func
data.code += "\tsyscall\n";
}
return fs::write("out.asm", format!("{}{}{}{}", ASSEMBLY_LINUX_X64_HEADER, data.code, data.arrays, data.strings));
fs::write("out.asm", format!("{}{}{}{}", ASSEMBLY_LINUX_X64_HEADER, data.code, data.arrays, data.strings))
}
fn generate_assembly_linux_x64_array_oob_check(length: i64) -> String
@ -465,7 +466,7 @@ fn generate_assembly_linux_x64_array_oob_check(length: i64) -> String
data += "\tcmp qword rax, 0\n";
data += "\tjl exception_array_read_out_of_bounds\n";
data += "\t\t;;Array bounds check over\n";
return data.clone();
data.clone()
}
fn generate_assembly_linux_x64_queue_oob_check() -> String
@ -475,7 +476,7 @@ fn generate_assembly_linux_x64_queue_oob_check() -> String
"\tjge exception_queue_read_out_of_bounds\n\t\t;;Queue bounds over\n";
}
fn generate_assembly_linux_x64_block(operations: &Vec<Operation>, functions: &Vec<Function>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, arrays: &Vec<Arr>, debug: bool) -> AssemblyData
fn generate_assembly_linux_x64_block(operations: &Vec<Operation>, functions: &Vec<Function>, arrays: &Vec<Arr>) -> AssemblyData
{
let mut data = AssemblyData
{
@ -584,7 +585,7 @@ fn generate_assembly_linux_x64_block(operations: &Vec<Operation>, functions: &Ve
data.code += format!("while_{}_{}:\n", line, col).as_str();
data.code += "\tinc r12\n";
data.code += ASSEMBLY_LINUX_X64_TRY_RESET_QUEUE;
merge_assemblies(&mut data, generate_assembly_linux_x64_block(while_operations, functions, intrinsics, arrays, debug));
merge_assemblies(&mut data, generate_assembly_linux_x64_block(while_operations, functions, arrays));
data.code += "\tcmp qword [queue+8*r12], 0\n";
data.code += format!("\tjne while_{}_{}\n", line, col).as_str();
data.code += format!("while_{}_{}_end:\n", line, col).as_str();
@ -598,14 +599,14 @@ fn generate_assembly_linux_x64_block(operations: &Vec<Operation>, functions: &Ve
data.code += format!("\tje else_{}_{}\n", line, col).as_str();
data.code += "\tinc r12\n";
data.code += ASSEMBLY_LINUX_X64_TRY_RESET_QUEUE;
merge_assemblies(&mut data, generate_assembly_linux_x64_block(if_operations, functions, intrinsics, arrays, debug));
merge_assemblies(&mut data, generate_assembly_linux_x64_block(if_operations, functions, arrays));
data.code += format!("\tjmp if_{}_{}_end\n", line, col).as_str();
data.code += format!("else_{}_{}:\n", line, col).as_str();
data.code += "\tinc r12\n";
data.code += ASSEMBLY_LINUX_X64_TRY_RESET_QUEUE;
if let Some(else_operations) = maybe_else_operations
{
merge_assemblies(&mut data, generate_assembly_linux_x64_block(else_operations, functions, intrinsics, arrays, debug));
merge_assemblies(&mut data, generate_assembly_linux_x64_block(else_operations, functions, arrays));
}
data.code += format!("if_{}_{}_end:\n", line, col).as_str();
}
@ -903,10 +904,10 @@ fn generate_assembly_linux_x64_block(operations: &Vec<Operation>, functions: &Ve
_ => todo!("{:?}", operation)
}
}
return data;
data
}
fn generate_assembly_linux_x64_function(name: &str, operations: &Vec<Operation>, functions: &Vec<Function>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, arrays: &Vec<Arr>, debug: bool) -> AssemblyData
fn generate_assembly_linux_x64_function(name: &str, operations: &Vec<Operation>, functions: &Vec<Function>, arrays: &Vec<Arr>) -> AssemblyData
{
let mut data = AssemblyData
{
@ -914,12 +915,12 @@ fn generate_assembly_linux_x64_function(name: &str, operations: &Vec<Operation>,
code: format!("{}:\n", name),
strings: String::new(),
};
merge_assemblies(&mut data, generate_assembly_linux_x64_block(operations, functions, intrinsics, arrays, debug));
merge_assemblies(&mut data, generate_assembly_linux_x64_block(operations, functions, arrays));
data.code += "\tret\n";
return data;
data
}
fn interpret_program(operations: &Vec<Operation>, queue: &mut Vec<String>, functions: &Vec<Function>, arrays: &mut Vec<Arr>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, debug: bool) -> Result<String,String>
fn interpret_program(operations: &Vec<Operation>, queue: &mut Vec<String>, functions: &Vec<Function>, arrays: &mut Vec<Arr>, debug: bool) -> Result<String,String>
{
let mut output = String::new();
for operation in operations
@ -964,7 +965,7 @@ fn interpret_program(operations: &Vec<Operation>, queue: &mut Vec<String>, funct
let val = queue.remove(0);
function_context.push(val);
}
output += interpret_program(&function.content, function_context, functions, arrays, intrinsics, debug)?.as_str();
output += interpret_program(&function.content, function_context, functions, arrays, debug)?.as_str();
for val in function_context
{
queue.push(val.to_string());
@ -975,11 +976,11 @@ fn interpret_program(operations: &Vec<Operation>, queue: &mut Vec<String>, funct
let val = queue.remove(0);
if val == "true"
{
output += interpret_program(if_block, queue, functions, arrays, intrinsics, debug)?.as_str();
output += interpret_program(if_block, queue, functions, arrays, debug)?.as_str();
}
else if let Some(else_block) = maybe_else_block
{
output += interpret_program(else_block, queue, functions, arrays, intrinsics, debug)?.as_str();
output += interpret_program(else_block, queue, functions, arrays, debug)?.as_str();
}
}
Operation::Intrinsic(intrinsic_name, line, col) =>
@ -988,7 +989,7 @@ fn interpret_program(operations: &Vec<Operation>, queue: &mut Vec<String>, funct
{
"print" =>
{
output += format!("{}", queue.remove(0)).as_str();
output += queue.remove(0).to_string().as_str();
}
"-" =>
{
@ -1132,7 +1133,7 @@ fn interpret_program(operations: &Vec<Operation>, queue: &mut Vec<String>, funct
{
break;
}
output += interpret_program(while_block, queue, functions, arrays, intrinsics, debug)?.as_str();
output += interpret_program(while_block, queue, functions, arrays, debug)?.as_str();
}
}
Operation::Depth(_, _) =>
@ -1151,7 +1152,7 @@ fn interpret_program(operations: &Vec<Operation>, queue: &mut Vec<String>, funct
println!("after: {:?}: {:?}, '{}'", operation, queue, output);
}
}
return Ok(output);
Ok(output)
}
fn typecheck(operations: &Vec<Operation>, functions: &Vec<Function>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, arrays: &Vec<Arr>, debug: bool) -> Result<(), String>
@ -1177,10 +1178,10 @@ fn typecheck(operations: &Vec<Operation>, functions: &Vec<Function>, intrinsics:
{
println!("Successfully typechecked main operations");
}
return Ok(());
Ok(())
}
fn typecheck_block(operations: &Vec<Operation>, ins: &Vec<Datatype>, outs: &Vec<Datatype>, functions: &Vec<Function>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, arrays: &Vec<Arr>, debug: bool) -> Result<(), String>
fn typecheck_block(operations: &Vec<Operation>, ins: &[Datatype], outs: &Vec<Datatype>, functions: &Vec<Function>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, arrays: &Vec<Arr>, debug: bool) -> Result<(), String>
{
let actual_outs = get_return_type(operations, ins, functions, intrinsics, arrays, debug)?;
if &actual_outs != outs
@ -1210,10 +1211,10 @@ fn typecheck_block(operations: &Vec<Operation>, ins: &Vec<Datatype>, outs: &Vec<
};
return Err(format!("Wrong queue state at the end of a block, expected {:?} but got {:?} at {}:{}", outs, actual_outs, line, col));
}
return Ok(());
Ok(())
}
fn get_return_type(operations: &Vec<Operation>, ins: &Vec<Datatype>, functions: &Vec<Function>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, arrays: &Vec<Arr>, debug: bool) -> Result<Vec<Datatype>, String>
fn get_return_type(operations: &Vec<Operation>, ins: &[Datatype], functions: &Vec<Function>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, arrays: &Vec<Arr>, debug: bool) -> Result<Vec<Datatype>, String>
{
let type_queue: &mut Vec<Datatype> = &mut Vec::new();
type_queue.extend_from_slice(ins);
@ -1242,7 +1243,7 @@ fn get_return_type(operations: &Vec<Operation>, ins: &Vec<Datatype>, functions:
{
if let Some(typ) = type_queue.get(0)
{
type_queue.push(typ.clone());
type_queue.push(*typ);
}
else
{
@ -1305,7 +1306,7 @@ fn get_return_type(operations: &Vec<Operation>, ins: &Vec<Datatype>, functions:
{
println!("Starting to typecheck if block");
}
let if_ret = get_return_type(if_block, &type_queue, functions, intrinsics, arrays, debug)?;
let if_ret = get_return_type(if_block, type_queue, functions, intrinsics, arrays, debug)?;
let else_ret =
if let Some(else_block) = maybe_else_block
{
@ -1313,7 +1314,7 @@ fn get_return_type(operations: &Vec<Operation>, ins: &Vec<Datatype>, functions:
{
println!("Starting to typecheck else block");
}
get_return_type(else_block, &type_queue, functions, intrinsics, arrays, debug)?
get_return_type(else_block, type_queue, functions, intrinsics, arrays, debug)?
}
else
{
@ -1414,28 +1415,28 @@ fn get_return_type(operations: &Vec<Operation>, ins: &Vec<Datatype>, functions:
println!("{} => {:?}", debug_string, type_queue);
}
}
return Ok(type_queue.clone());
Ok(type_queue.clone())
}
fn validate_function_calls(operations: &Vec<Operation>, functions: &Vec<Function>, arrays: &Vec<Arr>, debug: bool) -> Result<(), String>
fn validate_function_calls(operations: &Vec<Operation>, functions: &Vec<Function>, debug: bool) -> Result<(), String>
{
for function in functions
{
validate_function_calls_in_block(&function.content, functions, arrays, debug)?;
validate_function_calls_in_block(&function.content, functions)?;
if debug
{
println!("Successfully validated function calls in function '{}'", function.name);
}
}
validate_function_calls_in_block(operations, functions, arrays, debug)?;
validate_function_calls_in_block(operations, functions)?;
if debug
{
println!("Successfully validated function calls in main operations");
}
return Ok(());
Ok(())
}
fn validate_function_calls_in_block(block: &Vec<Operation>, functions: &Vec<Function>, arrays: &Vec<Arr>, debug: bool) -> Result<(), String>
fn validate_function_calls_in_block(block: &Vec<Operation>, functions: &Vec<Function>) -> Result<(), String>
{
for operation in block
{
@ -1452,22 +1453,22 @@ fn validate_function_calls_in_block(block: &Vec<Operation>, functions: &Vec<Func
}
Operation::If(if_block, maybe_else_block, _, _) =>
{
validate_function_calls_in_block(if_block, functions, arrays, debug)?;
validate_function_calls_in_block(if_block, functions)?;
if let Some(else_block) = maybe_else_block
{
validate_function_calls_in_block(else_block, functions, arrays, debug)?;
validate_function_calls_in_block(else_block, functions)?;
}
}
Operation::While(while_block, _, _) =>
{
validate_function_calls_in_block(while_block, functions, arrays, debug)?;
validate_function_calls_in_block(while_block, functions)?;
}
}
}
return Ok(());
Ok(())
}
fn extract_arrays(tokens: &mut Vec<Token>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, functions: &Vec<Function>, debug: bool) -> Result<Vec<Arr>, String>
fn extract_arrays(tokens: &mut Vec<Token>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, functions: &[Function], debug: bool) -> Result<Vec<Arr>, String>
{
let mut tokens_iter = tokens.iter().peekable();
let mut arrays: Vec<Arr> = Vec::new();
@ -1560,13 +1561,16 @@ fn extract_arrays(tokens: &mut Vec<Token>, intrinsics: &HashMap<&str, (Vec<Datat
}
tokens.clear();
tokens.extend_from_slice(&new_tokens);
return Ok(arrays);
Ok(arrays)
}
fn sanitize_name(name: String) -> String
{
if name == "test" {return "test_".to_string();}
return name.replace("-", "_").replace("+", "_").replace("%", "percent").replace("/", "slash");
if name == "test"
{
return "test_".to_string();
}
name.replace(['-', '+'], "_").replace('%', "percent").replace('/', "slash")
}
fn str_to_datatype(s: &str, line: i32, col: i32) -> Result<Datatype, String>
@ -1577,7 +1581,7 @@ fn str_to_datatype(s: &str, line: i32, col: i32) -> Result<Datatype, String>
"bool" => Ok(Datatype::Bool),
"int" => Ok(Datatype::Int),
"str" => Ok(Datatype::String),
_ => return Err(format!("Expected a datatype for the array, got {} instead at {}:{}", s, line, col))
_ => Err(format!("Expected a datatype for the array, got {} instead at {}:{}", s, line, col))
}
}
@ -1627,7 +1631,7 @@ fn extract_functions(tokens: &mut Vec<Token>, functions: &mut Vec<Function>, int
}
}
}
None => return Err(format!("Unexpected end of file while extracting a function"))
None => return Err("Unexpected end of file while extracting a function".to_string())
}
}
if debug
@ -1680,7 +1684,7 @@ fn extract_functions(tokens: &mut Vec<Token>, functions: &mut Vec<Function>, int
}
}
}
None => return Err(format!("Unexpected end of file while extracting a function"))
None => return Err("Unexpected end of file while extracting a function".to_string())
}
}
}
@ -1696,7 +1700,7 @@ fn extract_functions(tokens: &mut Vec<Token>, functions: &mut Vec<Function>, int
}
tokens.clear();
tokens.extend_from_slice(&new_tokens);
return Ok(());
Ok(())
}
fn parse_block(tokens_iter: &mut Peekable<std::slice::Iter<Token>>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, debug: bool) -> Result<Vec<Operation>, String>
@ -1710,9 +1714,9 @@ fn parse_block(tokens_iter: &mut Peekable<std::slice::Iter<Token>>, intrinsics:
}
else
{
return Err(format!("Expected '{{' to open a block"));
return Err("Expected '{' to open a block".to_string());
}
return parse_until_delimiter(tokens_iter, intrinsics, Some("}"), debug);
parse_until_delimiter(tokens_iter, intrinsics, Some("}"), debug)
}
fn parse_until_delimiter(tokens_iter: &mut Peekable<std::slice::Iter<Token>>, intrinsics: &HashMap<&str, (Vec<Datatype>, Vec<Datatype>)>, delimiter: Option<&str>, debug: bool) -> Result<Vec<Operation>, String>
@ -1826,7 +1830,7 @@ fn parse_until_delimiter(tokens_iter: &mut Peekable<std::slice::Iter<Token>>, in
{
if delimiter.is_some()
{
return Err(format!("Reached the end of the file while parsing a block"));
return Err("Reached the end of the file while parsing a block".to_string());
}
else
{
@ -1906,7 +1910,7 @@ fn tokenize(text: &str) -> Result<Vec<Token>, String>
state = TokenizerState::Whitespace;
if application_name.is_empty()
{
if let Ok(_) = word.parse::<i64>()
if word.parse::<i64>().is_ok()
{
tokens.push(Token::IntLit(word.clone(), line, col));
}
@ -1934,7 +1938,7 @@ fn tokenize(text: &str) -> Result<Vec<Token>, String>
{
match ch
{
'"' => return Err(format!("Having '\"' in the middle of a word is not allowed")),
'"' => return Err("Having '\"' in the middle of a word is not allowed".to_string()),
'.' =>
{
application_name = word.clone();
@ -1959,7 +1963,7 @@ fn tokenize(text: &str) -> Result<Vec<Token>, String>
{
TokenizerState::Quote =>
{
return Err(format!("Encountered EOF before closing string"));
return Err("Encountered EOF before closing string".to_string());
}
TokenizerState::Whitespace | TokenizerState::Comment => {},
TokenizerState::Keyword =>