0x4261756D/luaaaaah
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

better command line handling + big formatting stuff

Browse Source
This commit is contained in:
0x4261756D 2024-01-29 14:39:22 +01:00
parent 34cb88582d
commit 1646a79055
3 changed files with 357 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
.editorconfig
View File

@ -1,11 +1,94 @@
root = true root = true
[*]
indent_size = 4 indent_size = 4
indent_style = tab indent_style = tab
trim_trailing_whitespace = true
end_of_line = lf
insert_final_newline = true
charset = utf-8
[*.cs] [*.cs]
trim_trailing_whitespace = true
csharp_space_after_keywords_in_control_flow_statements = false csharp_space_after_keywords_in_control_flow_statements = false
csharp_indent_case_contents_when_block = false csharp_indent_case_contents_when_block = false
csharp_style_unused_value_expression_preference = unused_local_variable
csharp_prefer_braces = true
csharp_prefer_static_local_function = true
dotnet_style_prefer_foreach_explicit_cast_in_source = always
dotnet_diagnostic.IDE0001.severity = warning
dotnet_diagnostic.IDE0002.severity = warning
dotnet_diagnostic.IDE0004.severity = warning
dotnet_diagnostic.IDE0005.severity = warning
dotnet_diagnostic.IDE0011.severity = warning
dotnet_diagnostic.IDE0020.severity = warning
dotnet_diagnostic.IDE0028.severity = warning
dotnet_diagnostic.IDE0029.severity = warning
dotnet_diagnostic.IDE0030.severity = warning
dotnet_diagnostic.IDE0031.severity = warning
dotnet_diagnostic.IDE0035.severity = error
dotnet_diagnostic.IDE0038.severity = error
dotnet_diagnostic.IDE0041.severity = warning
dotnet_diagnostic.IDE0042.severity = warning
dotnet_diagnostic.IDE0051.severity = warning
dotnet_diagnostic.IDE0052.severity = warning
dotnet_diagnostic.IDE0054.severity = warning
dotnet_diagnostic.IDE0050.severity = warning
dotnet_diagnostic.IDE0056.severity = warning
dotnet_diagnostic.IDE0057.severity = warning
dotnet_diagnostic.IDE0058.severity = warning
dotnet_diagnostic.IDE0060.severity = warning
dotnet_diagnostic.IDE0062.severity = warning
dotnet_diagnostic.IDE0066.severity = warning
dotnet_diagnostic.IDE0071.severity = warning
dotnet_diagnostic.IDE0074.severity = warning
dotnet_diagnostic.IDE0075.severity = warning
dotnet_diagnostic.IDE0078.severity = warning
dotnet_diagnostic.IDE0080.severity = warning
dotnet_diagnostic.IDE0083.severity = warning
dotnet_diagnostic.IDE0090.severity = warning
dotnet_diagnostic.IDE0100.severity = warning
dotnet_diagnostic.IDE0180.severity = warning
dotnet_diagnostic.IDE0200.severity = warning
dotnet_diagnostic.IDE0220.severity = warning
dotnet_diagnostic.IDE0260.severity = warning
dotnet_diagnostic.IDE0270.severity = warning
dotnet_diagnostic.CA1508.severity = warning
dotnet_diagnostic.CA1514.severity = warning
dotnet_diagnostic.CA1515.severity = warning
dotnet_diagnostic.CA1801.severity = warning
dotnet_diagnostic.CA1802.severity = warning
dotnet_diagnostic.CA1805.severity = warning
dotnet_diagnostic.CA1806.severity = warning
dotnet_diagnostic.CA1810.severity = warning
dotnet_diagnostic.CA1812.severity = warning
dotnet_diagnostic.CA1814.severity = warning
dotnet_diagnostic.CA1820.severity = warning
dotnet_diagnostic.CA1822.severity = warning
dotnet_diagnostic.CA1823.severity = warning
dotnet_diagnostic.CA1825.severity = warning
dotnet_diagnostic.CA1826.severity = error
dotnet_diagnostic.CA1827.severity = error
dotnet_diagnostic.CA1829.severity = error
dotnet_diagnostic.CA1830.severity = error
dotnet_diagnostic.CA1833.severity = warning
dotnet_diagnostic.CA1834.severity = warning
dotnet_diagnostic.CA1835.severity = warning
dotnet_diagnostic.CA1836.severity = warning
dotnet_diagnostic.CA1841.severity = error
dotnet_diagnostic.CA1845.severity = warning
dotnet_diagnostic.CA1846.severity = warning
dotnet_diagnostic.CA1847.severity = warning
dotnet_diagnostic.CA1849.severity = warning
dotnet_diagnostic.CA1850.severity = warning
dotnet_diagnostic.CA1851.severity = warning
dotnet_diagnostic.CA1853.severity = warning
dotnet_diagnostic.CA1859.severity = warning
dotnet_diagnostic.CA1860.severity = warning
dotnet_diagnostic.CA1861.severity = warning
dotnet_diagnostic.CA1864.severity = warning
dotnet_diagnostic.CA1865.severity = warning
dotnet_diagnostic.CA1866.severity = warning
dotnet_diagnostic.CA1867.severity = warning
dotnet_diagnostic.CA2007.severity = warning
dotnet_diagnostic.CA2011.severity = error
dotnet_diagnostic.CA2248.severity = error

58
Program.cs
View File

@ -1,6 +1,54 @@
string text = File.ReadAllText(args[0]); namespace luaaaaah;
Token[] tokens = new Tokenizer().tokenize(text);
foreach (Token token in tokens) public class Program
{ {
Console.WriteLine($"{token.region}: {token.type} {{{token.data}}}"); public static void Main(string[] args)
} {
switch(args[0])
{
case "test":
{
Test(args[1]);
}
break;
case "run":
{
Run(args[1]);
}
break;
}
}
public static void Run(string file)
{
Token[] tokens = new Tokenizer().Tokenize(File.ReadAllText(file));
foreach(Token token in tokens)
{
Console.WriteLine($"{token.region}: {token.type} {{{token.data}}}");
}
}
public static void Test(string directory)
{
Dictionary<string, string> failedFiles = [];
foreach(string file in Directory.EnumerateFiles(directory))
{
if(file.EndsWith(".lua"))
{
try
{
Run(file);
}
catch(Exception e)
{
Console.WriteLine($"{file}: {e.Message}");
failedFiles.Add(file, e.Message);
}
}
}
Console.WriteLine("===FAILED===");
foreach(KeyValuePair<string, string> entry in failedFiles)
{
Console.WriteLine($"{entry.Key}: {entry.Value}");
}
}
}

434
Tokenizer.cs
View File

@ -1,20 +1,20 @@
namespace luaaaaah;
class Tokenizer class Tokenizer
{ {
private List<Token> tokens = []; private readonly List<Token> tokens = [];
private State state = State.Start; private State state = State.Start;
int? lastIndex = null; int? lastIndex;
int index = 0; int index;
int openingLongBracketLevel = 0; int openingLongBracketLevel;
int closingLongBracketLevel = 0; int closingLongBracketLevel;
Token? currentToken = null; Token? currentToken;
CodeLocation currentLocation = new(line: 0, col: 0); CodeLocation currentLocation = new(line: 0, col: 0);
public Token[] tokenize(string content) public Token[] Tokenize(string content)
{ {
while(index < content.Length) while(index < content.Length)
{ {
tokenizeChar(content[index]); TokenizeChar(content[index]);
if(content[index] == '\n') if(content[index] == '\n')
{ {
currentLocation.line += 1; currentLocation.line += 1;
@ -26,11 +26,11 @@ class Tokenizer
} }
index += 1; index += 1;
} }
tokenizeChar('\n'); TokenizeChar('\n');
return tokens.ToArray(); return [.. tokens];
} }
private void appendDataChar(char ch) private void AppendDataChar(char ch)
{ {
if((Token.StringData?)currentToken!.data == null) if((Token.StringData?)currentToken!.data == null)
{ {
@ -43,36 +43,36 @@ class Tokenizer
currentToken.region.end = new(currentLocation); currentToken.region.end = new(currentLocation);
} }
private void appendDataInt(char ch) private void AppendDataInt(char ch)
{ {
if((Token.NumeralData?)currentToken!.data == null) if((Token.NumeralData?)currentToken!.data == null)
{ {
currentToken!.data = new Token.NumeralData(new Numeral.Integer(ch - '0')); currentToken!.data = new Token.NumeralData(new INumeral.Integer(ch - '0'));
} }
else else
{ {
((Numeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value *= 10; ((INumeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value *= 10;
((Numeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value += ch - '0'; ((INumeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value += ch - '0';
} }
currentToken.region.end = new(currentLocation); currentToken.region.end = new(currentLocation);
} }
private void appendDataIntHex(char ch) private void AppendDataIntHex(char ch)
{ {
int v = char.IsAsciiDigit(ch) ? ch - '0' : 10 + char.ToLower(ch) - 'a'; int v = char.IsAsciiDigit(ch) ? ch - '0' : 10 + char.ToLower(ch) - 'a';
if((Token.NumeralData?)currentToken!.data == null) if((Token.NumeralData?)currentToken!.data == null)
{ {
currentToken!.data = new Token.NumeralData(new Numeral.Integer(v)); currentToken!.data = new Token.NumeralData(new INumeral.Integer(v));
} }
else else
{ {
((Numeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value *= 16; ((INumeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value *= 16;
((Numeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value += v; ((INumeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value += v;
} }
currentToken.region.end = new(currentLocation); currentToken.region.end = new(currentLocation);
} }
private void tokenizeChar(char ch) private void TokenizeChar(char ch)
{ {
switch(state) switch(state)
{ {
@ -343,7 +343,7 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Zero; state = State.Zero;
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Numeral, data: new Token.NumeralData(new Numeral.Integer(0))); currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Numeral, data: new Token.NumeralData(new INumeral.Integer(0)));
} /* tokenizeTerminalIntNum(TokenType.Numeral, TokenizerState.Zero, tokenNumeral, ch); */ } /* tokenizeTerminalIntNum(TokenType.Numeral, TokenizerState.Zero, tokenNumeral, ch); */
break; break;
case '"': case '"':
@ -373,7 +373,7 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Number; state = State.Number;
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Numeral, data: new Token.NumeralData(new Numeral.Integer(ch - '0'))); currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Numeral, data: new Token.NumeralData(new INumeral.Integer(ch - '0')));
} }
else else
{ {
@ -406,7 +406,7 @@ class Tokenizer
} }
else else
{ {
appendDataChar(ch); AppendDataChar(ch);
} }
} }
break; break;
@ -416,62 +416,62 @@ class Tokenizer
{ {
case 'a': case 'a':
{ {
appendDataChar('\u0007'); AppendDataChar('\u0007');
state = State.Quote; state = State.Quote;
} }
break; break;
case 'b': case 'b':
{ {
appendDataChar('\u0008'); AppendDataChar('\u0008');
state = State.Quote; state = State.Quote;
} }
break; break;
case 't': case 't':
{ {
appendDataChar('\t'); AppendDataChar('\t');
state = State.Quote; state = State.Quote;
} }
break; break;
case 'n': case 'n':
case '\n': case '\n':
{ {
appendDataChar('\n'); AppendDataChar('\n');
state = State.Quote; state = State.Quote;
} }
break; break;
case 'v': case 'v':
{ {
appendDataChar('\u000b'); AppendDataChar('\u000b');
state = State.Quote; state = State.Quote;
} }
break; break;
case 'f': case 'f':
{ {
appendDataChar('\u000c'); AppendDataChar('\u000c');
state = State.Quote; state = State.Quote;
} }
break; break;
case 'r': case 'r':
{ {
appendDataChar('\r'); AppendDataChar('\r');
state = State.Quote; state = State.Quote;
} }
break; break;
case '\\': case '\\':
{ {
appendDataChar('\\'); AppendDataChar('\\');
state = State.Quote; state = State.Quote;
} }
break; break;
case '"': case '"':
{ {
appendDataChar('"'); AppendDataChar('"');
state = State.Quote; state = State.Quote;
} }
break; break;
case '\'': case '\'':
{ {
appendDataChar('\''); AppendDataChar('\'');
state = State.Quote; state = State.Quote;
} }
break; break;
@ -506,7 +506,7 @@ class Tokenizer
} }
else if(!char.IsWhiteSpace(ch)) else if(!char.IsWhiteSpace(ch))
{ {
appendDataChar(ch); AppendDataChar(ch);
state = State.Quote; state = State.Quote;
} }
else else
@ -538,7 +538,7 @@ class Tokenizer
} }
else else
{ {
appendDataChar(ch); AppendDataChar(ch);
} }
} }
break; break;
@ -548,62 +548,62 @@ class Tokenizer
{ {
case 'a': case 'a':
{ {
appendDataChar('\u0007'); AppendDataChar('\u0007');
state = State.SingleQuote; state = State.SingleQuote;
} }
break; break;
case 'b': case 'b':
{ {
appendDataChar('\u0008'); AppendDataChar('\u0008');
state = State.SingleQuote; state = State.SingleQuote;
} }
break; break;
case 't': case 't':
{ {
appendDataChar('\t'); AppendDataChar('\t');
state = State.SingleQuote; state = State.SingleQuote;
} }
break; break;
case 'n': case 'n':
case '\n': case '\n':
{ {
appendDataChar('\n'); AppendDataChar('\n');
state = State.SingleQuote; state = State.SingleQuote;
} }
break; break;
case 'v': case 'v':
{ {
appendDataChar('\u000b'); AppendDataChar('\u000b');
state = State.SingleQuote; state = State.SingleQuote;
} }
break; break;
case 'f': case 'f':
{ {
appendDataChar('\u000c'); AppendDataChar('\u000c');
state = State.SingleQuote; state = State.SingleQuote;
} }
break; break;
case 'r': case 'r':
{ {
appendDataChar('\r'); AppendDataChar('\r');
state = State.SingleQuote; state = State.SingleQuote;
} }
break; break;
case '\\': case '\\':
{ {
appendDataChar('\\'); AppendDataChar('\\');
state = State.SingleQuote; state = State.SingleQuote;
} }
break; break;
case '"': case '"':
{ {
appendDataChar('"'); AppendDataChar('"');
state = State.SingleQuote; state = State.SingleQuote;
} }
break; break;
case '\'': case '\'':
{ {
appendDataChar('\''); AppendDataChar('\'');
state = State.SingleQuote; state = State.SingleQuote;
} }
break; break;
@ -638,7 +638,7 @@ class Tokenizer
} }
else if(!char.IsWhiteSpace(ch)) else if(!char.IsWhiteSpace(ch))
{ {
appendDataChar(ch); AppendDataChar(ch);
state = State.Quote; state = State.Quote;
} }
else else
@ -671,7 +671,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -703,7 +703,7 @@ class Tokenizer
else if(char.IsAsciiDigit(ch)) else if(char.IsAsciiDigit(ch))
{ {
lastIndex = index; lastIndex = index;
appendDataInt(ch); AppendDataInt(ch);
} }
else else
{ {
@ -726,7 +726,7 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
currentToken!.type = TokenType.Numeral; currentToken!.type = TokenType.Numeral;
appendDataIntHex(ch); AppendDataIntHex(ch);
state = State.HexNumber; state = State.HexNumber;
} }
else else
@ -755,7 +755,7 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
currentToken!.type = TokenType.Numeral; currentToken!.type = TokenType.Numeral;
appendDataIntHex(ch); AppendDataIntHex(ch);
} }
else else
{ {
@ -787,7 +787,7 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
currentToken!.type = TokenType.Numeral; currentToken!.type = TokenType.Numeral;
appendDataInt(ch); AppendDataInt(ch);
} }
else else
{ {
@ -1199,7 +1199,7 @@ class Tokenizer
} }
else else
{ {
appendDataChar(ch); AppendDataChar(ch);
} }
} }
break; break;
@ -1212,7 +1212,7 @@ class Tokenizer
{ {
state = State.StringWithLongBracket; state = State.StringWithLongBracket;
} }
appendDataChar(ch); AppendDataChar(ch);
} }
else if(ch == ']' && openingLongBracketLevel == closingLongBracketLevel) else if(ch == ']' && openingLongBracketLevel == closingLongBracketLevel)
{ {
@ -1237,7 +1237,7 @@ class Tokenizer
else else
{ {
closingLongBracketLevel = 0; closingLongBracketLevel = 0;
appendDataChar(ch); AppendDataChar(ch);
} }
} }
break; break;
@ -1247,14 +1247,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.An; state = State.An;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1278,14 +1278,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.And; state = State.And;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1310,7 +1310,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1335,14 +1335,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Wh; state = State.Wh;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1366,14 +1366,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Whi; state = State.Whi;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1397,14 +1397,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Whil; state = State.Whil;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1428,14 +1428,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.While; state = State.While;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1460,7 +1460,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1485,14 +1485,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Br; state = State.Br;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1516,14 +1516,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Bre; state = State.Bre;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1547,14 +1547,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Brea; state = State.Brea;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1578,14 +1578,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Break; state = State.Break;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1610,7 +1610,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1635,14 +1635,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Go; state = State.Go;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1666,14 +1666,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Got; state = State.Got;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1697,14 +1697,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Goto; state = State.Goto;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1729,7 +1729,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1754,14 +1754,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Re; state = State.Re;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1785,20 +1785,20 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Ret; state = State.Ret;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(ch == 'p') else if(ch == 'p')
{ {
lastIndex = index; lastIndex = index;
state = State.Rep; state = State.Rep;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1822,14 +1822,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Retu; state = State.Retu;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1853,14 +1853,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Retur; state = State.Retur;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1884,14 +1884,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Return; state = State.Return;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1916,7 +1916,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1941,14 +1941,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Repe; state = State.Repe;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -1972,14 +1972,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Repea; state = State.Repea;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2003,14 +2003,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Repeat; state = State.Repeat;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2035,7 +2035,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2060,20 +2060,20 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Ni; state = State.Ni;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(ch == 'o') else if(ch == 'o')
{ {
lastIndex= index; lastIndex = index;
state = State.No; state = State.No;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2097,14 +2097,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Nil; state = State.Nil;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2129,7 +2129,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2154,14 +2154,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Not; state = State.Not;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2186,7 +2186,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2211,20 +2211,20 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Th; state = State.Th;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(ch == 'r') else if(ch == 'r')
{ {
lastIndex = index; lastIndex = index;
state = State.Tr; state = State.Tr;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2248,14 +2248,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.The; state = State.The;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2279,14 +2279,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Then; state = State.Then;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2311,7 +2311,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2336,14 +2336,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Tru; state = State.Tru;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2367,14 +2367,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.True; state = State.True;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2399,7 +2399,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2424,20 +2424,20 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.El; state = State.El;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(ch == 'n') else if(ch == 'n')
{ {
lastIndex = index; lastIndex = index;
state = State.En; state = State.En;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2461,14 +2461,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Els; state = State.Els;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2492,14 +2492,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Else; state = State.Else;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2523,14 +2523,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Elsei; state = State.Elsei;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2555,14 +2555,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Elseif; state = State.Elseif;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2587,7 +2587,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2612,14 +2612,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.End; state = State.End;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2644,7 +2644,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2669,14 +2669,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Or; state = State.Or;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2701,7 +2701,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2726,14 +2726,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Do; state = State.Do;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2758,7 +2758,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2783,20 +2783,20 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.If; state = State.If;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(ch == 'n') else if(ch == 'n')
{ {
lastIndex = index; lastIndex = index;
state = State.In; state = State.In;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2821,7 +2821,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2847,7 +2847,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2872,26 +2872,26 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Fu; state = State.Fu;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(ch == 'a') else if(ch == 'a')
{ {
lastIndex = index; lastIndex = index;
state = State.Fa; state = State.Fa;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(ch == 'o') else if(ch == 'o')
{ {
lastIndex = index; lastIndex = index;
state = State.Fo; state = State.Fo;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2915,14 +2915,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Fun; state = State.Fun;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2946,14 +2946,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Func; state = State.Func;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -2977,14 +2977,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Funct; state = State.Funct;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3008,14 +3008,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Functi; state = State.Functi;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3039,14 +3039,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Functio; state = State.Functio;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3070,14 +3070,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Function; state = State.Function;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3102,7 +3102,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3127,14 +3127,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Fal; state = State.Fal;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3158,14 +3158,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Fals; state = State.Fals;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3189,14 +3189,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.False; state = State.False;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3221,7 +3221,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3246,14 +3246,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.For; state = State.For;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3278,7 +3278,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3303,14 +3303,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Lo; state = State.Lo;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3334,14 +3334,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Loc; state = State.Loc;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3365,14 +3365,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Loca; state = State.Loca;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3396,14 +3396,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Local; state = State.Local;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3428,7 +3428,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3453,14 +3453,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Un; state = State.Un;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3484,14 +3484,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Unt; state = State.Unt;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3515,14 +3515,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Unti; state = State.Unti;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3546,14 +3546,14 @@ class Tokenizer
{ {
lastIndex = index; lastIndex = index;
state = State.Until; state = State.Until;
appendDataChar(ch); AppendDataChar(ch);
} }
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{ {
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3578,7 +3578,7 @@ class Tokenizer
lastIndex = index; lastIndex = index;
state = State.Name; state = State.Name;
currentToken!.type = TokenType.Name; currentToken!.type = TokenType.Name;
appendDataChar(ch); AppendDataChar(ch);
} }
else else
{ {
@ -3656,22 +3656,22 @@ class CodeLocation(int line, int col)
} }
} }
class Token(CodeRegion region, TokenType? type = null, Token.Data? data = null) class Token(CodeRegion region, TokenType? type = null, Token.IData? data = null)
{ {
public CodeRegion region = region; public CodeRegion region = region;
public Data? data = data; public IData? data = data;
public TokenType? type = type; public TokenType? type = type;
public interface Data { } public interface IData { }
public class NumeralData(Numeral numeral) : Data public class NumeralData(INumeral numeral) : IData
{ {
public Numeral numeral = numeral; public INumeral numeral = numeral;
public override string ToString() public override string ToString()
{ {
return $"NumeralData {numeral}"; return $"NumeralData {numeral}";
} }
} }
public class StringData(string data) : Data public class StringData(string data) : IData
{ {
public string data = data; public string data = data;
public override string ToString() public override string ToString()
@ -3696,38 +3696,38 @@ public enum TokenType
StringLiteral, StringLiteral,
} }
public interface Numeral public interface INumeral
{ {
public class Integer(int value) : Numeral public class Integer(int value) : INumeral
{ {
public int value = value; public int value = value;
public bool rawEqual(Numeral other) public bool RawEqual(INumeral other)
{ {
if(other is Numeral.Integer) if(other is Integer integer)
{ {
return ((Numeral.Integer)other).value == value; return integer.value == value;
} }
// TODO: Check if this is actually doing what is expected // TODO: Check if this is actually doing what is expected
return ((Numeral.Float)other).value == value; return ((Float)other).value == value;
} }
public override string ToString() public override string ToString()
{ {
return $"Numeral Integer {value}"; return $"Numeral Integer {value}";
} }
} }
public class Float(float value) : Numeral public class Float(float value) : INumeral
{ {
public float value = value; public float value = value;
public bool rawEqual(Numeral other) public bool RawEqual(INumeral other)
{ {
if(other is Numeral.Float) if(other is Float float_val)
{ {
return ((Numeral.Float)other).value == value; return float_val.value == value;
} }
// TODO: Check if this is actually doing what is expected // TODO: Check if this is actually doing what is expected
return ((Numeral.Integer)other).value == value; return ((Integer)other).value == value;
} }
public override string ToString() public override string ToString()
{ {
@ -3735,5 +3735,5 @@ public interface Numeral
} }
} }
public bool rawEqual(Numeral other); public bool RawEqual(INumeral other);
} }