From 34cb88582d9064a6e5de486b608dd6323aa0d636 Mon Sep 17 00:00:00 2001 From: 0x4261756D <38735823+0x4261756D@users.noreply.github.com> Date: Tue, 16 Jan 2024 02:59:51 +0100 Subject: [PATCH] Port tokenizer to C# Another language change, another unrefactored (but already better tested) tokenizer --- .editorconfig | 11 + .gitignore | 6 +- .vscode/launch.json | 26 + .vscode/tasks.json | 41 + Program.cs | 6 + Tokenizer.cs | 3739 +++++++++++++++++++++++++++++++++++++++++ build.zig | 83 - luaaaaah.csproj | 9 + src/main.zig | 40 - src/parser.zig | 2572 ---------------------------- src/tokenizer.zig | 1246 -------------- src/treewalker.zig | 142 -- src/types.zig | 194 --- test/simpleInt.lua | 22 + test/simpleString.lua | 10 + 15 files changed, 3866 insertions(+), 4281 deletions(-) create mode 100644 .editorconfig create mode 100644 .vscode/launch.json create mode 100644 .vscode/tasks.json create mode 100644 Program.cs create mode 100644 Tokenizer.cs delete mode 100644 build.zig create mode 100644 luaaaaah.csproj delete mode 100644 src/main.zig delete mode 100644 src/parser.zig delete mode 100644 src/tokenizer.zig delete mode 100644 src/treewalker.zig delete mode 100644 src/types.zig create mode 100644 test/simpleInt.lua create mode 100644 test/simpleString.lua diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..dc97a31 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +root = true + +indent_size = 4 +indent_style = tab + +[*.cs] + +trim_trailing_whitespace = true + +csharp_space_after_keywords_in_control_flow_statements = false +csharp_indent_case_contents_when_block = false \ No newline at end of file diff --git a/.gitignore b/.gitignore index 27c5979..cbbd0b5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,2 @@ -zig-out/ -zig-cache/ -.vscode/ -target/ +bin/ +obj/ \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..e7a791f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,26 @@ +{ + "version": "0.2.0", + "configurations": [ + { + // Use IntelliSense to find out which attributes exist for C# debugging + // Use hover for the description of the existing attributes + // For further information visit https://github.com/dotnet/vscode-csharp/blob/main/debugger-launchjson.md + "name": ".NET Core Launch (console)", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build", + // If you have changed target frameworks, make sure to update the program path. + "program": "${workspaceFolder}/bin/Debug/net8.0/luaaaaah.dll", + "args": ["test/simpleString.lua"], + "cwd": "${workspaceFolder}", + // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console + "console": "internalConsole", + "stopAtEntry": false + }, + { + "name": ".NET Core Attach", + "type": "coreclr", + "request": "attach" + } + ] +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..b5af5d4 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,41 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "build", + "command": "dotnet", + "type": "process", + "args": [ + "build", + "${workspaceFolder}/luaaaaah.csproj", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary;ForceNoAlign" + ], + "problemMatcher": "$msCompile" + }, + { + "label": "publish", + "command": "dotnet", + "type": "process", + "args": [ + "publish", + "${workspaceFolder}/luaaaaah.csproj", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary;ForceNoAlign" + ], + "problemMatcher": "$msCompile" + }, + { + "label": "watch", + "command": "dotnet", + "type": "process", + "args": [ + "watch", + "run", + "--project", + "${workspaceFolder}/luaaaaah.csproj" + ], + "problemMatcher": "$msCompile" + } + ] +} \ No newline at end of file diff --git a/Program.cs b/Program.cs new file mode 100644 index 0000000..a4830f5 --- /dev/null +++ b/Program.cs @@ -0,0 +1,6 @@ +string text = File.ReadAllText(args[0]); +Token[] tokens = new Tokenizer().tokenize(text); +foreach (Token token in tokens) +{ + Console.WriteLine($"{token.region}: {token.type} {{{token.data}}}"); +} \ No newline at end of file diff --git a/Tokenizer.cs b/Tokenizer.cs new file mode 100644 index 0000000..5ac5e68 --- /dev/null +++ b/Tokenizer.cs @@ -0,0 +1,3739 @@ + +class Tokenizer +{ + private List tokens = []; + private State state = State.Start; + int? lastIndex = null; + int index = 0; + int openingLongBracketLevel = 0; + int closingLongBracketLevel = 0; + Token? currentToken = null; + CodeLocation currentLocation = new(line: 0, col: 0); + + public Token[] tokenize(string content) + { + while(index < content.Length) + { + tokenizeChar(content[index]); + if(content[index] == '\n') + { + currentLocation.line += 1; + currentLocation.col = 0; + } + else + { + currentLocation.col += 1; + } + index += 1; + } + tokenizeChar('\n'); + return tokens.ToArray(); + } + + private void appendDataChar(char ch) + { + if((Token.StringData?)currentToken!.data == null) + { + currentToken!.data = new Token.StringData($"{ch}"); + } + else + { + ((Token.StringData?)currentToken!.data!).data += ch; + } + currentToken.region.end = new(currentLocation); + } + + private void appendDataInt(char ch) + { + if((Token.NumeralData?)currentToken!.data == null) + { + currentToken!.data = new Token.NumeralData(new Numeral.Integer(ch - '0')); + } + else + { + ((Numeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value *= 10; + ((Numeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value += ch - '0'; + } + currentToken.region.end = new(currentLocation); + } + + private void appendDataIntHex(char ch) + { + int v = char.IsAsciiDigit(ch) ? ch - '0' : 10 + char.ToLower(ch) - 'a'; + if((Token.NumeralData?)currentToken!.data == null) + { + currentToken!.data = new Token.NumeralData(new Numeral.Integer(v)); + } + else + { + ((Numeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value *= 16; + ((Numeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value += v; + } + currentToken.region.end = new(currentLocation); + } + + private void tokenizeChar(char ch) + { + switch(state) + { + case State.Start: + { + switch(ch) + { + case '-': + { + lastIndex = index; + state = State.Minus; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Minus); + } /* tokenizeTerminalBase(TokenType.Minus, TokenizerState.Minus); */ + break; + case ',': + { + lastIndex = index; + state = State.Comma; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Comma); + } /* tokenizeTerminalBase(TokenType.Comma, TokenizerState.Comma); */ + break; + case '=': + { + lastIndex = index; + state = State.Equals; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Equals); + } /* tokenizeTerminalBase(TokenType.Equals, TokenizerState.Equals); */ + break; + case '(': + { + lastIndex = index; + state = State.RoundOpen; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.RoundOpen); + } /* tokenizeTerminalBase(TokenType.RoundOpen, TokenizerState.RoundOpen); */ + break; + case ')': + { + lastIndex = index; + state = State.RoundClosed; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.RoundClosed); + } /* tokenizeTerminalBase(TokenType.RoundClosed, TokenizerState.RoundClosed); */ + break; + case '.': + { + lastIndex = index; + state = State.Dot; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Dot); + } /* tokenizeTerminalBase(TokenType.Dot, TokenizerState.Dot); */ + break; + case ':': + { + lastIndex = index; + state = State.Colon; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Colon); + } /* tokenizeTerminalBase(TokenType.Colon, TokenizerState.Colon); */ + break; + case '{': + { + lastIndex = index; + state = State.CurlyOpen; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.CurlyOpen); + } /* tokenizeTerminalBase(TokenType.CurlyOpen, TokenizerState.CurlyOpen); */ + break; + case '}': + { + lastIndex = index; + state = State.CurlyClosed; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.CurlyClosed); + } /* tokenizeTerminalBase(TokenType.CurlyClosed, TokenizerState.CurlyClosed); */ + break; + case '[': + { + lastIndex = index; + state = State.SquareOpen; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.SquareOpen); + } /* tokenizeTerminalBase(TokenType.SquareOpen, TokenizerState.SquareOpen); */ + break; + case ']': + { + lastIndex = index; + state = State.SquareClosed; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.SquareClosed); + } /* tokenizeTerminalBase(TokenType.SquareClosed, TokenizerState.SquareClosed); */ + break; + case '+': + { + lastIndex = index; + state = State.Plus; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Plus); + } /* tokenizeTerminalBase(TokenType.Plus, TokenizerState.Plus); */ + break; + case '~': + { + lastIndex = index; + state = State.Tilde; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Tilde); + } /* tokenizeTerminalBase(TokenType.Tilde, TokenizerState.Tilde); */ + break; + case '>': + { + lastIndex = index; + state = State.Gt; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Gt); + } /* tokenizeTerminalBase(TokenType.Gt, TokenizerState.Gt); */ + break; + case '<': + { + lastIndex = index; + state = State.Lt; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Lt); + } /* tokenizeTerminalBase(TokenType.Lt, TokenizerState.Lt); */ + break; + case '#': + { + lastIndex = index; + state = State.Hash; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Hash); + } /* tokenizeTerminalBase(TokenType.Hash, TokenizerState.Hash); */ + break; + case '|': + { + lastIndex = index; + state = State.Pipe; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Pipe); + } /* tokenizeTerminalBase(TokenType.Pipe, TokenizerState.Pipe); */ + break; + case '&': + { + lastIndex = index; + state = State.Ampersand; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Equals); + } /* tokenizeTerminalBase(TokenType.Ampersand, TokenizerState.Ampersand); */ + break; + case '%': + { + lastIndex = index; + state = State.Percent; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Percent); + } /* tokenizeTerminalBase(TokenType.Percent, TokenizerState.Percent); */ + break; + case '*': + { + lastIndex = index; + state = State.Star; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Star); + } /* tokenizeTerminalBase(TokenType.Star, TokenizerState.Star); */ + break; + case '/': + { + lastIndex = index; + state = State.Slash; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Slash); + } /* tokenizeTerminalBase(TokenType.Slash, TokenizerState.Slash); */ + break; + case ';': + { + lastIndex = index; + state = State.Semicolon; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Semicolon); + } /* tokenizeTerminalBase(TokenType.Semicolon, TokenizerState.Semicolon); */ + break; + case '^': + { + lastIndex = index; + state = State.Caret; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Caret); + } /* tokenizeTerminalBase(TokenType.Caret, TokenizerState.Caret); */ + break; + case 'a': + { + lastIndex = index; + state = State.A; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.A, tokenStr, ch); */ + break; + case 'b': + { + lastIndex = index; + state = State.B; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.B, tokenStr, ch); */ + break; + case 'd': + { + lastIndex = index; + state = State.D; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.D, tokenStr, ch); */ + break; + case 'e': + { + lastIndex = index; + state = State.E; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.E, tokenStr, ch); */ + break; + case 'f': + { + lastIndex = index; + state = State.F; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.F, tokenStr, ch); */ + break; + case 'i': + { + lastIndex = index; + state = State.I; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.I, tokenStr, ch); */ + break; + case 'g': + { + lastIndex = index; + state = State.G; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.G, tokenStr, ch); */ + break; + case 'l': + { + lastIndex = index; + state = State.L; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.L, tokenStr, ch); */ + break; + case 'n': + { + lastIndex = index; + state = State.N; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.N, tokenStr, ch); */ + break; + case 'o': + { + lastIndex = index; + state = State.O; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.O, tokenStr, ch); */ + break; + case 'r': + { + lastIndex = index; + state = State.R; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.R, tokenStr, ch); */ + break; + case 't': + { + lastIndex = index; + state = State.T; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.T, tokenStr, ch); */ + break; + case 'u': + { + lastIndex = index; + state = State.U; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.U, tokenStr, ch); */ + break; + case 'w': + { + lastIndex = index; + state = State.W; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } /* tokenizeTerminalStr(TokenType.Name, TokenizerState.W, tokenStr, ch); */ + break; + case '0': + { + lastIndex = index; + state = State.Zero; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Numeral, data: new Token.NumeralData(new Numeral.Integer(0))); + } /* tokenizeTerminalIntNum(TokenType.Numeral, TokenizerState.Zero, tokenNumeral, ch); */ + break; + case '"': + { + currentToken = null; + state = State.Quote; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral); + } + break; + case '\'': + { + currentToken = null; + state = State.SingleQuote; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral); + } + break; + default: + { + if(char.IsWhiteSpace(ch)) { } + else if(char.IsAsciiLetter(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}")); + } + else if(char.IsDigit(ch)) + { + lastIndex = index; + state = State.Number; + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Numeral, data: new Token.NumeralData(new Numeral.Integer(ch - '0'))); + } + else + { + throw new NotImplementedException(ch.ToString()); + } + } + break; + } + } + break; + case State.Quote: + { + if(ch == '\\') + { + state = State.QuoteBackslash; + } + else if(ch == '"') + { + lastIndex = index; + state = State.String; + if(currentToken == null || currentToken.type == null) + { + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral); + } + else + { + currentToken.type = TokenType.StringLiteral; + currentToken.region.end = new(currentLocation); + } + } + else + { + appendDataChar(ch); + } + } + break; + case State.QuoteBackslash: + { + switch(ch) + { + case 'a': + { + appendDataChar('\u0007'); + state = State.Quote; + } + break; + case 'b': + { + appendDataChar('\u0008'); + state = State.Quote; + } + break; + case 't': + { + appendDataChar('\t'); + state = State.Quote; + } + break; + case 'n': + case '\n': + { + appendDataChar('\n'); + state = State.Quote; + } + break; + case 'v': + { + appendDataChar('\u000b'); + state = State.Quote; + } + break; + case 'f': + { + appendDataChar('\u000c'); + state = State.Quote; + } + break; + case 'r': + { + appendDataChar('\r'); + state = State.Quote; + } + break; + case '\\': + { + appendDataChar('\\'); + state = State.Quote; + } + break; + case '"': + { + appendDataChar('"'); + state = State.Quote; + } + break; + case '\'': + { + appendDataChar('\''); + state = State.Quote; + } + break; + case 'z': + { + state = State.QuoteBackslashZ; + } + break; + default: throw new Exception($"Unknown escape sequence: \\{ch}"); + } + } + break; + case State.QuoteBackslashZ: + { + if(ch == '\\') + { + state = State.QuoteBackslash; + } + else if(ch == '"') + { + lastIndex = index; + state = State.String; + if(currentToken == null || currentToken.type == null) + { + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral); + } + else + { + currentToken.type = TokenType.StringLiteral; + currentToken.region.end = new(currentLocation); + } + } + else if(!char.IsWhiteSpace(ch)) + { + appendDataChar(ch); + state = State.Quote; + } + else + { + // Noop, https://www.lua.org/manual/5.4/manual.html#3.1: + // "The escape sequence '\z' skips the following span of whitespace characters, including line breaks;" + } + } + break; + case State.SingleQuote: + { + if(ch == '\\') + { + state = State.SingleQuoteBackslash; + } + else if(ch == '\'') + { + lastIndex = index; + state = State.String; + if(currentToken == null || currentToken.type == null) + { + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral); + } + else + { + currentToken.type = TokenType.StringLiteral; + currentToken.region.end = new(currentLocation); + } + } + else + { + appendDataChar(ch); + } + } + break; + case State.SingleQuoteBackslash: + { + switch(ch) + { + case 'a': + { + appendDataChar('\u0007'); + state = State.SingleQuote; + } + break; + case 'b': + { + appendDataChar('\u0008'); + state = State.SingleQuote; + } + break; + case 't': + { + appendDataChar('\t'); + state = State.SingleQuote; + } + break; + case 'n': + case '\n': + { + appendDataChar('\n'); + state = State.SingleQuote; + } + break; + case 'v': + { + appendDataChar('\u000b'); + state = State.SingleQuote; + } + break; + case 'f': + { + appendDataChar('\u000c'); + state = State.SingleQuote; + } + break; + case 'r': + { + appendDataChar('\r'); + state = State.SingleQuote; + } + break; + case '\\': + { + appendDataChar('\\'); + state = State.SingleQuote; + } + break; + case '"': + { + appendDataChar('"'); + state = State.SingleQuote; + } + break; + case '\'': + { + appendDataChar('\''); + state = State.SingleQuote; + } + break; + case 'z': + { + state = State.SingleQuoteBackslashZ; + } + break; + default: throw new Exception($"Unknown escape sequence: \\{ch}"); + } + } + break; + case State.SingleQuoteBackslashZ: + { + if(ch == '\\') + { + state = State.SingleQuoteBackslash; + } + else if(ch == '\'') + { + lastIndex = index; + state = State.String; + if(currentToken == null || currentToken.type == null) + { + currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral); + } + else + { + currentToken.type = TokenType.StringLiteral; + currentToken.region.end = new(currentLocation); + } + } + else if(!char.IsWhiteSpace(ch)) + { + appendDataChar(ch); + state = State.Quote; + } + else + { + // Noop, https://www.lua.org/manual/5.4/manual.html#3.1: + // "The escape sequence '\z' skips the following span of whitespace characters, including line breaks;" + } + } + break; + case State.String: + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.StringLiteral; + //currentToken.region.end = new(currentLocation); + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + break; + case State.Name: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Zero: + { + if(ch == 'x') + { + currentToken!.type = null; + state = State.HexNumberX; + } + else if(ch == '.') + { + throw new NotImplementedException(); + } + else if(char.IsAsciiDigit(ch)) + { + lastIndex = index; + appendDataInt(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.HexNumberX: + { + if(char.IsAsciiHexDigit(ch)) + { + lastIndex = index; + currentToken!.type = TokenType.Numeral; + appendDataIntHex(ch); + state = State.HexNumber; + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.HexNumber: + { + if(ch == 'p') + { + currentToken!.type = null; + state = State.HexExpNumber; + } + else if(char.IsAsciiHexDigit(ch)) + { + lastIndex = index; + currentToken!.type = TokenType.Numeral; + appendDataIntHex(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Number: + { + if(ch == 'e') + { + currentToken!.type = null; + state = State.ExpNumber; + } + else if(ch == '.') + { + throw new NotImplementedException(); + } + else if(char.IsAsciiDigit(ch)) + { + lastIndex = index; + currentToken!.type = TokenType.Numeral; + appendDataInt(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.SquareOpen: + { + if(ch == '[') + { + currentToken = new Token(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral); + state = State.StringWithLongBracket; + } + else if(ch == '=') + { + openingLongBracketLevel = 1; + state = State.StringStartLongBracket; + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Comma: + case State.RoundOpen: + case State.RoundClosed: + case State.CurlyOpen: + case State.CurlyClosed: + case State.Plus: + case State.TildeEquals: + case State.EqualsEquals: + case State.Hash: + case State.GtEquals: + case State.LtEquals: + case State.SquareClosed: + case State.Pipe: + case State.Ampersand: + case State.Percent: + case State.Star: + case State.Semicolon: + case State.Caret: + case State.DotDotDot: + case State.GtGt: + case State.LtLt: + case State.ColonColon: + case State.SlashSlash: + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + break; + case State.Tilde: + { + if(ch == '=') + { + lastIndex = index; + state = State.TildeEquals; + currentToken!.type = TokenType.TildeEquals; + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Gt: + { + if(ch == '=') + { + lastIndex = index; + state = State.GtEquals; + currentToken!.type = TokenType.GtEquals; + } + else if(ch == '>') + { + lastIndex = index; + state = State.GtGt; + currentToken!.type = TokenType.GtGt; + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Lt: + { + if(ch == '=') + { + lastIndex = index; + state = State.LtEquals; + currentToken!.type = TokenType.LtEquals; + } + else if(ch == '<') + { + lastIndex = index; + state = State.LtLt; + currentToken!.type = TokenType.LtLt; + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Slash: + { + if(ch == '/') + { + lastIndex = index; + state = State.SlashSlash; + currentToken!.type = TokenType.SlashSlash; + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Dot: + { + if(ch == '.') + { + lastIndex = index; + state = State.DotDot; + currentToken!.type = TokenType.DotDot; + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + + case State.DotDot: + { + if(ch == '.') + { + lastIndex = index; + state = State.DotDotDot; + currentToken!.type = TokenType.DotDotDot; + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Colon: + { + if(ch == ':') + { + lastIndex = index; + state = State.ColonColon; + currentToken!.type = TokenType.ColonColon; + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Equals: + { + if(ch == '=') + { + lastIndex = index; + state = State.EqualsEquals; + currentToken!.type = TokenType.EqualsEquals; + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Minus: + { + if(ch == '-') + { + lastIndex = index; + state = State.SmallCommentStart; + currentToken = null; + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.SmallCommentStart: + { + if(ch == '[') + { + state = State.BigCommentStartLongBracket; + } + else if(ch == '\n') + { + state = State.Start; + lastIndex = null; + } + else + { + state = State.SmallComment; + } + } + break; + case State.SmallComment: + { + if(ch == '\n') + { + state = State.Start; + lastIndex = null; + } + } + break; + case State.BigCommentStartLongBracket: + { + if(ch == '=') + { + openingLongBracketLevel += 1; + } + else if(ch == '[') + { + state = State.BigComment; + } + else if(ch == '\n') + { + state = State.Start; + } + else + { + state = State.SmallComment; + } + } + break; + case State.BigComment: + { + if(ch == ']') + { + state = State.BigCommentEndLongBracket; + closingLongBracketLevel = 0; + } + } + break; + case State.BigCommentEndLongBracket: + { + if(ch == '=') + { + closingLongBracketLevel += 1; + if(openingLongBracketLevel < closingLongBracketLevel) + { + state = State.BigComment; + } + } + else if(ch == ']' && openingLongBracketLevel == closingLongBracketLevel) + { + state = State.Start; + openingLongBracketLevel = 0; + closingLongBracketLevel = 0; + } + else + { + closingLongBracketLevel = 0; + } + } + break; + case State.StringStartLongBracket: + { + if(ch == '=') + { + openingLongBracketLevel += 1; + + } + else if(ch == '[') + { + state = State.StringWithLongBracket; + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.StringWithLongBracket: + { + if(ch == ']') + { + state = State.StringEndLongBracket; + closingLongBracketLevel = 0; + } + else + { + appendDataChar(ch); + } + } + break; + case State.StringEndLongBracket: + { + if(ch == '=') + { + closingLongBracketLevel += 1; + if(openingLongBracketLevel < closingLongBracketLevel) + { + state = State.StringWithLongBracket; + } + appendDataChar(ch); + } + else if(ch == ']' && openingLongBracketLevel == closingLongBracketLevel) + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + if((Token.StringData?)currentToken.data == null) + { + currentToken.data = new Token.StringData(""); + } + currentToken.type = TokenType.StringLiteral; + ((Token.StringData)currentToken.data).data = ((Token.StringData)currentToken.data).data.Remove(((Token.StringData)currentToken.data).data.Length - closingLongBracketLevel); + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + lastIndex = null; + state = State.Start; + openingLongBracketLevel = 0; + closingLongBracketLevel = 0; + } + else + { + closingLongBracketLevel = 0; + appendDataChar(ch); + } + } + break; + case State.A: + { + if(ch == 'n') + { + lastIndex = index; + state = State.An; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.An: + { + if(ch == 'd') + { + lastIndex = index; + state = State.And; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.And: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.And; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.W: + { + if(ch == 'h') + { + lastIndex = index; + state = State.Wh; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Wh: + { + if(ch == 'i') + { + lastIndex = index; + state = State.Whi; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Whi: + { + if(ch == 'l') + { + lastIndex = index; + state = State.Whil; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Whil: + { + if(ch == 'e') + { + lastIndex = index; + state = State.While; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.While: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.While; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.B: + { + if(ch == 'r') + { + lastIndex = index; + state = State.Br; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Br: + { + if(ch == 'e') + { + lastIndex = index; + state = State.Bre; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Bre: + { + if(ch == 'a') + { + lastIndex = index; + state = State.Brea; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Brea: + { + if(ch == 'k') + { + lastIndex = index; + state = State.Break; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Break: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Break; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.G: + { + if(ch == 'o') + { + lastIndex = index; + state = State.Go; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Go: + { + if(ch == 't') + { + lastIndex = index; + state = State.Got; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Got: + { + if(ch == 'o') + { + lastIndex = index; + state = State.Goto; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Goto: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Goto; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.R: + { + if(ch == 'e') + { + lastIndex = index; + state = State.Re; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Re: + { + if(ch == 't') + { + lastIndex = index; + state = State.Ret; + appendDataChar(ch); + } + else if(ch == 'p') + { + lastIndex = index; + state = State.Rep; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Ret: + { + if(ch == 'u') + { + lastIndex = index; + state = State.Retu; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Retu: + { + if(ch == 'r') + { + lastIndex = index; + state = State.Retur; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Retur: + { + if(ch == 'n') + { + lastIndex = index; + state = State.Return; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Return: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Return; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Rep: + { + if(ch == 'e') + { + lastIndex = index; + state = State.Repe; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Repe: + { + if(ch == 'a') + { + lastIndex = index; + state = State.Repea; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Repea: + { + if(ch == 't') + { + lastIndex = index; + state = State.Repeat; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Repeat: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Repeat; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.N: + { + if(ch == 'i') + { + lastIndex = index; + state = State.Ni; + appendDataChar(ch); + } + else if(ch == 'o') + { + lastIndex= index; + state = State.No; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Ni: + { + if(ch == 'l') + { + lastIndex = index; + state = State.Nil; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Nil: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Nil; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.No: + { + if(ch == 't') + { + lastIndex = index; + state = State.Not; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Not: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Not; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.T: + { + if(ch == 'h') + { + lastIndex = index; + state = State.Th; + appendDataChar(ch); + } + else if(ch == 'r') + { + lastIndex = index; + state = State.Tr; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Th: + { + if(ch == 'e') + { + lastIndex = index; + state = State.The; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.The: + { + if(ch == 'n') + { + lastIndex = index; + state = State.Then; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Then: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Then; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Tr: + { + if(ch == 'u') + { + lastIndex = index; + state = State.Tru; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Tru: + { + if(ch == 'e') + { + lastIndex = index; + state = State.True; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.True: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.True; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.E: + { + if(ch == 'l') + { + lastIndex = index; + state = State.El; + appendDataChar(ch); + } + else if(ch == 'n') + { + lastIndex = index; + state = State.En; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.El: + { + if(ch == 's') + { + lastIndex = index; + state = State.Els; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Els: + { + if(ch == 'e') + { + lastIndex = index; + state = State.Else; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Else: + { + if(ch == 'i') + { + lastIndex = index; + state = State.Elsei; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Else; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Elsei: + { + if(ch == 'f') + { + lastIndex = index; + state = State.Elseif; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Elseif: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Elseif; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.En: + { + if(ch == 'd') + { + lastIndex = index; + state = State.End; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.End: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.End; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.O: + { + if(ch == 'r') + { + lastIndex = index; + state = State.Or; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Or: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Or; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.D: + { + if(ch == 'o') + { + lastIndex = index; + state = State.Do; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Do: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Do; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.I: + { + if(ch == 'f') + { + lastIndex = index; + state = State.If; + appendDataChar(ch); + } + else if(ch == 'n') + { + lastIndex = index; + state = State.In; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.In: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.In; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.If: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.If; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.F: + { + if(ch == 'u') + { + lastIndex = index; + state = State.Fu; + appendDataChar(ch); + } + else if(ch == 'a') + { + lastIndex = index; + state = State.Fa; + appendDataChar(ch); + } + else if(ch == 'o') + { + lastIndex = index; + state = State.Fo; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Fu: + { + if(ch == 'n') + { + lastIndex = index; + state = State.Fun; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Fun: + { + if(ch == 'c') + { + lastIndex = index; + state = State.Func; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Func: + { + if(ch == 't') + { + lastIndex = index; + state = State.Funct; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Funct: + { + if(ch == 'i') + { + lastIndex = index; + state = State.Functi; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Functi: + { + if(ch == 'o') + { + lastIndex = index; + state = State.Functio; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Functio: + { + if(ch == 'n') + { + lastIndex = index; + state = State.Function; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Function: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Function; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Fa: + { + if(ch == 'l') + { + lastIndex = index; + state = State.Fal; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Fal: + { + if(ch == 's') + { + lastIndex = index; + state = State.Fals; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Fals: + { + if(ch == 'e') + { + lastIndex = index; + state = State.False; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.False: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.False; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Fo: + { + if(ch == 'r') + { + lastIndex = index; + state = State.For; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.For: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.For; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.L: + { + if(ch == 'o') + { + lastIndex = index; + state = State.Lo; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Lo: + { + if(ch == 'c') + { + lastIndex = index; + state = State.Loc; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Loc: + { + if(ch == 'a') + { + lastIndex = index; + state = State.Loca; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Loca: + { + if(ch == 'l') + { + lastIndex = index; + state = State.Local; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Local: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Local; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.U: + { + if(ch == 'n') + { + lastIndex = index; + state = State.Un; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Un: + { + if(ch == 't') + { + lastIndex = index; + state = State.Unt; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Unt: + { + if(ch == 'i') + { + lastIndex = index; + state = State.Unti; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Unti: + { + if(ch == 'l') + { + lastIndex = index; + state = State.Until; + appendDataChar(ch); + } + else if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Name; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + case State.Until: + { + if(char.IsAsciiLetterOrDigit(ch) || ch == '_') + { + lastIndex = index; + state = State.Name; + currentToken!.type = TokenType.Name; + appendDataChar(ch); + } + else + { + if(currentToken == null || currentToken.type == null) + { + throw new Exception($"Lexer error at {currentLocation}"); + } + currentToken.type = TokenType.Until; + currentToken.data = null; + currentLocation = new(currentToken.region.end); + tokens.Add(currentToken); + currentToken = null; + index = lastIndex!.Value; + lastIndex = null; + state = State.Start; + } + } + break; + + default: + throw new NotImplementedException(state.ToString()); + } + } + + private enum State + { + Start, + Quote, SingleQuote, Name, Number, Zero, + A, B, D, E, F, G, I, L, N, O, R, T, U, W, + Plus, Minus, Star, Slash, Percent, Caret, Hash, + Ampersand, Tilde, Pipe, Lt, Gt, Equals, RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, StringStartLongBracket, StringWithLongBracket, StringEndLongBracket, + Colon, Semicolon, Comma, Dot, + + An, Br, Do, El, En, Fa, Fo, Fu, Go, If, In, Lo, Ni, No, Or, Re, Th, Tr, Un, Wh, + LtLt, GtGt, SlashSlash, EqualsEquals, TildeEquals, LtEquals, GtEquals, ColonColon, DotDot, + SmallCommentStart, QuoteBackslash, SingleQuoteBackslash, String, HexNumberX, ExpNumber, + + And, Bre, Els, End, Fal, For, Fun, Got, Loc, Nil, Not, Rep, Ret, The, Tru, Unt, Whi, + DotDotDot, HexNumber, QuoteBackslashZ, SingleQuoteBackslashZ, + SmallComment, BigComment, BigCommentStartLongBracket, BigCommentEndLongBracket, + + Brea, Else, Fals, Func, Goto, Loca, Repe, Retu, Then, True, Unti, Whil, HexExpNumber, + + Break, Elsei, False, Funct, Local, Repea, Retur, Until, While, + + Elseif, Functi, Repeat, Return, + + Functio, + + Function, + } +} + +class CodeRegion(CodeLocation start, CodeLocation end) +{ + public CodeLocation start = start; + public CodeLocation end = end; + + public override string ToString() + { + return $"{start}-{end}"; + } +} + +class CodeLocation(int line, int col) +{ + public int line = line; + public int col = col; + + public CodeLocation(CodeLocation other) : this(line: other.line, col: other.col) { } + + public override string ToString() + { + return $"{line + 1}:{col + 1}"; + } +} + +class Token(CodeRegion region, TokenType? type = null, Token.Data? data = null) +{ + public CodeRegion region = region; + public Data? data = data; + public TokenType? type = type; + + public interface Data { } + public class NumeralData(Numeral numeral) : Data + { + public Numeral numeral = numeral; + public override string ToString() + { + return $"NumeralData {numeral}"; + } + } + public class StringData(string data) : Data + { + public string data = data; + public override string ToString() + { + return $"StringData \"{data}\""; + } + } +} +public enum TokenType +{ + Name, + And, Break, Do, Else, Elseif, End, + False, For, Function, Goto, If, In, + Local, Nil, Not, Or, Repeat, Return, + Then, True, Until, While, + Plus, Minus, Star, Slash, Percent, Caret, Hash, + Ampersand, Tilde, Pipe, LtLt, GtGt, SlashSlash, + EqualsEquals, TildeEquals, LtEquals, GtEquals, Lt, Gt, Equals, + RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, ColonColon, + Semicolon, Colon, Comma, Dot, DotDot, DotDotDot, + Numeral, + StringLiteral, +} + +public interface Numeral +{ + public class Integer(int value) : Numeral + { + public int value = value; + + public bool rawEqual(Numeral other) + { + if(other is Numeral.Integer) + { + return ((Numeral.Integer)other).value == value; + } + // TODO: Check if this is actually doing what is expected + return ((Numeral.Float)other).value == value; + } + public override string ToString() + { + return $"Numeral Integer {value}"; + } + } + public class Float(float value) : Numeral + { + public float value = value; + + public bool rawEqual(Numeral other) + { + if(other is Numeral.Float) + { + return ((Numeral.Float)other).value == value; + } + // TODO: Check if this is actually doing what is expected + return ((Numeral.Integer)other).value == value; + } + public override string ToString() + { + return $"Numeral Float {value}"; + } + } + + public bool rawEqual(Numeral other); +} \ No newline at end of file diff --git a/build.zig b/build.zig deleted file mode 100644 index 9f8cd5d..0000000 --- a/build.zig +++ /dev/null @@ -1,83 +0,0 @@ -const std = @import("std"); - -// Although this function looks imperative, note that its job is to -// declaratively construct a build graph that will be executed by an external -// runner. -pub fn build(b: *std.Build) void { - // Standard target options allows the person running `zig build` to choose - // what target to build for. Here we do not override the defaults, which - // means any target is allowed, and the default is native. Other options - // for restricting supported target set are available. - const target = b.standardTargetOptions(.{}); - - // Standard optimization options allow the person running `zig build` to select - // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall. Here we do not - // set a preferred release mode, allowing the user to decide how to optimize. - const optimize = b.standardOptimizeOption(.{}); - - const exe = b.addExecutable(.{ - .name = "luaaaaah_zig", - // In this case the main source file is merely a path, however, in more - // complicated build scripts, this could be a generated file. - .root_source_file = .{ .path = "src/main.zig" }, - .target = target, - .optimize = optimize, - }); - - exe.addModule("types", b.addModule("types", .{ - .source_file = .{ .path = "src/types.zig" } - })); - exe.addModule("tokenizer", b.addModule("tokenizer", .{ - .source_file = .{ .path = "src/tokenizer.zig" }, - })); - exe.addModule("parser", b.addModule("parser", .{ - .source_file = .{ .path = "src/parser.zig" }, - })); - exe.addModule("treewalker", b.addModule("treewalker", .{ - .source_file = .{ .path = "src/treewalker.zig" }, - })); - - // This declares intent for the executable to be installed into the - // standard location when the user invokes the "install" step (the default - // step when running `zig build`). - b.installArtifact(exe); - - // This *creates* a Run step in the build graph, to be executed when another - // step is evaluated that depends on it. The next line below will establish - // such a dependency. - const run_cmd = b.addRunArtifact(exe); - - // By making the run step depend on the install step, it will be run from the - // installation directory rather than directly from within the cache directory. - // This is not necessary, however, if the application depends on other installed - // files, this ensures they will be present and in the expected location. - run_cmd.step.dependOn(b.getInstallStep()); - - // This allows the user to pass arguments to the application in the build - // command itself, like this: `zig build run -- arg1 arg2 etc` - if (b.args) |args| { - run_cmd.addArgs(args); - } - - // This creates a build step. It will be visible in the `zig build --help` menu, - // and can be selected like this: `zig build run` - // This will evaluate the `run` step rather than the default, which is "install". - const run_step = b.step("run", "Run the app"); - run_step.dependOn(&run_cmd.step); - - // Creates a step for unit testing. This only builds the test executable - // but does not run it. - const unit_tests = b.addTest(.{ - .root_source_file = .{ .path = "src/main.zig" }, - .target = target, - .optimize = optimize, - }); - - const run_unit_tests = b.addRunArtifact(unit_tests); - - // Similar to creating the run step earlier, this exposes a `test` step to - // the `zig build --help` menu, providing a way for the user to request - // running the unit tests. - const test_step = b.step("test", "Run unit tests"); - test_step.dependOn(&run_unit_tests.step); -} diff --git a/luaaaaah.csproj b/luaaaaah.csproj new file mode 100644 index 0000000..d06603b --- /dev/null +++ b/luaaaaah.csproj @@ -0,0 +1,9 @@ + + + Exe + net8.0 + enable + enable + + \ No newline at end of file diff --git a/src/main.zig b/src/main.zig deleted file mode 100644 index ff437e1..0000000 --- a/src/main.zig +++ /dev/null @@ -1,40 +0,0 @@ -const std = @import("std"); -const tokenize = @import("tokenizer.zig").tokenize; -const parse = @import("parser.zig").parse; -const treewalk = @import("treewalker.zig").interpret; - -pub fn main() !void -{ - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - const args = try std.process.argsAlloc(allocator); - defer std.process.argsFree(allocator, args); - const file = try std.fs.cwd().openFile(args[1], .{}); - defer file.close(); - const content = try file.readToEndAlloc(allocator, 13000); - defer allocator.free(content); - const tokens = try tokenize(content, allocator); - defer - { - var i: usize = 0; - while(i < tokens.len) - { - switch(tokens[i].tokenData) - { - .string => |*data| - { - allocator.free(data.*); - }, - else => {} - } - i += 1; - } - allocator.free(tokens); - } - var parserAllocator = std.heap.ArenaAllocator.init(std.heap.page_allocator); - defer parserAllocator.deinit(); - const root = try parse(tokens, &parserAllocator); - try treewalk(root, allocator); -} diff --git a/src/parser.zig b/src/parser.zig deleted file mode 100644 index 19dea29..0000000 --- a/src/parser.zig +++ /dev/null @@ -1,2572 +0,0 @@ -const Token = @import("tokenizer.zig").Token; -const TokenType = @import("tokenizer.zig").TokenType; -const std = @import("std"); -const types = @import("types.zig"); -const CodeRegion = @import("types.zig").CodeRegion; - -pub const ChunkNode = struct -{ - block: BlockNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - pub fn dump(self: *const ChunkNode, indent: usize) void - { - std.debug.print("ChunkNode ({} - {}):\n", .{self.startRegion, self.endRegion}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("block: ", .{}); - self.block.dump(indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; - -pub const BlockNode = struct -{ - stats: std.ArrayList(StatNode), - retstat: ?RetstatNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const BlockNode, indent: usize) void - { - std.debug.print("BlockNode:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("stats:\n", .{}); - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("[\n", .{}); - for(self.stats.items) |stat| - { - for (0..indent + 2) |_| - { - std.debug.print("\t", .{}); - } - dumpStatNode(stat, indent + 2); - } - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("]\n", .{}); - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("retstat: ", .{}); - if(self.retstat == null) - { - std.debug.print("null\n", .{}); - } - else - { - self.retstat.?.dump(indent + 1); - } - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const RetstatNode = struct -{ - values: ?ExplistNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const RetstatNode, indent: usize) void - { - std.debug.print("Retstat Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("values: ", .{}); - if(self.values == null) - { - std.debug.print("null\n", .{}); - } - else - { - self.values.?.dump(indent + 1); - } - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; - -pub const StatNode = union(enum) -{ - Semicolon, - Assignment: AssignmentNode, - Functioncall: FunctioncallNode, - Label: []u8, - Break, - Goto: []u8, - Do: BlockNode, - While: WhileNode, - Repeat: RepeatNode, - If: IfNode, - ForNumerical: ForNumericalNode, - ForGeneric: ForGenericNode, - Function: FunctionNode, - LocalFunction: LocalFunctionNode, - Local: LocalNode, -}; -fn dumpStatNode(stat: StatNode, indent: usize) void -{ - switch(stat) - { - .Semicolon => std.debug.print("Semicolon\n", .{}), - .Assignment => |*node| node.dump(indent), - .Functioncall => |*node| node.dump(indent), - .Label => |*label| std.debug.print("Label: '{s}'\n", .{label.*}), - .Break => std.debug.print("Break\n", .{}), - .Goto => |*label| std.debug.print("Goto: '{s}'\n", .{label.*}), - .Do => |*node| node.dump(indent), - .While => |*node| node.dump(indent), - .Repeat => |*node| node.dump(indent), - .If => |*node| node.dump(indent), - .ForNumerical => |*node| node.dump(indent), - .ForGeneric => |*node| node.dump(indent), - .Function => |*node| node.dump(indent), - .LocalFunction => |*node| node.dump(indent), - .Local => |*node| node.dump(indent), - } -} - -pub const AssignmentNode = struct -{ - lhs: VarlistNode, rhs: ExplistNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const AssignmentNode, indent: usize) void - { - std.debug.print("Assignment Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("lhs: ", .{}); - self.lhs.dump(indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("rhs: ", .{}); - self.rhs.dump(indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const WhileNode = struct -{ - condition: ExpNode, - body: BlockNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const WhileNode, indent: usize) void - { - std.debug.print("While Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("condition: ", .{}); - dumpExpNode(self.condition, indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("body: ", .{}); - self.body.dump(indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const RepeatNode = struct -{ - condition: ExpNode, - body: BlockNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const RepeatNode, indent: usize) void - { - std.debug.print("Repeat Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("condition: ", .{}); - dumpExpNode(self.condition, indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("body: ", .{}); - self.body.dump(indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const IfNode = struct -{ - condition: ExpNode, - body: BlockNode, - elseifs: std.ArrayList(ElseifNode), - else_: ?BlockNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const IfNode, indent: usize) void - { - std.debug.print("If Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("condition: ", .{}); - dumpExpNode(self.condition, indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("body: ", .{}); - self.body.dump(indent + 1); - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("elseifs:\n", .{}); - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("[\n", .{}); - for(self.elseifs.items) |elseif| - { - for (0..indent + 2) |_| - { - std.debug.print("\t", .{}); - } - elseif.dump(indent + 2); - } - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("]\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("else: ", .{}); - if(self.else_ == null) - { - std.debug.print("null\n", .{}); - } - else - { - self.else_.?.dump(indent + 1); - } - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const ForNumericalNode = struct -{ - variable: []u8, - start: ExpNode, - end: ExpNode, - change: ?ExpNode, - body: BlockNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const ForNumericalNode, indent: usize) void - { - std.debug.print("For Numerical Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("variable: '{s}'\n", .{self.variable}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("start: ", .{}); - dumpExpNode(self.start, indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("end: ", .{}); - dumpExpNode(self.end, indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("change: ", .{}); - if(self.change == null) - { - std.debug.print("null\n", .{}); - } - else - { - dumpExpNode(self.start, indent + 1); - } - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("body: ", .{}); - self.body.dump(indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const ForGenericNode = struct -{ - vars: std.ArrayList([]u8), - exps: ExplistNode, - body: BlockNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const ForGenericNode, indent: usize) void - { - std.debug.print("For Generic Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("vars:\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("[\n", .{}); - for(self.vars.items) |v| - { - for (0..(indent + 2)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("'{s}'\n", .{v}); - } - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("]\n", .{}); - std.debug.print("exps: ", .{}); - self.exps.dump(indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("body: ", .{}); - self.body.dump(indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const FunctionNode = struct -{ - name: FuncnameNode, - body: FuncbodyNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const FunctionNode, indent: usize) void - { - std.debug.print("Function Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("name: ", .{}); - self.name.dump(indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("body: ", .{}); - self.body.dump(indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const LocalFunctionNode = struct -{ - name: []u8, - body: FuncbodyNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const LocalFunctionNode, indent: usize) void - { - std.debug.print("Local Function Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("name: '{s}'\n", .{self.name}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("body: ", .{}); - self.body.dump(indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } - -}; -pub const LocalNode = struct -{ - attnames: AttnamelistNode, - values: ?ExplistNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const LocalNode, indent: usize) void - { - std.debug.print("Local Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("attnames: ", .{}); - self.attnames.dump(indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("values: ", .{}); - if(self.values == null) - { - std.debug.print("null\n", .{}); - } - else - { - self.values.?.dump(indent + 1); - } - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const FunctioncallNode = struct -{ - function: SuffixexpNode, - objectArg: ?[]u8, - args: ArgsNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const FunctioncallNode, indent: usize) void - { - std.debug.print("Functioncall Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("function: ", .{}); - dumpSuffixExpNode(self.function, indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("object arg: ", .{}); - if(self.objectArg == null) - { - std.debug.print("null\n", .{}); - } - else - { - std.debug.print("'{s}'\n", .{self.objectArg.?}); - } - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("args: ", .{}); - dumpArgsNode(self.args, indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const VarlistNode = struct -{ - vars: std.ArrayList(VarNode), - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const VarlistNode, indent: usize) void - { - std.debug.print("Varlist Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("vars:\n", .{}); - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("[\n", .{}); - for(self.vars.items) |item| - { - for (0..indent + 2) |_| - { - std.debug.print("\t", .{}); - } - dumpVarNode(item, indent + 2); - } - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("]\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const ExplistNode = struct -{ - exps: std.ArrayList(ExpNode), - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const ExplistNode, indent: usize) void - { - std.debug.print("Explist Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("exps:\n", .{}); - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("[\n", .{}); - for(self.exps.items) |item| - { - for (0..indent + 2) |_| - { - std.debug.print("\t", .{}); - } - dumpExpNode(item, indent + 2); - } - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("]\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const ExpNode = union(enum) -{ - Nil, - False, - True, - Numeral: types.Numeral, - LiteralString: []u8, - Varargs, - Functiondef: FuncbodyNode, - Suffixexp: *SuffixexpNode, - Tableconstructor: TableconstructorNode, - Unop: UnopNode, - Binop: *BinopNode, -}; -fn dumpExpNode(expNode: ExpNode, indent: usize) void -{ - switch(expNode) - { - .Nil => std.debug.print("Nil\n", .{}), - .False => std.debug.print("False\n", .{}), - .True => std.debug.print("True\n", .{}), - .Numeral => |*numeral| std.debug.print("{}\n", .{numeral.*}), - .LiteralString => |*string| std.debug.print("LiteralString: '{s}'\n", .{string.*}), - .Varargs => std.debug.print("Varargs", .{}), - .Functiondef => |*node| node.dump(indent), - .Suffixexp => |*node| dumpSuffixExpNode(node.*.*, indent), - .Tableconstructor => |*node| node.dump(indent), - .Unop => |*node| node.dump(indent), - .Binop => |*node| node.*.dump(indent), - } -} -pub const UnopNode = struct -{ - unopType: UnopType, - exp: *ExpNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const UnopNode, indent: usize) void - { - std.debug.print("Unop Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("unop type: {}\n", .{self.unopType}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("exp: ", .{}); - dumpExpNode(self.exp.*, indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const BinopNode = struct -{ - lhs: ExpNode, - op: BinopType, - rhs: ExpNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const BinopNode, indent: usize) void - { - std.debug.print("Binop Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("lhs: ", .{}); - dumpExpNode(self.lhs, indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("op: {}\n", .{self.op}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("rhs: ", .{}); - dumpExpNode(self.rhs, indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const ElseifNode = struct -{ - condition: ExpNode, - body: BlockNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const ElseifNode, indent: usize) void - { - std.debug.print("Elseif Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("condition: ", .{}); - dumpExpNode(self.condition, indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("body: ", .{}); - self.body.dump(indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const FuncnameNode = struct -{ - name: []u8, - dottedNames: std.ArrayList([]u8), - firstArg: ?[]u8, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const FuncnameNode, indent: usize) void - { - std.debug.print("Funcname Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("name: '{s}'\n", .{self.name}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("dottedNames:\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("[\n", .{}); - for(self.dottedNames.items) |dottedName| - { - for (0..(indent + 2)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("'{s}'\n", .{dottedName}); - } - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("]\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("firstArg: ", .{}); - if(self.firstArg == null) - { - std.debug.print("null\n", .{}); - } - else - { - std.debug.print("'{s}'\n", .{self.firstArg.?}); - } - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } - -}; -pub const FuncbodyNode = struct -{ - pars: ?ParlistNode, - body: BlockNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const FuncbodyNode, indent: usize) void - { - std.debug.print("Funcbody Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("pars: ", .{}); - if(self.pars == null) - { - std.debug.print("null\n", .{}); - } - else - { - self.pars.?.dump(indent + 1); - } - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - self.body.dump(indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const AttnamelistNode = struct -{ - attnames: std.ArrayList(AttnameNode), - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const AttnamelistNode, indent: usize) void - { - std.debug.print("Attnamelist Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("attNames:\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("[\n", .{}); - for(self.attnames.items) |attNames| - { - for (0..(indent + 2)) |_| - { - std.debug.print("\t", .{}); - } - attNames.dump(indent + 2); - } - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("]\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const AttnameNode = struct -{ - name: []u8, - attribute: ?[]u8, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const AttnameNode, indent: usize) void - { - std.debug.print("Funcname Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("name: '{s}'\n", .{self.name}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("attribute: ", .{}); - if(self.attribute == null) - { - std.debug.print("null\n", .{}); - } - else - { - std.debug.print("'{s}'\n", .{self.attribute.?}); - } - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const SuffixexpNode = union(enum) -{ - Normal: NormalSuffixNode, - Functioncall: *FunctioncallNode, -}; -fn dumpSuffixExpNode(suffixexpNode: SuffixexpNode, indent: usize) void -{ - switch(suffixexpNode) - { - .Normal => |*node| node.dump(indent), - .Functioncall => |*node| node.*.dump(indent), - } -} -pub const ArgsNode = union(enum) -{ - Bracketed: ?ExplistNode, - Tableconstructor: TableconstructorNode, - Literal: []u8, -}; -fn dumpArgsNode(argsNode: ArgsNode, indent: usize) void -{ - switch(argsNode) - { - .Bracketed => |*name| - { - if(name.* == null) - { - std.debug.print("null\n", .{}); - } - else - { - name.*.?.dump(indent); - } - }, - .Tableconstructor => |*node| node.dump(indent), - .Literal => |*string| std.debug.print("Literal: '{s}'\n", .{string}), - } -} -pub const VarNode = union(enum) -{ - Name: []u8, - Indexed: IndexedVarNode, - Member: MemberVarNode, -}; -fn dumpVarNode(varNode: VarNode, indent: usize) void -{ - switch(varNode) - { - .Name => |*name| std.debug.print("Name: '{s}'\n", .{name.*}), - .Indexed => |*node| node.dump(indent), - .Member => |*node| node.dump(indent), - } -} -pub const IndexedVarNode = struct -{ - value: SuffixexpNode, - index: ExpNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const IndexedVarNode, indent: usize) void - { - std.debug.print("Indexed Var Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("value: ", .{}); - dumpSuffixExpNode(self.value, indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("index: ", .{}); - dumpExpNode(self.index, indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const MemberVarNode = struct -{ - value: SuffixexpNode, - name: []u8, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const MemberVarNode, indent: usize) void - { - std.debug.print("Member Var Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("value: ", .{}); - dumpSuffixExpNode(self.value, indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("name: '{s}'\n", .{self.name}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } - -}; -pub const TableconstructorNode = struct -{ - exps: ?FieldlistNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const TableconstructorNode, indent: usize) void - { - std.debug.print("Tableconstructor Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("exps: ", .{}); - if(self.exps == null) - { - std.debug.print("null\n", .{}); - } - else - { - self.exps.?.dump(indent + 1); - } - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const UnopType = enum -{ - Minus, LogicalNot, Length, BinaryNot, -}; -pub const BinopType = enum -{ - LogicalOr, - LocicalAnd, - Lt, Gt, LtEquals, GtEquals, NotEquals, Equals, - BinaryOr, - BinaryNot, - BinaryAnd, - Shl, Shr, - Concat, - Add, Sub, - Mul, Div, IntDiv, Mod, - Exp, -}; -pub const ParlistNode = struct -{ - names: std.ArrayList([]u8), - hasVarargs: bool, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const ParlistNode, indent: usize) void - { - std.debug.print("Parlist Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("names:\n", .{}); - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("[\n", .{}); - for(self.names.items) |name| - { - for (0..indent + 2) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("'{s}'\n", .{name}); - } - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("]\n", .{}); - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("has Varargs: {}\n", .{self.hasVarargs}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const NormalSuffixNode = struct -{ - firstPart: SuffixexpFirstPart, - suffixes: std.ArrayList(SuffixexpSuffix), - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const NormalSuffixNode, indent: usize) void - { - std.debug.print("Normal Suffix Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("First Part: ", .{}); - dumpSuffixExpFirstPart(self.firstPart, indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("Suffixes:\n", .{}); - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("[\n", .{}); - for(self.suffixes.items) |suffix| - { - for (0..(indent + 2)) |_| - { - std.debug.print("\t", .{}); - } - dumpSuffixSuffix(suffix, indent + 2); - } - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("]\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const SuffixexpFirstPart = union(enum) -{ - Name: []u8, - BracketedExpr: ExpNode, -}; -fn dumpSuffixExpFirstPart(suffixexpFirstPart: SuffixexpFirstPart, indent: usize) void -{ - switch(suffixexpFirstPart) - { - .Name => |*name| std.debug.print("Name: '{s}'\n", .{name.*}), - .BracketedExpr => |*node| dumpExpNode(node.*, indent), - } -} -pub const SuffixexpSuffix = union(enum) -{ - Dot: []u8, - Indexed: ExpNode, - Args: ArgsNode, - ArgsFirstArg: ArgsFirstArgNode, -}; -fn dumpSuffixSuffix(suffixexpSuffix: SuffixexpSuffix, indent: usize) void -{ - switch(suffixexpSuffix) - { - .Dot => |*name| std.debug.print("Dot: '{s}'\n", .{name.*}), - .Indexed => |*node| dumpExpNode(node.*, indent), - .Args => |*node| dumpArgsNode(node.*, indent), - .ArgsFirstArg => |*node| node.dump(indent), - } -} - -pub const ArgsFirstArgNode = struct -{ - name: []u8, - rest: ArgsNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const ArgsFirstArgNode, indent: usize) void - { - std.debug.print("Args First Arg Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("name: '{s}'\n", .{self.name}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("rest: ", .{}); - dumpArgsNode(self.rest, indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } - -}; -pub const FieldlistNode = struct -{ - exps: std.ArrayList(FieldNode), - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const FieldlistNode, indent: usize) void - { - std.debug.print("Fieldlist Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("exps: ", .{}); - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("[\n", .{}); - for(self.exps.items) |exp| - { - for (0..(indent + 2)) |_| - { - std.debug.print("\t", .{}); - } - dumpFieldNode(exp, indent + 2); - } - for (0..indent + 1) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("]\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -pub const FieldNode = union(enum) -{ - IndexedAssignment: IndexedAssignmentNode, - Assignment: FieldAssignmentNode, - Exp: ExpNode, -}; -pub const FieldAssignmentNode = struct -{ - lhs: []u8, - rhs: ExpNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const FieldAssignmentNode, indent: usize) void - { - std.debug.print("Field Assignment Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("lhs: {s}\n", .{self.lhs}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("rhs: ", .{}); - dumpExpNode(self.rhs, indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; -fn dumpFieldNode(fieldNode: FieldNode, indent: usize) void -{ - switch(fieldNode) - { - .IndexedAssignment => |*node| node.dump(indent), - .Assignment => |*node| node.dump(indent), - .Exp => |*node| dumpExpNode(node.*, indent), - } -} -pub const IndexedAssignmentNode = struct -{ - index: ExpNode, - rhs: ExpNode, - startRegion: CodeRegion, - endRegion: CodeRegion, - - fn dump(self: *const IndexedAssignmentNode, indent: usize) void - { - std.debug.print("Indexed Assignment Node:\n", .{}); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("{{\n", .{}); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("index: ", .{}); - dumpExpNode(self.index, indent + 1); - for (0..(indent + 1)) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("rhs: ", .{}); - dumpExpNode(self.rhs, indent + 1); - for (0..indent) |_| - { - std.debug.print("\t", .{}); - } - std.debug.print("}}\n", .{}); - } -}; - -pub fn parse(tokens: []Token, allocator: *std.heap.ArenaAllocator) !ChunkNode -{ - var i: usize = 0; - const maybeParsedChunk = parseChunk(tokens, &i, allocator) catch |err| - { - //std.debug.print("{any}: data: {any}, type: {any}\n", .{tokens[i].region, tokens[i].tokenData, tokens[i].tokenType}); - return err; - }; - return maybeParsedChunk; -} - -const ParserError = error -{ - NotImplemented, - ReachedEOFParsing, - ReachedEOFExpectedNameForGoto, - ExpectedNameForGoto, - MissingEndForDoBlock, - MissingDoAfterWhileCondition, - MissingEndForWhileBody, - ExpectedUntilAfterRepeatBody, - ExpectedThenAfterIfCondition, - ReachedEOFAfterIfBody, - ExpectedThenAfterElseifCondition, - ReachedEOFAfterElseifs, - ExpectedEndClosingIf, - ExpectedNameAfterFor, - ExpectedCommaAfterForEqStartValue, - ReachedEOFAfterForEqEndValue, - ExpectedDoAfterForEqHead, - ExpectedAnotherNameInForInNamelist, - ReachedEOFAfterNameInForInNamelist, - ExpectedInAfterForInNamelist, - ExpectedDoAfterForInExplist, - ExpectedEndAfterForInBody, - ExpectedEndAfterForEqBody, - UnexpectedTokenAfterFirstNameInFor, - ReachedEOFInLocal, - ExpectedLocalFunctionName, - ExpectedNameAfterDoubleColonInLabelDeclaration, - ExpectedDoubleColonAfterNameInLabelDeclaration, - ExpectedFunctioncall, - ExpectedEqAfterAssignmentVarList, - ReachedEOFInSuffixExp, - ExpectedRoundClosedClosingBracketedPrimaryExp, - UnexpectedTokenAsFirstPartOfSuffixExp, - ExpectedNameInDottedSuffixExp, - ExpectedSquareClosedClosingIndexedSuffixExp, - ExpectedNameInArgsFirstArgSuffixExp, - ExpectedDotOrIndexedSuffixWhenConvertingSuffixExpToVar, - ReachedEOFExpectedPrimaryExpression, - ReachedEOFInArgs, - ReachedEOFInBracketedArgs, - ExpectedRoundClosedClosingBracketedArgs, - UnexpectedTokenInArgs, - NoPrecedenceForOperator, - NoBinopTypeForOperator, - ExpectednameInAttribName, - ExpectedAttributeInAttrib, - ExpectedGtInAttrib, - ExpectedFuncname, - ExpectedNameInDottedFuncname, - ExpectedNameOfFirstArgInFuncname, - ExpectedRoundOpenStartingFuncbody, - ReachedEOFInFuncbodyParlist, - ExpectedRoundClosedClosingFuncbodyParlist, - ExpectedEndClosingFuncbody, - ReachedEOFInParlist, - ExpectedNameStartingParlist, - ReachedEOFInParlistNameList, - UnexpectedTokenInParlistNameList, - ExpectedReturnStartingRetstat, - ExpectedCurlyOpenOpeningTableconstructor, - ExpectedCurlyClosedClosingTableconstructor, - ReachedEOFInField, - ExpectedSquareClosedClosingIndexedField, - ExpectedEqualsInIndexedFieldExpression, - OutOfMemory, -}; - -fn parseChunk(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ChunkNode -{ - const block = try parseBlock(tokens, i, allocator); - return ChunkNode { .block = block, .startRegion = block.startRegion, .endRegion = block.endRegion }; -} -fn parseBlock(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) ParserError!BlockNode -{ - var ret = BlockNode { .stats = std.ArrayList(StatNode).init(allocator.*.allocator()), .retstat = null, .startRegion = tokens[i.*].region, .endRegion = tokens[i.*].region }; - while(i.* < tokens.len and - tokens[i.*].tokenType != TokenType.Return and - tokens[i.*].tokenType != TokenType.End and - tokens[i.*].tokenType != TokenType.Elseif and - tokens[i.*].tokenType != TokenType.Else - ) - { - try ret.stats.append(try parseStat(tokens, i, allocator)); - } - if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Return) - { - ret.retstat = try parseRetstat(tokens, i, allocator); - } - ret.endRegion = if(i.* - 1 < tokens.len) tokens[i.* - 1].region else tokens[tokens.len - 1].region; - return ret; -} -fn parseStat(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !StatNode -{ - if(i.* >= tokens.len) - { - return error.ReachedEOFParsing; - } - switch(tokens[i.*].tokenType) - { - TokenType.Semicolon => - { - i.* += 1; - return StatNode.Semicolon; - }, - TokenType.Break => - { - i.* += 1; - return StatNode.Break; - }, - TokenType.Goto => - { - i.* += 1; - if(i.* >= tokens.len) - { - return error.ReachedEOFExpectedNameForGoto; - } - if(tokens[i.*].tokenType == TokenType.Name) - { - const name = tokens[i.*].tokenData.string; - i.* += 1; - return StatNode { .Goto = name }; - } - return error.ExpectedNameForGoto; - }, - TokenType.Do => - { - i.* += 1; - const body = try parseBlock(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End) - { - return error.MissingEndForDoBlock; - } - i.* += 1; - return StatNode { .Do = body }; - }, - TokenType.While => - { - const startRegion = tokens[i.*].region; - i.* += 1; - const condition = try parseExp(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Do) - { - return error.MissingDoAfterWhileCondition; - } - const body = try parseBlock(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End) - { - return error.MissingEndForWhileBody; - } - const endRegion = tokens[i.*].region; - i.* += 1; - return StatNode { .While = WhileNode { .body = body, .condition = condition, .startRegion = startRegion, .endRegion = endRegion } }; - }, - TokenType.Repeat => - { - const startRegion = tokens[i.*].region; - i.* += 1; - const body = try parseBlock(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Until) - { - return error.ExpectedUntilAfterRepeatBody; - } - const endRegion = tokens[i.*].region; - i.* += 1; - return StatNode { .Repeat = RepeatNode { .body = body, .condition = try parseExp(tokens, i, allocator), .startRegion = startRegion, .endRegion = endRegion } }; - }, - TokenType.If => - { - const startRegion = tokens[i.*].region; - i.* += 1; - const condition = try parseExp(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Then) - { - return error.ExpectedThenAfterIfCondition; - } - i.* += 1; - const body = try parseBlock(tokens, i, allocator); - if(i.* >= tokens.len) - { - return error.ReachedEOFAfterIfBody; - } - var ifNode = IfNode { .body = body, .condition = condition, .elseifs = std.ArrayList(ElseifNode).init(allocator.*.allocator()), .else_ = null, .startRegion = startRegion, .endRegion = startRegion }; - while(tokens[i.*].tokenType == TokenType.Elseif) - { - i.* += 1; - const elseifCondition = try parseExp(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Then) - { - return error.ExpectedThenAfterElseifCondition; - } - const endRegion = tokens[i.*].region; - i.* += 1; - try ifNode.elseifs.append(ElseifNode { .body = try parseBlock(tokens, i, allocator), .condition = elseifCondition, .startRegion = startRegion, .endRegion = endRegion }); - } - if(i.* >= tokens.len) - { - return error.ReachedEOFAfterElseifs; - } - if(tokens[i.*].tokenType == TokenType.Else) - { - i.* += 1; - ifNode.else_ = try parseBlock(tokens, i, allocator); - } - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End) - { - return error.ExpectedEndClosingIf; - } - ifNode.endRegion = tokens[i.*].region; - i.* += 1; - return StatNode { .If = ifNode }; - }, - TokenType.For => - { - const startRegion = tokens[i.*].region; - i.* += 1; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name) - { - return error.ExpectedNameAfterFor; - } - const variable = tokens[i.*].tokenData.string; - i.* += 1; - switch(tokens[i.*].tokenType) - { - TokenType.Equals => - { - i.* += 1; - const start = try parseExp(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Comma) - { - return error.ExpectedCommaAfterForEqStartValue; - } - i.* += 1; - const end = try parseExp(tokens, i, allocator); - if(i.* >= tokens.len) - { - return error.ReachedEOFAfterForEqEndValue; - } - var change: ?ExpNode = null; - if(tokens[i.*].tokenType == TokenType.Comma) - { - i.* += 1; - change = try parseExp(tokens, i, allocator); - } - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Do) - { - return error.ExpectedDoAfterForEqHead; - } - i.* += 1; - const body = try parseBlock(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End) - { - return error.ExpectedEndAfterForEqBody; - } - const endRegion = tokens[i.*].region; - i.* += 1; - return StatNode { .ForNumerical = ForNumericalNode { .variable = variable, .start = start, .end = end, .change = change, .body = body, .startRegion = startRegion, .endRegion = endRegion } }; - }, - TokenType.Comma => - { - var names = std.ArrayList([]u8).init(allocator.*.allocator()); - try names.append(variable); - while(tokens[i.*].tokenType == TokenType.Comma) - { - i.* += 1; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name) - { - return error.ExpectedAnotherNameInForInNamelist; - } - try names.append(tokens[i.*].tokenData.string); - i.* += 1; - if(i.* >= tokens.len) - { - return error.ReachedEOFAfterNameInForInNamelist; - } - } - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.In) - { - return error.ExpectedInAfterForInNamelist; - } - i.* += 1; - const exps = try parseExplist(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Do) - { - return error.ExpectedDoAfterForInExplist; - } - i.* += 1; - const body = try parseBlock(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End) - { - return error.ExpectedEndAfterForInBody; - } - const endRegion = tokens[i.*].region; - i.* += 1; - return StatNode { .ForGeneric = ForGenericNode { .vars = names, .exps = exps, .body = body, .startRegion = startRegion, .endRegion = endRegion } }; - }, - TokenType.In => - { - i.* += 1; - const exps = try parseExplist(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Do) - { - return error.ExpectedDoAfterForInExplist; - } - i.* += 1; - const body = try parseBlock(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End) - { - return error.ExpectedEndAfterForInBody; - } - const endRegion = tokens[i.*].region; - i.* += 1; - var names = try std.ArrayList([]u8).initCapacity(allocator.allocator(), 1); - try names.insert(0, variable); - return StatNode { .ForGeneric = ForGenericNode { .vars = names, .exps = exps, .body = body, .startRegion = startRegion, .endRegion = endRegion } }; - }, - else => return error.UnexpectedTokenAfterFirstNameInFor, - } - }, - TokenType.Function => - { - const startRegion = tokens[i.*].region; - i.* += 1; - const name = try parseFuncname(tokens, i, allocator); - const body = try parseFuncbody(tokens, i, allocator); - return StatNode { .Function = FunctionNode { .name = name, .body = body, .startRegion = startRegion, .endRegion = body.endRegion } }; - }, - TokenType.Local => - { - const startRegion = tokens[i.*].region; - i.* += 1; - if(i.* >= tokens.len) - { - return error.ReachedEOFInLocal; - } - if(tokens[i.*].tokenType == TokenType.Function) - { - i.* += 1; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name) - { - return error.ExpectedLocalFunctionName; - } - const name = tokens[i.*].tokenData.string; - const endRegion = tokens[i.*].region; - i.* += 1; - return StatNode { .LocalFunction = LocalFunctionNode { .name = name, .body = try parseFuncbody(tokens, i, allocator), .startRegion = startRegion, .endRegion = endRegion } }; - } - else - { - var ret = LocalNode { .attnames = try parseAttnamelist(tokens, i, allocator), .values = null, .startRegion = startRegion, .endRegion = startRegion }; - if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Equals) - { - i.* += 1; - ret.values = try parseExplist(tokens, i, allocator); - ret.endRegion = ret.values.?.endRegion; - } - return StatNode { .Local = ret }; - } - }, - TokenType.ColonColon => - { - i.* += 1; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name) - { - return error.ExpectedNameAfterDoubleColonInLabelDeclaration; - } - const name = tokens[i.*].tokenData.string; - i.* += 1; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.ColonColon) - { - return error.ExpectedDoubleColonAfterNameInLabelDeclaration; - } - i.* += 1; - return StatNode { .Label = name }; - }, - TokenType.Name, TokenType.RoundOpen => - { - const startRegion = tokens[i.*].region; - const suffixExp = try parseSuffixExp(tokens, i, allocator); - if(i.* >= tokens.len) - { - switch(suffixExp) - { - .Normal => return error.ExpectedFunctioncall, - .Functioncall => |functioncall| return StatNode { .Functioncall = functioncall.* }, - } - } - else - { - switch(tokens[i.*].tokenType) - { - TokenType.Equals => - { - const endRegion = tokens[i.*].region; - i.* += 1; - var lhs = std.ArrayList(VarNode).init(allocator.allocator()); - try lhs.append(try suffixExpToVar(suffixExp, startRegion, endRegion)); - const rhs = try parseExplist(tokens, i, allocator); - return StatNode { .Assignment = AssignmentNode { .lhs = VarlistNode { .vars = lhs, .startRegion = endRegion, .endRegion = tokens[@min(i.*, tokens.len) - 1].region }, .rhs = rhs, .startRegion = startRegion, .endRegion = rhs.endRegion } }; - }, - TokenType.Comma => - { - var varlistNode = VarlistNode { .vars = std.ArrayList(VarNode).init(allocator.allocator()), .startRegion = startRegion, .endRegion = startRegion }; - try varlistNode.vars.append(try suffixExpToVar(suffixExp, startRegion, tokens[@min(i.*, tokens.len) - 1].region)); - while(tokens[i.*].tokenType == TokenType.Comma) - { - i.* += 1; - try varlistNode.vars.append(try parseVar(tokens, i, allocator)); - } - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Equals) - { - return error.ExpectedEqAfterAssignmentVarList; - } - varlistNode.endRegion = tokens[i.*].region; - i.* += 1; - const rhs = try parseExplist(tokens, i, allocator); - return StatNode { .Assignment = AssignmentNode { .lhs = varlistNode, .rhs = rhs, .startRegion = startRegion, .endRegion = rhs.endRegion } }; - }, - else => - { - switch(suffixExp) - { - .Normal => return error.ExpectedFunctioncall, - .Functioncall => |functioncall| return StatNode { .Functioncall = functioncall.* }, - } - } - } - } - }, - else => - { - std.debug.print("{}\n", .{tokens[i.*]}); - return error.NotImplemented; - } - } -} -fn parseRetstat(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !RetstatNode -{ - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Return) - { - return error.ExpectedReturnStartingRetstat; - } - const startRegion = tokens[i.*].region; - i.* += 1; - if(i.* >= tokens.len or - tokens[i.*].tokenType == TokenType.Semicolon or tokens[i.*].tokenType == TokenType.Else or tokens[i.*].tokenType == TokenType.Elseif or tokens[i.*].tokenType == TokenType.End) - { - if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Semicolon) - { - i.* += 1; - } - return RetstatNode { .values = null, .startRegion = startRegion, .endRegion = tokens[@min(i.*, tokens.len) - 1].region }; - } - const values = try parseExplist(tokens, i, allocator); - var endRegion = values.endRegion; - if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Semicolon) - { - endRegion = tokens[i.*].region; - i.* += 1; - } - return RetstatNode { .values = values, .startRegion = startRegion, .endRegion = endRegion }; -} -fn parseExp(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) ParserError!ExpNode -{ - const lhs = try parseExpPrimary(tokens, i, allocator); - return parseExpPrecedence(tokens, i, allocator, lhs, 0); -} -fn parseExpPrimary(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ExpNode -{ - if(i.* >= tokens.len) - { - return error.ReachedEOFExpectedPrimaryExpression; - } - const startRegion = tokens[i.*].region; - switch(tokens[i.*].tokenType) - { - TokenType.Nil => - { - i.* += 1; - return ExpNode.Nil; - }, - TokenType.True => - { - i.* += 1; - return ExpNode.True; - }, - TokenType.False => - { - i.* += 1; - return ExpNode.False; - }, - TokenType.Numeral => - { - const numeral = tokens[i.*].tokenData.numeral; - i.* += 1; - return ExpNode { .Numeral = numeral }; - }, - TokenType.StringLiteral => - { - const string = tokens[i.*].tokenData.string; - i.* += 1; - return ExpNode { .LiteralString = string }; - }, - TokenType.DotDotDot => - { - i.* += 1; - return ExpNode.Varargs; - }, - TokenType.Function => - { - i.* += 1; - return ExpNode { .Functiondef = try parseFuncbody(tokens, i, allocator) }; - }, - TokenType.CurlyOpen => return ExpNode { .Tableconstructor = try parseTableconstructor(tokens, i, allocator) }, - TokenType.Minus => - { - i.* += 1; - const unop = try allocator.allocator().create(ExpNode); - unop.* = try parseExp(tokens, i, allocator); - const endRegion = tokens[@min(i.*, tokens.len) - 1].region; - return ExpNode { .Unop = UnopNode { .unopType = UnopType.Minus, .exp = unop, .startRegion = startRegion, .endRegion = endRegion } }; - }, - TokenType.Hash => - { - i.* += 1; - const unop = try allocator.allocator().create(ExpNode); - unop.* = try parseExp(tokens, i, allocator); - const endRegion = tokens[@min(i.*, tokens.len) - 1].region; - return ExpNode { .Unop = UnopNode { .unopType = UnopType.Length, .exp = unop, .startRegion = startRegion, .endRegion = endRegion } }; - }, - TokenType.Not => - { - i.* += 1; - const unop = try allocator.allocator().create(ExpNode); - unop.* = try parseExp(tokens, i, allocator); - const endRegion = tokens[@min(i.*, tokens.len) - 1].region; - return ExpNode { .Unop = UnopNode { .unopType = UnopType.LogicalNot, .exp = unop, .startRegion = startRegion, .endRegion = endRegion } }; - }, - TokenType.Tilde => - { - i.* += 1; - const unop = try allocator.allocator().create(ExpNode); - unop.* = try parseExp(tokens, i, allocator); - const endRegion = tokens[@min(i.*, tokens.len) - 1].region; - return ExpNode { .Unop = UnopNode { .unopType = UnopType.BinaryNot, .exp = unop, .startRegion = startRegion, .endRegion = endRegion } }; - }, - else => - { - const suffixexp = try allocator.allocator().create(SuffixexpNode); - suffixexp.* = try parseSuffixExp(tokens, i, allocator); - return ExpNode { .Suffixexp = suffixexp }; - } - } -} -fn parseExpPrecedence(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator, lhs: ExpNode, minPrecedence: u8) !ExpNode -{ - var currentLhs = lhs; - while(i.* < tokens.len and isBinop(tokens[i.*])) - { - const startRegion = tokens[i.*].region; - const precedence = try getPrecedence(tokens[i.*]); - if(precedence < minPrecedence) - { - break; - } - const op = try getBinopType(tokens[i.*]); - i.* += 1; - var rhs = try parseExpPrimary(tokens, i, allocator); - while(i.* < tokens.len and isBinop(tokens[i.*]) and - (try getPrecedence(tokens[i.*]) > precedence or - (try getPrecedence(tokens[i.*]) == precedence and isRightAssociative(tokens[i.*])))) - { - const associativityBoost: u8 = if(try getPrecedence(tokens[i.*]) == precedence) 0 else 1; - rhs = try parseExpPrecedence(tokens, i, allocator, rhs, precedence + associativityBoost); - } - const binop = try allocator.allocator().create(BinopNode); - binop.* = BinopNode { .lhs = currentLhs, .op = op, .rhs = rhs, .startRegion = startRegion, .endRegion = tokens[@min(i.*, tokens.len) - 1].region }; - currentLhs = ExpNode { .Binop = binop }; - } - return currentLhs; -} -fn isRightAssociative(token: Token) bool -{ - return token.tokenType == TokenType.DotDot or token.tokenType == TokenType.Caret; -} -fn getBinopType(token: Token) !BinopType -{ - return switch(token.tokenType) - { - TokenType.Or => BinopType.LogicalOr, - TokenType.And => BinopType.LocicalAnd, - TokenType.Lt => BinopType.Lt, - TokenType.Gt => BinopType.Gt, - TokenType.LtEquals => BinopType.LtEquals, - TokenType.GtEquals => BinopType.GtEquals, - TokenType.LtLt => BinopType.Shl, - TokenType.GtGt=> BinopType.Shr, - TokenType.TildeEquals => BinopType.NotEquals, - TokenType.EqualsEquals => BinopType.Equals, - TokenType.Pipe => BinopType.BinaryOr, - TokenType.Tilde => BinopType.BinaryNot, - TokenType.Ampersand => BinopType.BinaryAnd, - TokenType.DotDot => BinopType.Concat, - TokenType.Plus => BinopType.Add, - TokenType.Minus => BinopType.Sub, - TokenType.Star => BinopType.Mul, - TokenType.Slash => BinopType.Div, - TokenType.SlashSlash => BinopType.IntDiv, - TokenType.Percent => BinopType.Mod, - TokenType.Caret => BinopType.Exp, - else => error.NoBinopTypeForOperator, - }; - -} -fn getPrecedence(token: Token) !u8 -{ - return switch(token.tokenType) - { - TokenType.Or => 2, - TokenType.And => 4, - TokenType.Lt, TokenType.Gt, TokenType.LtEquals, TokenType.GtEquals, TokenType.TildeEquals, TokenType.EqualsEquals => 6, - TokenType.Pipe => 8, - TokenType.Tilde => 10, - TokenType.Ampersand => 12, - TokenType.LtLt, TokenType.GtGt => 14, - TokenType.DotDot => 16, - TokenType.Plus, TokenType.Minus => 18, - TokenType.Star, TokenType.Slash, TokenType.SlashSlash, TokenType.Percent => 20, - TokenType.Caret => 22, - else => error.NoPrecedenceForOperator, - }; -} -fn isBinop(token: Token) bool -{ - return switch(token.tokenType) - { - TokenType.Or, TokenType.And, TokenType.Lt, TokenType.Gt, TokenType.LtEquals, TokenType.GtEquals, TokenType.TildeEquals, TokenType.EqualsEquals, - TokenType.Pipe, TokenType.Tilde, TokenType.Ampersand, TokenType.LtLt, TokenType.GtGt, TokenType.DotDot, TokenType.Plus, TokenType.Minus, - TokenType.Star, TokenType.Slash, TokenType.SlashSlash, TokenType.Percent, TokenType.Caret => true, - else => false - }; -} -fn parseExplist(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ExplistNode -{ - const startRegion = tokens[@min(i.*, tokens.len) - 1].region; - var ret = ExplistNode { .exps = std.ArrayList(ExpNode).init(allocator.allocator()), .startRegion = startRegion, .endRegion = startRegion }; - try ret.exps.append(try parseExp(tokens, i, allocator)); - while(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Comma) - { - i.* += 1; - try ret.exps.append(try parseExp(tokens, i, allocator)); - } - ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region; - return ret; -} -fn parseFuncname(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !FuncnameNode -{ - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name) - { - return error.ExpectedFuncname; - } - const startRange = tokens[i.*].region; - var ret = FuncnameNode { .name = tokens[i.*].tokenData.string, .dottedNames = std.ArrayList([]u8).init(allocator.allocator()), .firstArg = null, .startRegion = startRange, .endRegion = startRange }; - i.* += 1; - while(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Dot) - { - i.* += 1; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name) - { - return error.ExpectedNameInDottedFuncname; - } - try ret.dottedNames.append(tokens[i.*].tokenData.string); - } - if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Colon) - { - i.* += 1; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name) - { - return error.ExpectedNameOfFirstArgInFuncname; - } - ret.firstArg = tokens[i.*].tokenData.string; - i.* += 1; - } - ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region; - return ret; -} -fn parseFuncbody(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !FuncbodyNode -{ - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.RoundOpen) - { - return error.ExpectedRoundOpenStartingFuncbody; - } - const startRegion = tokens[i.*].region; - i.* += 1; - if(i.* >= tokens.len) - { - return error.ReachedEOFInFuncbodyParlist; - } - var pars: ?ParlistNode = null; - if(tokens[i.*].tokenType == TokenType.RoundClosed) - { - i.* += 1; - } - else - { - pars = try parseParlist(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.RoundClosed) - { - return error.ExpectedRoundClosedClosingFuncbodyParlist; - } - i.* += 1; - } - const ret = FuncbodyNode { .body = try parseBlock(tokens, i, allocator), .pars = pars, .startRegion = startRegion, .endRegion = tokens[i.*].region }; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End) - { - return error.ExpectedEndClosingFuncbody; - } - i.* += 1; - return ret; -} -fn parseParlist(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ParlistNode -{ - if(i.* >= tokens.len) - { - return error.ReachedEOFInParlist; - } - const startRegion = tokens[i.*].region; - if(tokens[i.*].tokenType == TokenType.DotDotDot) - { - const endRegion = tokens[i.*].region; - i.* += 1; - return ParlistNode { .names = std.ArrayList([]u8).init(allocator.allocator()), .hasVarargs = true, .startRegion = startRegion, .endRegion = endRegion }; - } - if(tokens[i.*].tokenType != TokenType.Name) - { - return error.ExpectedNameStartingParlist; - } - var ret = ParlistNode { .names = std.ArrayList([]u8).init(allocator.allocator()), .hasVarargs = false, .startRegion = startRegion, .endRegion = startRegion }; - try ret.names.append(tokens[i.*].tokenData.string); - i.* += 1; - while(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Comma) - { - i.* += 1; - if(i.* >= tokens.len) - { - return error.ReachedEOFInParlistNameList; - } - switch(tokens[i.*].tokenType) - { - TokenType.Name => - { - try ret.names.append(tokens[i.*].tokenData.string); - i.* += 1; - }, - TokenType.DotDotDot => - { - i.* += 1; - ret.hasVarargs = true; - break; - }, - else => return error.UnexpectedTokenInParlistNameList, - } - } - ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region; - return ret; -} -fn parseAttnamelist(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !AttnamelistNode -{ - // TODO: What happens if this is reaches EOF? - var ret = AttnamelistNode { .attnames = std.ArrayList(AttnameNode).init(allocator.allocator()), .startRegion = tokens[i.*].region, .endRegion = tokens[i.*].region }; - try ret.attnames.append(try parseAttname(tokens, i)); - while(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Comma) - { - i.* += 1; - try ret.attnames.append(try parseAttname(tokens, i)); - } - ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region; - return ret; -} -fn parseSuffixExp(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !SuffixexpNode -{ - // primaryexp { '.' 'Name' | '[' exp']' | ':' 'Name' args | args } - if(i.* >= tokens.len) - { - return error.ReachedEOFInSuffixExp; - } - const startRegion = tokens[i.*].region; - const firstPart = try switch(tokens[i.*].tokenType) - { - TokenType.Name => - nameBlock: { - const name = tokens[i.*].tokenData.string; - i.* += 1; - break :nameBlock SuffixexpFirstPart { .Name = name }; - }, - TokenType.RoundOpen => - roundOpenBlock: { - i.* += 1; - const ret = SuffixexpFirstPart { .BracketedExpr = try parseExp(tokens, i, allocator) }; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.RoundClosed) - { - return error.ExpectedRoundClosedClosingBracketedPrimaryExp; - } - i.* += 1; - break :roundOpenBlock ret; - }, - else => error.UnexpectedTokenAsFirstPartOfSuffixExp, - }; - var suffixes = std.ArrayList(SuffixexpSuffix).init(allocator.allocator()); - while(i.* < tokens.len) - { - switch(tokens[i.*].tokenType) - { - TokenType.Dot => - { - i.* += 1; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name) - { - return error.ExpectedNameInDottedSuffixExp; - } - const name = tokens[i.*].tokenData.string; - i.* += 1; - try suffixes.append(SuffixexpSuffix { .Dot = name }); - }, - TokenType.SquareOpen => - { - i.* += 1; - try suffixes.append(SuffixexpSuffix { .Indexed = try parseExp(tokens, i, allocator) }); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.SquareClosed) - { - return error.ExpectedSquareClosedClosingIndexedSuffixExp; - } - i.* += 1; - }, - TokenType.Colon => - { - const argsFirstArgStartRegion = tokens[i.*].region; - i.* += 1; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name) - { - return error.ExpectedNameInArgsFirstArgSuffixExp; - } - const name = tokens[i.*].tokenData.string; - const argsFirstArgEndRegion = tokens[i.*].region; - i.* += 1; - try suffixes.append(SuffixexpSuffix { .ArgsFirstArg = ArgsFirstArgNode { .name = name, .rest = try parseArgs(tokens, i, allocator), .startRegion = argsFirstArgStartRegion, .endRegion = argsFirstArgEndRegion } }); - }, - TokenType.RoundOpen, TokenType.CurlyOpen, TokenType.StringLiteral => - { - try suffixes.append(SuffixexpSuffix { .Args = try parseArgs(tokens, i, allocator) }); - }, - else => break, - } - } - const endRegion = tokens[@min(i.*, tokens.len) - 1].region; - const last = suffixes.getLastOrNull(); - if(last != null) - { - switch(last.?) - { - SuffixexpSuffix.Args => |*args| - { - _ = suffixes.pop(); - const functioncall = try allocator.allocator().create(FunctioncallNode); - functioncall.* = FunctioncallNode - { - .function = SuffixexpNode { .Normal = NormalSuffixNode { .firstPart = firstPart, .suffixes = suffixes, .startRegion = startRegion, .endRegion = endRegion } }, - .args = args.*, - .objectArg = null, - .startRegion = startRegion, - .endRegion = endRegion, - }; - return SuffixexpNode - { - .Functioncall = functioncall, - }; - }, - SuffixexpSuffix.ArgsFirstArg => |*node| - { - _ = suffixes.pop(); - const functioncall = try allocator.allocator().create(FunctioncallNode); - functioncall.* = FunctioncallNode - { - .function = SuffixexpNode { .Normal = NormalSuffixNode { .firstPart = firstPart, .suffixes = suffixes, .startRegion = startRegion, .endRegion = endRegion } }, - .args = node.rest, - .objectArg = node.name, - .startRegion = startRegion, - .endRegion = endRegion, - }; - return SuffixexpNode - { - .Functioncall = functioncall, - }; - }, - else => {} - } - } - return SuffixexpNode { .Normal = NormalSuffixNode { .firstPart = firstPart, .suffixes = suffixes, .startRegion = startRegion, .endRegion = endRegion } }; -} -fn parseVar(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !VarNode -{ - const startRegion = tokens[i.*].region; - return suffixExpToVar(try parseSuffixExp(tokens, i, allocator), startRegion, tokens[@min(i.*, tokens.len) - 1].region); -} -fn parseAttname(tokens: []Token, i: *usize) !AttnameNode -{ - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name) - { - return error.ExpectednameInAttribName; - } - const name = tokens[i.*].tokenData.string; - const startRegion = tokens[i.*].region; - i.* += 1; - var ret = AttnameNode { .name = name, .attribute = null, .startRegion = startRegion, .endRegion = startRegion }; - if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Lt) - { - ret.attribute = tokens[i.*].tokenData.string; - i.* += 1; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name) - { - return error.ExpectedAttributeInAttrib; - } - ret.endRegion = tokens[i.*].region; - i.* += 1; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Gt) - { - return error.ExpectedGtInAttrib; - } - } - return ret; -} -fn parseArgs(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ArgsNode -{ - if(i.* >= tokens.len) - { - return error.ReachedEOFInArgs; - } - switch(tokens[i.*].tokenType) - { - TokenType.RoundOpen => - { - i.* += 1; - if(i.* >= tokens.len) - { - return error.ReachedEOFInBracketedArgs; - } - if(tokens[i.*].tokenType == TokenType.RoundClosed) - { - i.* += 1; - return ArgsNode { .Bracketed = null }; - } - const exps = try parseExplist(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.RoundClosed) - { - return error.ExpectedRoundClosedClosingBracketedArgs; - } - i.* += 1; - return ArgsNode { .Bracketed = exps }; - }, - TokenType.CurlyOpen => return ArgsNode { .Tableconstructor = try parseTableconstructor(tokens, i, allocator) }, - TokenType.StringLiteral => - { - const value = tokens[i.*].tokenData.string; - i.* += 1; - return ArgsNode { .Literal = value }; - }, - else => return error.UnexpectedTokenInArgs, - } -} -fn parseTableconstructor(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !TableconstructorNode -{ - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.CurlyOpen) - { - return error.ExpectedCurlyOpenOpeningTableconstructor; - } - const startRegion = tokens[i.*].region; - i.* += 1; - if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.CurlyClosed) - { - const endRegion = tokens[i.*].region; - i.* += 1; - return TableconstructorNode { .exps = null, .startRegion = startRegion, .endRegion = endRegion }; - } - var ret = TableconstructorNode { .exps = try parseFieldlist(tokens, i, allocator), .startRegion = startRegion, .endRegion = startRegion }; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.CurlyClosed) - { - return error.ExpectedCurlyClosedClosingTableconstructor; - } - ret.endRegion = tokens[i.*].region; - i.* += 1; - return ret; -} -fn parseFieldlist(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !FieldlistNode -{ - const startRegion = tokens[@min(i.*, tokens.len) - 1].region; - var ret = FieldlistNode { .exps = std.ArrayList(FieldNode).init(allocator.allocator()), .startRegion = startRegion, .endRegion = startRegion }; - try ret.exps.append(try parseField(tokens, i, allocator)); - while(i.* < tokens.len and isFieldsep(tokens[i.*])) - { - i.* += 1; - try ret.exps.append(try parseField(tokens, i, allocator)); - } - if(i.* < tokens.len and isFieldsep(tokens[i.*])) - { - i.* += 1; - } - ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region; - return ret; -} -fn isFieldsep(token: Token) bool -{ - return token.tokenType == TokenType.Comma or token.tokenType == TokenType.Semicolon; -} -fn parseField(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !FieldNode -{ - if(i.* >= tokens.len) - { - return error.ReachedEOFInField; - } - const startRegion = tokens[i.*].region; - switch(tokens[i.*].tokenType) - { - TokenType.SquareOpen => - { - i.* += 1; - const index = try parseExp(tokens, i, allocator); - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.SquareClosed) - { - return error.ExpectedSquareClosedClosingIndexedField; - } - i.* += 1; - if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Equals) - { - return error.ExpectedEqualsInIndexedFieldExpression; - } - const endRegion = tokens[i.*].region; - i.* += 1; - return FieldNode { .IndexedAssignment = IndexedAssignmentNode { .index = index, .rhs = try parseExp(tokens, i, allocator), .startRegion = startRegion, .endRegion = endRegion } }; - }, - TokenType.Name => - { - if(i.* + 1 < tokens.len and tokens[i.* + 1].tokenType == TokenType.Equals) - { - const name = tokens[i.*].tokenData.string; - i.* += 2; - return FieldNode { .Assignment = FieldAssignmentNode { .lhs = name, .rhs = try parseExp(tokens, i, allocator), .startRegion = startRegion, .endRegion = tokens[i.* - 1].region } }; - } - return FieldNode { .Exp = try parseExp(tokens, i, allocator) }; - }, - else => return FieldNode { .Exp = try parseExp(tokens, i, allocator) }, - } -} -fn suffixExpToVar(suffixexp: SuffixexpNode, startRegion: CodeRegion, endRegion: CodeRegion) !VarNode -{ - var exp = suffixexp.Normal; - if(exp.suffixes.items.len == 0) - { - return VarNode { .Name = exp.firstPart.Name }; - } - const last = exp.suffixes.pop(); - return switch(last) - { - SuffixexpSuffix.Dot => |*name| VarNode { .Member = MemberVarNode { .name = name.*, .value = SuffixexpNode { .Normal = exp }, .startRegion = startRegion, .endRegion = endRegion } }, - SuffixexpSuffix.Indexed => |*index| VarNode { .Indexed = IndexedVarNode { .index = index.*, .value = SuffixexpNode { .Normal = exp }, .startRegion = startRegion, .endRegion = endRegion } }, - else => error.ExpectedDotOrIndexedSuffixWhenConvertingSuffixExpToVar, - }; -} diff --git a/src/tokenizer.zig b/src/tokenizer.zig deleted file mode 100644 index f0863dd..0000000 --- a/src/tokenizer.zig +++ /dev/null @@ -1,1246 +0,0 @@ -const types = @import("types.zig"); -const std = @import("std"); -const CodeRegion = @import("types.zig").CodeRegion; -const CodeLocation = @import("types.zig").CodeLocation; - -pub const TokenType = enum -{ - Name, - And, Break, Do, Else, Elseif, End, - False, For, Function, Goto, If, In, - Local, Nil, Not, Or, Repeat, Return, - Then, True, Until, While, - Plus, Minus, Star, Slash, Percent, Caret, Hash, - Ampersand, Tilde, Pipe, LtLt, GtGt, SlashSlash, - EqualsEquals, TildeEquals, LtEquals, GtEquals, Lt, Gt, Equals, - RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, ColonColon, - Semicolon, Colon, Comma, Dot, DotDot, DotDotDot, - Numeral, - StringLiteral, - -}; - -const TokenData = union(enum) -{ - string: []u8, - numeral: types.Numeral, - none, -}; - -pub const Token = struct -{ - tokenType: TokenType, - tokenData: TokenData, - region: CodeRegion, -}; - -const TokenizerState = enum -{ - Start, - Quote, SingleQuote, Name, Number, Zero, - A, B, D, E, F, G, I, L, N, O, R, T, U, W, - Plus, Minus, Star, Slash, Percent, Caret, Hash, - Ampersand, Tilde, Pipe, Lt, Gt, Equals, RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, - Colon, Semicolon, Comma, Dot, - - An, Br, Do, El, En, Fa, Fo, Fu, Go, If, In, Lo, Ni, No, Or, Re, Th, Tr, Un, Wh, - LtLt, GtGt, SlashSlash, EqualsEquals, TildeEquals, LtEquals, GtEquals, ColonColon, DotDot, - SmallCommentStart, QuoteBackslash, SingleQuoteBackslash, String, HexNumberX, ExpNumber, - - And, Bre, Els, End, Fal, For, Fun, Got, Loc, Nil, Not, Rep, Ret, The, Tru, Unt, Whi, - DotDotDot, HexNumber, QuoteBackslashZ, SingleQuoteBackslashZ, - BigCommentLongBracketStart, SmallComment, - - Brea, Else, Fals, Func, Goto, Loca, Repe, Retu, Then, True, Unti, Whil, HexExpNumber, - BigComment, BigCommentLongBracketEnd, - - Break, Elsei, False, Funct, Local, Repea, Retur, Until, While, - - Elseif, Functi, Repeat, Return, - - Functio, - - Function, -}; - -fn tokenizeUpdateIndexAndState(lastIndex: *?usize, index: ?usize, state: *TokenizerState, newState: TokenizerState, region: *CodeRegion) void -{ - lastIndex.* = index; - state.* = newState; - if(index == null) - { - region.*.start = null; - region.*.length = 0; - } - else - { - if(region.*.start == null) - { - // TODO: There is no line/col info here and plumbing it to here would be pain. - region.*.start = CodeLocation { .col = 0, .line = 0 }; - } - region.*.length += 1; - } -} -fn tokenizeTerminalBase(lastIndex: *?usize, index: ?usize, tokenType: *?TokenType, state: *TokenizerState, newTokenType: ?TokenType, newState: TokenizerState, region: *CodeRegion) void -{ - tokenizeUpdateIndexAndState(lastIndex, index, state, newState, region); - tokenType.* = newTokenType; -} -fn tokenizeTerminalStr(lastIndex: *?usize, index: usize, tokenType: *?TokenType, state: *TokenizerState, newTokenType: ?TokenType, newState: TokenizerState, tokenStr: *std.ArrayList(u8), ch: u8, region: *CodeRegion) !void -{ - tokenizeTerminalBase(lastIndex, index, tokenType, state, newTokenType, newState, region); - try tokenStr.append(ch); -} -fn tokenizeTerminalIntNum(lastIndex: *?usize, index: usize, tokenType: *?TokenType, state: *TokenizerState, newTokenType: TokenType, newState: TokenizerState, tokenNumeral: *?types.Numeral, ch: u8, region: *CodeRegion) !void -{ - tokenizeTerminalBase(lastIndex, index, tokenType, state, newTokenType, newState, region); - if(!std.ascii.isDigit(ch)) - { - return error.NoDigit; - } - const digitValue = @as(i64, ch - '0'); - if(tokenNumeral.* == null) - { - tokenNumeral.* = types.Numeral { .Integer = digitValue }; - } - else - { - switch(tokenNumeral.*.?) - { - .Integer => |*n| n.* = n.* * 10 + digitValue, - .Float => return error.ExpectedIntGotFloat - } - } -} -fn tokenizeTerminalNoToken(lastIndex: *?usize, index: usize, state: *TokenizerState, newState: TokenizerState, tokenStr: *std.ArrayList(u8), ch: u8, region: *CodeRegion) !void -{ - tokenizeUpdateIndexAndState(lastIndex, index, state, newState, region); - try tokenStr.*.append(ch); -} -fn tokenizeBacktrack(lastIndex: *?usize, index: *usize, tokens: *std.ArrayList(Token), tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, state: *TokenizerState, allocator: std.mem.Allocator, region: *CodeRegion) !void -{ - try tokenizeBacktrackCustomToken(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, tokenType.*.?, allocator, region); -} -fn tokenizeBacktrackCustomToken(lastIndex: *?usize, index: *usize, tokens: *std.ArrayList(Token), tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, state: *TokenizerState, newTokenType: TokenType, allocator: std.mem.Allocator, region: *CodeRegion) !void -{ - if(lastIndex.* == null or tokenType.* == null) - { - return error.LexError; - } - if(newTokenType == TokenType.StringLiteral or newTokenType == TokenType.Name) - { - const content = try allocator.alloc(u8, tokenStr.*.items.len); - @memcpy(content, tokenStr.*.items); - try tokens.append(Token { .tokenType = newTokenType, .tokenData = TokenData { .string = content }, .region = region.* }); - } - else - { - try tokens.append(Token { .tokenType = newTokenType, .region = region.*, .tokenData = if(tokenType.*.? == TokenType.Numeral) TokenData { .numeral = tokenNumeral.*.? } - else TokenData.none - }); - } - tokenNumeral.* = null; - index.* = lastIndex.*.?; - tokenStr.*.clearAndFree(); - // region is reset in tokenizeTerminalBase since null is passed as index - tokenizeTerminalBase(lastIndex, null, tokenType, state, null, TokenizerState.Start, region); -} -fn tokenizeAlphanumericNonstart(lastIndex: *?usize, index: *usize, tokens: *std.ArrayList(Token), tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, state: *TokenizerState, ch: u8, newTokenType: TokenType, allocator: std.mem.Allocator, region: *CodeRegion) !void -{ - if(std.ascii.isAlphanumeric(ch) or ch == '_') - { - try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.Name, tokenStr, ch, region); - } - else - { - try tokenizeBacktrackCustomToken(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, newTokenType, allocator, region); - } -} -fn tokenizeChar(state: *TokenizerState, ch: u8, lastIndex: *?usize, index: *usize, tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, tokens: *std.ArrayList(Token), longBracketLevel: *u32, region: *CodeRegion, allocator: std.mem.Allocator) !void -{ - switch(state.*) - { - TokenizerState.Start => - { - switch(ch) - { - '-' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Minus, TokenizerState.Minus, region), - ',' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Comma, TokenizerState.Comma, region), - '=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Equals, TokenizerState.Equals, region), - '(' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.RoundOpen, TokenizerState.RoundOpen, region), - ')' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.RoundClosed, TokenizerState.RoundClosed, region), - '.' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Dot, TokenizerState.Dot, region), - ':' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Colon, TokenizerState.Colon, region), - '{' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.CurlyOpen, TokenizerState.CurlyOpen, region), - '}' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.CurlyClosed, TokenizerState.CurlyClosed, region), - '[' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.SquareOpen, TokenizerState.SquareOpen, region), - ']' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.SquareClosed, TokenizerState.SquareClosed, region), - '+' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Plus, TokenizerState.Plus, region), - '~' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Tilde, TokenizerState.Tilde, region), - '>' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Gt, TokenizerState.Gt, region), - '<' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Lt, TokenizerState.Lt, region), - '#' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Hash, TokenizerState.Hash, region), - '|' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Pipe, TokenizerState.Pipe, region), - '&' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Ampersand, TokenizerState.Ampersand, region), - '%' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Percent, TokenizerState.Percent, region), - '*' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Star, TokenizerState.Star, region), - '/' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Slash, TokenizerState.Slash, region), - ';' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Semicolon, TokenizerState.Semicolon, region), - '^' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Caret, TokenizerState.Caret, region), - 'a' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.A, tokenStr, ch, region), - 'b' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.B, tokenStr, ch, region), - 'd' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.D, tokenStr, ch, region), - 'e' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.E, tokenStr, ch, region), - 'f' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.F, tokenStr, ch, region), - 'i' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.I, tokenStr, ch, region), - 'g' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.G, tokenStr, ch, region), - 'l' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.L, tokenStr, ch, region), - 'n' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.N, tokenStr, ch, region), - 'o' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.O, tokenStr, ch, region), - 'r' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.R, tokenStr, ch, region), - 't' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.T, tokenStr, ch, region), - 'u' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.U, tokenStr, ch, region), - 'w' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.W, tokenStr, ch, region), - '0' => try tokenizeTerminalIntNum(lastIndex, index.*, tokenType, state, TokenType.Numeral, TokenizerState.Zero, tokenNumeral, ch, region), - '"' => - { - tokenType.* = null; - state.* = TokenizerState.Quote; - }, - '\'' => - { - tokenType.* = null; - state.* = TokenizerState.SingleQuote; - }, - else => - { - if(std.ascii.isWhitespace(ch)) - { - - } - else if(std.ascii.isAlphabetic(ch) or ch == '_') - { - try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.Name, tokenStr, ch, region); - } - else if(std.ascii.isDigit(ch)) - { - try tokenizeTerminalIntNum(lastIndex, index.*, tokenType, state, TokenType.Numeral, TokenizerState.Number, tokenNumeral, ch, region); - } - else - { - std.debug.print("{}: {c}\n", .{state.*, ch}); - return error.NotImplemented; - } - } - } - }, - TokenizerState.Quote => - { - switch(ch) - { - '\\' => state.* = TokenizerState.QuoteBackslash, - '"' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region), - else => try tokenStr.*.append(ch), - } - }, - TokenizerState.QuoteBackslash => - { - switch(ch) - { - 'a' => - { - try tokenStr.append('\u{0007}'); - state.* = TokenizerState.Quote; - }, - 'b' => - { - try tokenStr.append('\u{0008}'); - state.* = TokenizerState.Quote; - }, - 't' => - { - try tokenStr.append('\t'); - state.* = TokenizerState.Quote; - }, - 'n' | '\n' => - { - try tokenStr.append('\n'); - state.* = TokenizerState.Quote; - }, - 'v' => - { - try tokenStr.append('\u{000b}'); - state.* = TokenizerState.Quote; - }, - 'f' => - { - try tokenStr.append('\u{000c}'); - state.* = TokenizerState.Quote; - }, - 'r' => - { - try tokenStr.append('\r'); - state.* = TokenizerState.Quote; - }, - '\\' => - { - try tokenStr.append('\\'); - state.* = TokenizerState.Quote; - }, - '"' => - { - try tokenStr.append('\"'); - state.* = TokenizerState.Quote; - }, - '\'' => - { - try tokenStr.append('\''); - state.* = TokenizerState.Quote; - }, - 'z' => - { - state.* = TokenizerState.QuoteBackslashZ; - }, - else => return error.UnknownEscapeSequence, - } - }, - TokenizerState.QuoteBackslashZ => - { - switch(ch) - { - '\\' => state.* = TokenizerState.QuoteBackslash, - '"' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region), - else => - { - if(!std.ascii.isWhitespace(ch)) - { - try tokenStr.append(ch); - state.* = TokenizerState.Quote; - } - else - { - // Noop, https://www.lua.org/manual/5.4/manual.html#3.1: - // "The escape sequence '\z' skips the following span of whitespace characters, including line breaks;" - } - } - } - }, - TokenizerState.SingleQuote => - { - switch(ch) - { - '\\' => state.* = TokenizerState.SingleQuoteBackslash, - '\'' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region), - else => try tokenStr.append(ch), - } - }, - TokenizerState.SingleQuoteBackslash => - { - switch(ch) - { - 'a' => - { - try tokenStr.append('\u{0007}'); - state.* = TokenizerState.SingleQuote; - }, - 'b' => - { - try tokenStr.append('\u{0008}'); - state.* = TokenizerState.SingleQuote; - }, - 't' => - { - try tokenStr.append('\t'); - state.* = TokenizerState.SingleQuote; - }, - 'n' | '\n' => - { - try tokenStr.append('\n'); - state.* = TokenizerState.SingleQuote; - }, - 'v' => - { - try tokenStr.append('\u{000b}'); - state.* = TokenizerState.SingleQuote; - }, - 'f' => - { - try tokenStr.append('\u{000c}'); - state.* = TokenizerState.SingleQuote; - }, - 'r' => - { - try tokenStr.append('\r'); - state.* = TokenizerState.SingleQuote; - }, - '\\' => - { - try tokenStr.append('\\'); - state.* = TokenizerState.SingleQuote; - }, - '"' => - { - try tokenStr.append('\"'); - state.* = TokenizerState.SingleQuote; - }, - '\'' => - { - try tokenStr.append('\''); - state.* = TokenizerState.SingleQuote; - }, - 'z' => - { - state.* = TokenizerState.SingleQuoteBackslashZ; - }, - else => return error.UnknownEscapeSequence, - } - }, - TokenizerState.SingleQuoteBackslashZ => - { - switch(ch) - { - '\\' => state.* = TokenizerState.SingleQuoteBackslash, - '\'' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region), - else => - { - if(!std.ascii.isWhitespace(ch)) - { - try tokenStr.append(ch); - state.* = TokenizerState.SingleQuote; - } - else - { - // Noop, https://www.lua.org/manual/5.4/manual.html#3.1: - // "The escape sequence '\z' skips the following span of whitespace characters, including line breaks;" - } - } - } - }, - TokenizerState.String => try tokenizeBacktrackCustomToken(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, TokenType.StringLiteral, allocator, region), - TokenizerState.Name => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - TokenizerState.Zero => - { - switch(ch) - { - 'x' => - { - try tokenStr.*.append(ch); - tokenType.* = null; - state.* = TokenizerState.HexNumberX; - }, - '.' => return error.NotImplemented, - else => - { - if(std.ascii.isDigit(ch)) - { - const digitValue = @as(i64, ch - '0'); - lastIndex.* = index.*; - if(tokenNumeral.* == null) - { - tokenNumeral.* = types.Numeral { .Integer = digitValue }; - tokenType.* = TokenType.Numeral; - } - else - { - tokenNumeral.*.?.Integer = tokenNumeral.*.?.Integer * 10 + digitValue; - } - } - else - { - try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region); - } - } - } - }, - TokenizerState.HexNumberX => - { - if(std.ascii.isHex(ch)) - { - lastIndex.* = index.*; - tokenType.* = TokenType.Numeral; - if(std.ascii.isDigit(ch)) - { - tokenNumeral.* = types.Numeral { .Integer = @as(i64, ch - '0') }; - } - else - { - tokenNumeral.* = types.Numeral { .Integer = @as(i64, std.ascii.toLower(ch) - 'a') }; - } - } - else - { - try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region); - } - }, - TokenizerState.HexNumber => - { - switch(ch) - { - 'p' => - { - try tokenStr.*.append(ch); - tokenType.* = null; - state.* = TokenizerState.HexExpNumber; - }, - else => - { - if(std.ascii.isHex(ch)) - { - lastIndex.* = index.*; - tokenType.* = TokenType.Numeral; - if(std.ascii.isDigit(ch)) - { - tokenNumeral.* = types.Numeral { .Integer = @as(i64, ch - '0') }; - } - else - { - tokenNumeral.* = types.Numeral { .Integer = @as(i64, std.ascii.toLower(ch) - 'a') }; - } - } - else - { - try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region); - } - } - } - }, - TokenizerState.Number => - { - switch(ch) - { - 'e' => - { - try tokenStr.*.append(ch); - tokenType.* = null; - state.* = TokenizerState.ExpNumber; - }, - '.' => return error.NotImplemented, - else => - { - if(std.ascii.isDigit(ch)) - { - const digitValue = @as(i64, ch - '0'); - lastIndex.* = index.*; - if(tokenNumeral.* == null) - { - tokenNumeral.* = types.Numeral { .Integer = digitValue }; - tokenType.* = TokenType.Numeral; - } - else - { - tokenNumeral.*.?.Integer = tokenNumeral.*.?.Integer * 10 + digitValue; - } - } - else - { - try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region); - } - - } - } - }, - TokenizerState.Comma, TokenizerState.RoundOpen, TokenizerState.RoundClosed, - TokenizerState.CurlyOpen, TokenizerState.CurlyClosed, TokenizerState.Plus, - TokenizerState.TildeEquals, TokenizerState.EqualsEquals, TokenizerState.Hash, - TokenizerState.GtEquals, TokenizerState.LtEquals, TokenizerState.SquareOpen, - TokenizerState.SquareClosed, TokenizerState.Pipe, TokenizerState.Ampersand, - TokenizerState.Percent, TokenizerState.Star, TokenizerState.Semicolon, - TokenizerState.Caret, TokenizerState.DotDotDot, TokenizerState.GtGt, - TokenizerState.LtLt, TokenizerState.SlashSlash => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), - TokenizerState.Tilde => - { - switch(ch) - { - '=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.TildeEquals, TokenizerState.TildeEquals, region), - else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), - } - }, - TokenizerState.Gt => - { - switch (ch) - { - '>' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.GtGt, TokenizerState.GtGt, region), - '=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.GtEquals, TokenizerState.GtEquals, region), - else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), - } - }, - TokenizerState.Lt => - { - switch(ch) - { - '<' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.LtLt, TokenizerState.LtLt, region), - '=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.LtEquals, TokenizerState.LtEquals, region), - else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), - } - }, - TokenizerState.Slash => - { - switch(ch) - { - '/' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.SlashSlash, TokenizerState.SlashSlash, region), - else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), - } - }, - TokenizerState.Dot => - { - switch(ch) - { - '.' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.DotDot, TokenizerState.DotDot, region), - else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), - } - }, - TokenizerState.DotDot => - { - switch(ch) - { - '.' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.DotDotDot, TokenizerState.DotDotDot, region), - else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), - } - }, - TokenizerState.Colon => - { - switch(ch) - { - ':' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.ColonColon, TokenizerState.ColonColon, region), - else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), - } - }, - TokenizerState.Equals => - { - switch(ch) - { - '=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.EqualsEquals, TokenizerState.EqualsEquals, region), - else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), - } - }, - TokenizerState.Minus => - { - switch(ch) - { - '-' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, null, TokenizerState.SmallCommentStart, region), - else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), - } - }, - TokenizerState.SmallCommentStart => - { - switch(ch) - { - '[' => - { - tokenType.* = null; - state.* = TokenizerState.BigCommentLongBracketStart; - }, - '\n' => - { - state.* = TokenizerState.Start; - lastIndex.* = null; - }, - else => - { - state.* = TokenizerState.SmallComment; - }, - } - }, - TokenizerState.SmallComment => - { - switch(ch) - { - '\n' => - { - state.* = TokenizerState.Start; - lastIndex.* = null; - }, - else => { } - } - }, - TokenizerState.BigCommentLongBracketStart => - { - switch(ch) - { - '=' => - { - longBracketLevel.* += 1; - }, - '[' => - { - state.* = TokenizerState.BigComment; - }, - else => return error.LongBracketMalformedStartBigComment, - } - }, - TokenizerState.BigComment => - { - switch(ch) - { - ']' => - { - state.* = TokenizerState.BigCommentLongBracketEnd; - }, - else => { }, - } - }, - TokenizerState.BigCommentLongBracketEnd => - { - switch(ch) - { - '=' => - { - if(longBracketLevel.* == 0) - { - return error.LongBracketLevelTooBigEndBigComment; - } - longBracketLevel.* -= 1; - }, - ']' => - { - if(longBracketLevel.* != 0) - { - return error.LongBracketLevelTooSmallEndBigComment; - } - state.* = TokenizerState.Start; - }, - else => return error.LongBracketMalformedSmallEndBigComment, - } - }, - TokenizerState.A => - { - switch(ch) - { - 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.An, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.An => - { - switch(ch) - { - 'd' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.And, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.And => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.And, allocator, region), - TokenizerState.W => - { - switch(ch) - { - 'h' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Wh, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Wh => - { - switch(ch) - { - 'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Whi, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Whi => - { - switch(ch) - { - 'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Whil, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Whil => - { - switch(ch) - { - 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.While, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.While => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.While, allocator, region), - TokenizerState.B => - { - switch(ch) - { - 'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Br, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Br => - { - switch(ch) - { - 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Bre, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Bre => - { - switch(ch) - { - 'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Brea, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Brea => - { - switch(ch) - { - 'k' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Break, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Break => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Break, allocator, region), - TokenizerState.G => - { - switch(ch) - { - 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Go, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Go => - { - switch(ch) - { - 't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Got, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Got => - { - switch(ch) - { - 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Goto, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Goto => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Goto, allocator, region), - TokenizerState.R => - { - switch(ch) - { - 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Re, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Re => - { - switch(ch) - { - 't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Ret, tokenStr, ch, region), - 'p' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Rep, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Ret => - { - switch(ch) - { - 'u' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Retu, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Retu => - { - switch(ch) - { - 'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Retur, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Retur => - { - switch(ch) - { - 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Return, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Return => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Return, allocator, region), - TokenizerState.Rep => - { - switch(ch) - { - 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Repe, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Repe => - { - switch(ch) - { - 'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Repea, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Repea => - { - switch(ch) - { - 't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Repeat, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Repeat => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Repeat, allocator, region), - TokenizerState.N => - { - switch(ch) - { - 'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Ni, tokenStr, ch, region), - 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.No, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.No => - { - switch(ch) - { - 't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Not, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Not => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Not, allocator, region), - TokenizerState.Ni => - { - switch(ch) - { - 'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Nil, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Nil => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Nil, allocator, region), - TokenizerState.T => - { - switch(ch) - { - 'h' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Th, tokenStr, ch, region), - 'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Tr, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Th => - { - switch(ch) - { - 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.The, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.The => - { - switch(ch) - { - 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Then, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Then => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Then, allocator, region), - TokenizerState.Tr => - { - switch(ch) - { - 'u' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Tru, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Tru => - { - switch(ch) - { - 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.True, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.True => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.True, allocator, region), - TokenizerState.E => - { - switch(ch) - { - 'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.El, tokenStr, ch, region), - 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.En, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.En => - { - switch(ch) - { - 'd' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.End, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.End => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.End, allocator, region), - TokenizerState.El => - { - switch(ch) - { - 's' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Els, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Els => - { - switch(ch) - { - 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Else, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Else => - { - switch(ch) - { - 'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Elsei, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Else, allocator, region), - } - }, - TokenizerState.Elsei => - { - switch(ch) - { - 'f' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Elseif, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Elseif => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Elseif, allocator, region), - TokenizerState.O => - { - switch(ch) - { - 'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Or, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Or => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Or, allocator, region), - TokenizerState.D => - { - switch(ch) - { - 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Do, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Do => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Do, allocator, region), - TokenizerState.I => - { - switch(ch) - { - 'f' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.If, tokenStr, ch, region), - 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.In, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.In => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.In, allocator, region), - TokenizerState.If => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.If, allocator, region), - TokenizerState.F => - { - switch(ch) - { - 'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fa, tokenStr, ch, region), - 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fo, tokenStr, ch, region), - 'u' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fu, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Fu => - { - switch(ch) - { - 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fun, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Fun => - { - switch(ch) - { - 'c' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Func, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Func => - { - switch(ch) - { - 't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Funct, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Funct => - { - switch(ch) - { - 'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Functi, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Functi => - { - switch(ch) - { - 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Functio, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Functio => - { - switch(ch) - { - 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Function, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Function => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Function, allocator, region), - TokenizerState.Fa => - { - switch(ch) - { - 'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fal, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Fal => - { - switch(ch) - { - 's' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fals, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Fals => - { - switch(ch) - { - 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.False, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.False => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.False, allocator, region), - TokenizerState.Fo => - { - switch(ch) - { - 'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.For, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.For => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.For, allocator, region), - TokenizerState.L => - { - switch(ch) - { - 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Lo, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Lo => - { - switch(ch) - { - 'c' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Loc, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Loc => - { - switch(ch) - { - 'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Loca, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Loca => - { - switch(ch) - { - 'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Local, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Local => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Local, allocator, region), - TokenizerState.U => - { - switch(ch) - { - 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Un, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Un => - { - switch(ch) - { - 't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Unt, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Unt => - { - switch(ch) - { - 'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Unti, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Unti => - { - switch(ch) - { - 'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Until, tokenStr, ch, region), - else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), - } - }, - TokenizerState.Until => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Until, allocator, region), - else => - { - std.debug.print("{}\n", . {state.*}); - return error.NotImplemented; - } - } -} - -pub fn tokenize(fileContent: []u8, allocator: std.mem.Allocator) ![]Token -{ - var tokens = std.ArrayList(Token).init(allocator); - var state: TokenizerState = TokenizerState.Start; - var lastIndex: ?usize = null; - var index: usize = 0; - var tokenType: ?TokenType = null; - var tokenStr = std.ArrayList(u8).init(allocator); - defer tokenStr.deinit(); - var tokenNumeral: ?types.Numeral = null; - var longBracketLevel: u32 = 0; - var region = CodeRegion { .start = null, .length = 0 }; - - while(index < fileContent.len) - { - const ch = fileContent[index]; - try tokenizeChar(&state, ch, &lastIndex, &index, &tokenType, &tokenStr, &tokenNumeral, &tokens, &longBracketLevel, ®ion, allocator); - if(region.start != null and region.start.?.col == 0 and region.start.?.line == 0) - { - region.start = calculatePoint(fileContent, index); - } - index += 1; - } - if(longBracketLevel != 0) - { - return error.UnbalancedLongBracketLevel; - } - try tokenizeChar(&state, '\n', &lastIndex, &index, &tokenType, &tokenStr, &tokenNumeral, &tokens, &longBracketLevel, ®ion, allocator); - if(region.start != null and region.start.?.col == 0 and region.start.?.line == 0) - { - region.start = calculatePoint(fileContent, index); - } - return tokens.toOwnedSlice(); -} - -fn calculatePoint(fileContent: []u8, index: usize) CodeLocation -{ - var ret = CodeLocation { .col = 1, .line = 1 }; - for(0..index) |i| - { - ret.col += 1; - if(fileContent[i] == '\n') - { - ret.line += 1; - ret.col = 1; - } - } - return ret; -} diff --git a/src/treewalker.zig b/src/treewalker.zig deleted file mode 100644 index bc3aa7f..0000000 --- a/src/treewalker.zig +++ /dev/null @@ -1,142 +0,0 @@ -const std = @import("std"); -const parser = @import("parser.zig"); -const types = @import("types.zig"); - -pub fn interpret(root: parser.ChunkNode, allocator: std.mem.Allocator) !void -{ - var _ENV = types.Table { .entries= std.ArrayList(types.TableEntry).init(allocator) }; - try walkChunk(root, &_ENV, allocator); -} - -fn walkChunk(node: parser.ChunkNode, environment: *types.Table, allocator: std.mem.Allocator) !void -{ - try walkBlock(node.block, environment, allocator); -} - -fn walkBlock(node: parser.BlockNode, environment: *types.Table, allocator: std.mem.Allocator) !void -{ - for(node.stats.items) |stat| - { - try walkStat(stat, environment, allocator); - } - if(node.retstat != null) - { - try walkRetstat(node.retstat.?, environment, allocator); - } -} - -fn walkStat(node: parser.StatNode, environment: *types.Table, allocator: std.mem.Allocator) !void -{ - switch(node) - { - .Assignment => |assignmentNode| - { - return try walkAssignmentNode(assignmentNode, environment, allocator); - }, - else => - { - std.debug.print("{any}\n", .{node}); - return error.NotImplemented; - } - } -} - -fn walkRetstat(node: parser.RetstatNode, environment: *types.Table, allocator: std.mem.Allocator) !void -{ - _ = node; - _ = environment; - _ = allocator; - return error.NotImplemented; -} - -fn walkAssignmentNode(node: parser.AssignmentNode, environment: *types.Table, allocator: std.mem.Allocator) !void -{ - const results = try walkExplist(node.rhs, environment, allocator); - var i: usize = 0; - _ = results; - _ = i; - for(node.lhs.vars.items) |variable| - { - switch(variable) - { - .Indexed => |indexedNode| - { - _ = indexedNode; - return error.NotImplemented; - }, - else => return error.NotImplemented, - } - } -} - -fn walkExplist(node: parser.ExplistNode, environment: *types.Table, allocator: std.mem.Allocator) ![]types.Value -{ - var results = std.ArrayList(types.Value).init(allocator); - for(node.exps.items) |exp| - { - try results.append(try walkExp(exp, environment, allocator, false)); - } - return results.toOwnedSlice(); -} - -fn walkExp(node: parser.ExpNode, environment: *types.Table, allocator: std.mem.Allocator, isVariadicFunction: bool) !types.Value -{ - switch(node) - { - .Nil => return types.Value.Nil, - .False => return types.Value { .Bool = false }, - .True => return types.Value { .Bool = true }, - .Numeral => |numeral| return types.Value { .Numeral = numeral }, - .LiteralString => |string| return types.Value { .String = string }, - .Varargs => - { - if(isVariadicFunction) - { - return error.NotImplemented; - } - else - { - return error.UseVarargsOutsideVariadicFunction; - } - }, - .Suffixexp => |suffixExp| - { - return walkSuffixexp(suffixExp.*, environment, allocator); - }, - else => - { - std.debug.print("{}\n", .{node}); - return error.NotImplemented; - } - } - return error.NotImplemented; -} - -fn walkSuffixexp(node: parser.SuffixexpNode, environment: *types.Table, allocator: std.mem.Allocator) !types.Value -{ - _ = allocator; - switch(node) - { - .Normal => |normal| - { - switch(normal.firstPart) - { - .Name => |name| - { - std.debug.print("name: {s}\n", .{name}); - std.debug.print("val: {!}\n", .{environment.get(types.Value { .String = name })}); - }, - else => - { - std.debug.print("{}\n", .{normal.firstPart}); - } - } - return error.NotImplemented; - }, - else => - { - std.debug.print("{}\n", .{node}); - return error.NotImplemented; - } - } -} diff --git a/src/types.zig b/src/types.zig deleted file mode 100644 index 144b925..0000000 --- a/src/types.zig +++ /dev/null @@ -1,194 +0,0 @@ -const std = @import("std"); - -pub const NumeralTag = enum -{ - Integer, - Float, -}; - -pub fn isFloatExactInt(value: f64) bool -{ - return value == 0 or (std.math.isNormal(value) and @floor(value) == value); -} - -pub const Numeral = union(NumeralTag) -{ - Integer: i64, - Float: f64, - - pub fn rawEqual(self: Numeral, other: Numeral) bool - { - if(@as(NumeralTag, self) == @as(NumeralTag, other)) - { - switch(self) - { - .Float => |value| return value == other.Float, - .Integer => |value| return value == other.Integer, - } - } - // Other is the respective other type - switch(self) - { - .Float => |value| - { - return isFloatExactInt(value) and @as(u64, @intFromFloat(value)) == other.Integer; - }, - .Integer => |value| - { - return isFloatExactInt(other.Float) and @as(u64, @intFromFloat(other.Float)) == value; - } - } - } -}; - -test "float int equality" -{ - const a = Numeral { .Float = 12.0 }; - const b = Numeral { .Integer = 12 }; - try std.testing.expect(a.rawEqual(b)); - try std.testing.expect(b.rawEqual(a)); - try std.testing.expect(a.rawEqual(a)); - try std.testing.expect(b.rawEqual(b)); - const c = Numeral { .Float = (0.2 + 0.1) * 10.0 }; - const d = Numeral { .Integer = 3 }; - try std.testing.expect(c.rawEqual(d)); - try std.testing.expect(d.rawEqual(c)); - try std.testing.expect(c.rawEqual(c)); - try std.testing.expect(d.rawEqual(d)); - const e = Numeral { .Float = 3.2 }; - try std.testing.expect(!a.rawEqual(e)); - try std.testing.expect(!b.rawEqual(e)); - try std.testing.expect(!c.rawEqual(e)); - try std.testing.expect(!d.rawEqual(e)); - try std.testing.expect(!e.rawEqual(a)); - try std.testing.expect(!e.rawEqual(b)); - try std.testing.expect(!e.rawEqual(c)); - try std.testing.expect(!e.rawEqual(d)); -} - -pub const TableEntry = struct -{ - key: Value, - value: Value, -}; - -pub const Table = struct -{ - entries: std.ArrayList(TableEntry), - - pub fn get(self: Table, key: Value) Value - { - if(@as(ValueTag, key) == ValueTag.Nil) - { - return Value.Nil; - } - for (self.entries.items) |entry| - { - if(entry.key.rawEqual(key)) - { - return entry.value; - } - } - return Value.Nil; - } -}; - -pub const ValueTag = enum -{ - Nil, - Bool, - Numeral, - String, - Table, -}; -pub const Value = union(ValueTag) -{ - Nil, - Bool: bool, - Numeral: Numeral, - String: []const u8, - Table: *Table, - - pub fn rawEqual(self: Value, other: Value) bool - { - if(@as(ValueTag, self) != @as(ValueTag, other)) - { - return false; - } - switch(self) - { - .Nil => return true, - .Bool => |value| return value == other.Bool, - .Numeral => |value| return value.rawEqual(other.Numeral), - .String => |value| return std.mem.eql(u8, value, other.String), - .Table => |value| return value == other.Table, - } - } -}; - -test "Value equalities" -{ - const a = Value { .Bool = true }; - const b = Value { .Numeral = Numeral { .Integer = 1 } }; - // true != 1 - try std.testing.expect(!a.rawEqual(b)); - // 1 != true - try std.testing.expect(!b.rawEqual(a)); - const c = Value { .Bool = false }; - // true != false - try std.testing.expect(!a.rawEqual(c)); - // false!= true - try std.testing.expect(!c.rawEqual(a)); - const d = Value { .Bool = true }; - // true == true - try std.testing.expect(a.rawEqual(d)); - // true == true - try std.testing.expect(d.rawEqual(a)); - const e = Value { .String = "foo" }; - const f = Value { .String = "bar" }; - // foo != bar - try std.testing.expect(!e.rawEqual(f)); - // bar != foo - try std.testing.expect(!f.rawEqual(e)); - const g = Value { .String = "foo" }; - // foo != foo - try std.testing.expect(e.rawEqual(g)); - // foo != foo - try std.testing.expect(g.rawEqual(e)); - var table = Table { .entries = std.ArrayList(TableEntry).init(std.testing.allocator) }; - const h = Value { .Table = &table }; - var table2 = Table { .entries = std.ArrayList(TableEntry).init(std.testing.allocator) }; - const i = Value { .Table = &table2 }; - const j = Value { .Table = &table2 }; - - try std.testing.expect(h.rawEqual(h)); - try std.testing.expect(i.rawEqual(i)); - try std.testing.expect(!h.rawEqual(i)); - try std.testing.expect(!i.rawEqual(h)); - try std.testing.expect(i.rawEqual(j)); - try std.testing.expect(!h.rawEqual(j)); -} - -test "Table get" -{ - var a = Table { .entries = std.ArrayList(TableEntry).init(std.testing.allocator) }; - defer a.entries.deinit(); - try a.entries.append(TableEntry { .key = Value { .Bool = true }, .value = Value { .String = "foo" } }); - try std.testing.expectEqualStrings(a.get(Value { .Bool = true }).String, "foo"); - try std.testing.expectEqual(a.get(Value.Nil), Value.Nil); - try std.testing.expectEqual(a.get(Value { .Numeral = Numeral { .Integer = 12 } }), Value.Nil); - var c = a.get(Value { .Bool = true }); - c.String = "bar"; - try std.testing.expectEqual(a.get(Value { .Bool = true }).String, "foo"); -} - -pub const CodeRegion = struct -{ - start: ?CodeLocation, - length: usize, -}; -pub const CodeLocation = struct -{ - line: usize, - col: usize, -}; diff --git a/test/simpleInt.lua b/test/simpleInt.lua new file mode 100644 index 0000000..e052cd9 --- /dev/null +++ b/test/simpleInt.lua @@ -0,0 +1,22 @@ +0xff11 123 , () ~ ~= > >> >= < << <= / // . .. ... : :: = == - -- +-- asdf +--[[askf]]zzz +--[==========[asdfasdf]==========] zzzz +[[fsad]] +[= +] +[===[test]]=]==]===] zzzzz +a ab an anb ando and +w wo wh who whi whio whil whilo while whileo +b bo br bro bre breo brea breao break breako +g gz go goz got gotz goto gotoz +r ro re reo ret rep reto repo retu repe retuo repeo retur repea returo repeao return repeat returno repeato +n nz ni no niz noz nil not nilz notz +t to th tr tho the tro tru then true theno trueo +e eo el en elo eno els end elso endo else elso elsei elseif elseio elseifo +o oo or oro +d dz do doz +i io if in ifo ino +f fz fo fa fu foz faz fuz for fal fun forz falz funz fals func falsz funcz false funct falsez functz functi functiz functio functioz function functionz +l lz lo loz loc locz loca locaz local localz +u uo un uno unt unto unti untio until untilo \ No newline at end of file diff --git a/test/simpleString.lua b/test/simpleString.lua new file mode 100644 index 0000000..6e9fc37 --- /dev/null +++ b/test/simpleString.lua @@ -0,0 +1,10 @@ +"test" "\z + + + + + + + + +abc" "123" "sdlfkgj<3" "asldkfj" zzz "" "" "" "" "" "fasd!" "afd" "" "as" zzzz \ No newline at end of file