0x4261756D/luaaaaah
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: 0x4261756D:csharp
Branches Tags
0x4261756D:zig
0x4261756D:csharp
0x4261756D:rust
..
compare: 0x4261756D:zig
Branches Tags
0x4261756D:csharp
0x4261756D:zig
0x4261756D:rust

No commits in common. "csharp" and "zig" have entirely different histories.
csharp ... zig

20 changed files with 4281 additions and 4870 deletions
Show Stats Expand all files Collapse all files

101
.editorconfig
View File

@ -1,101 +0,0 @@
root = true
[*]
indent_size = 4
indent_style = tab
trim_trailing_whitespace = true
end_of_line = lf
insert_final_newline = true
charset = utf-8
[*.cs]
csharp_space_after_keywords_in_control_flow_statements = false
csharp_indent_case_contents_when_block = false
csharp_style_unused_value_expression_preference = unused_local_variable
csharp_prefer_braces = true
csharp_prefer_static_local_function = true
dotnet_style_prefer_foreach_explicit_cast_in_source = always
dotnet_style_prefer_collection_expression = when_types_loosely_match
dotnet_diagnostic.IDE0001.severity = warning
dotnet_diagnostic.IDE0002.severity = warning
dotnet_diagnostic.IDE0004.severity = warning
dotnet_diagnostic.IDE0005.severity = warning
dotnet_diagnostic.IDE0011.severity = warning
dotnet_diagnostic.IDE0020.severity = warning
dotnet_diagnostic.IDE0028.severity = warning
dotnet_diagnostic.IDE0029.severity = warning
dotnet_diagnostic.IDE0030.severity = warning
dotnet_diagnostic.IDE0031.severity = warning
dotnet_diagnostic.IDE0035.severity = error
dotnet_diagnostic.IDE0038.severity = error
dotnet_diagnostic.IDE0041.severity = warning
dotnet_diagnostic.IDE0042.severity = warning
dotnet_diagnostic.IDE0051.severity = warning
dotnet_diagnostic.IDE0052.severity = warning
dotnet_diagnostic.IDE0054.severity = warning
dotnet_diagnostic.IDE0050.severity = warning
dotnet_diagnostic.IDE0056.severity = warning
dotnet_diagnostic.IDE0057.severity = warning
dotnet_diagnostic.IDE0058.severity = warning
dotnet_diagnostic.IDE0060.severity = warning
dotnet_diagnostic.IDE0062.severity = warning
dotnet_diagnostic.IDE0066.severity = warning
dotnet_diagnostic.IDE0071.severity = warning
dotnet_diagnostic.IDE0074.severity = warning
dotnet_diagnostic.IDE0075.severity = warning
dotnet_diagnostic.IDE0078.severity = warning
dotnet_diagnostic.IDE0080.severity = warning
dotnet_diagnostic.IDE0083.severity = warning
dotnet_diagnostic.IDE0090.severity = warning
dotnet_diagnostic.IDE0100.severity = warning
dotnet_diagnostic.IDE0150.severity = warning
dotnet_diagnostic.IDE0180.severity = warning
dotnet_diagnostic.IDE0200.severity = warning
dotnet_diagnostic.IDE0220.severity = warning
dotnet_diagnostic.IDE0260.severity = warning
dotnet_diagnostic.IDE0270.severity = warning
dotnet_diagnostic.IDE0300.severity = warning
dotnet_diagnostic.IDE0301.severity = warning
dotnet_diagnostic.IDE0302.severity = warning
dotnet_diagnostic.IDE0303.severity = warning
dotnet_diagnostic.IDE0304.severity = warning
dotnet_diagnostic.IDE0305.severity = warning
dotnet_diagnostic.CA1508.severity = warning
dotnet_diagnostic.CA1514.severity = warning
dotnet_diagnostic.CA1515.severity = warning
dotnet_diagnostic.CA1801.severity = warning
dotnet_diagnostic.CA1802.severity = warning
dotnet_diagnostic.CA1805.severity = warning
dotnet_diagnostic.CA1806.severity = warning
dotnet_diagnostic.CA1810.severity = warning
dotnet_diagnostic.CA1814.severity = warning
dotnet_diagnostic.CA1820.severity = warning
dotnet_diagnostic.CA1822.severity = warning
dotnet_diagnostic.CA1823.severity = warning
dotnet_diagnostic.CA1825.severity = warning
dotnet_diagnostic.CA1826.severity = error
dotnet_diagnostic.CA1827.severity = error
dotnet_diagnostic.CA1829.severity = error
dotnet_diagnostic.CA1830.severity = error
dotnet_diagnostic.CA1833.severity = warning
dotnet_diagnostic.CA1834.severity = warning
dotnet_diagnostic.CA1835.severity = warning
dotnet_diagnostic.CA1836.severity = warning
dotnet_diagnostic.CA1841.severity = error
dotnet_diagnostic.CA1845.severity = warning
dotnet_diagnostic.CA1846.severity = warning
dotnet_diagnostic.CA1847.severity = warning
dotnet_diagnostic.CA1849.severity = warning
dotnet_diagnostic.CA1850.severity = warning
dotnet_diagnostic.CA1851.severity = warning
dotnet_diagnostic.CA1853.severity = warning
dotnet_diagnostic.CA1859.severity = warning
dotnet_diagnostic.CA1860.severity = warning
dotnet_diagnostic.CA1861.severity = warning
dotnet_diagnostic.CA1864.severity = warning
dotnet_diagnostic.CA1865.severity = warning
dotnet_diagnostic.CA1866.severity = warning
dotnet_diagnostic.CA1867.severity = warning
dotnet_diagnostic.CA2007.severity = warning
dotnet_diagnostic.CA2011.severity = error
dotnet_diagnostic.CA2248.severity = error

6
.gitignore vendored
View File

@ -1,2 +1,4 @@
bin/ zig-out/
obj/ zig-cache/
.vscode/
target/

26
.vscode/launch.json vendored
View File

@ -1,26 +0,0 @@
{
"version": "0.2.0",
"configurations": [
{
// Use IntelliSense to find out which attributes exist for C# debugging
// Use hover for the description of the existing attributes
// For further information visit https://github.com/dotnet/vscode-csharp/blob/main/debugger-launchjson.md
"name": ".NET Core Launch (console)",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "build",
// If you have changed target frameworks, make sure to update the program path.
"program": "${workspaceFolder}/bin/Debug/net8.0/luaaaaah.dll",
"args": ["run", "test/stringDataNotSet.lua"],
"cwd": "${workspaceFolder}",
// For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
"console": "internalConsole",
"stopAtEntry": false
},
{
"name": ".NET Core Attach",
"type": "coreclr",
"request": "attach"
}
]
}

41
.vscode/tasks.json vendored
View File

@ -1,41 +0,0 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "build",
"command": "dotnet",
"type": "process",
"args": [
"build",
"${workspaceFolder}/luaaaaah.csproj",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary;ForceNoAlign"
],
"problemMatcher": "$msCompile"
},
{
"label": "publish",
"command": "dotnet",
"type": "process",
"args": [
"publish",
"${workspaceFolder}/luaaaaah.csproj",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary;ForceNoAlign"
],
"problemMatcher": "$msCompile"
},
{
"label": "watch",
"command": "dotnet",
"type": "process",
"args": [
"watch",
"run",
"--project",
"${workspaceFolder}/luaaaaah.csproj"
],
"problemMatcher": "$msCompile"
}
]
}

71
LuaTypes.cs
View File

@ -1,71 +0,0 @@
using System.Text.Json.Serialization;
namespace luaaaaah;
[JsonDerivedType(typeof(Integer), typeDiscriminator: "int")]
[JsonDerivedType(typeof(Float), typeDiscriminator: "float")]
public interface INumeral
{
public class Integer(int value) : INumeral
{
public int value = value;
public bool RawEqual(INumeral other)
{
if(other is Integer integer)
{
return integer.value == value;
}
// TODO: Check if this is actually doing what is expected
return ((Float)other).value == value;
}
public override string ToString()
{
return $"Numeral Integer {value}";
}
}
public class Float(float value) : INumeral
{
public float value = value;
public bool RawEqual(INumeral other)
{
if(other is Float float_val)
{
return float_val.value == value;
}
// TODO: Check if this is actually doing what is expected
return ((Integer)other).value == value;
}
public override string ToString()
{
return $"Numeral Float {value}";
}
}
public bool RawEqual(INumeral other);
}
class CodeRegion(CodeLocation start, CodeLocation end)
{
public CodeLocation start = start;
public CodeLocation end = end;
public override string ToString()
{
return $"{start}-{end}";
}
}
class CodeLocation(int line, int col)
{
public int line = line;
public int col = col;
public CodeLocation(CodeLocation other) : this(line: other.line, col: other.col) { }
public override string ToString()
{
return $"{line + 1}:{col + 1}";
}
}

1639
Parser.cs
View File

@ -1,1639 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace luaaaaah;
internal class Parser
{
public class ChunkNode(BlockNode block, CodeRegion startRegion, CodeRegion endRegion)
{
public BlockNode block = block;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class BlockNode(List<StatNode> stats, CodeRegion startRegion, CodeRegion endRegion)
{
public List<StatNode> stats = stats;
public RetstatNode? retstat;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
[JsonDerivedType(typeof(Semicolon), typeDiscriminator: "st Semicolon")]
[JsonDerivedType(typeof(Assignment), typeDiscriminator: "st Assignment")]
[JsonDerivedType(typeof(Functioncall), typeDiscriminator: "st Functioncall")]
[JsonDerivedType(typeof(Label), typeDiscriminator: "st Label")]
[JsonDerivedType(typeof(Break), typeDiscriminator: "st Break")]
[JsonDerivedType(typeof(Goto), typeDiscriminator: "st Goto")]
[JsonDerivedType(typeof(Do), typeDiscriminator: "st Do")]
[JsonDerivedType(typeof(While), typeDiscriminator: "st While")]
[JsonDerivedType(typeof(Repeat), typeDiscriminator: "st Repeat")]
[JsonDerivedType(typeof(If), typeDiscriminator: "st If")]
[JsonDerivedType(typeof(ForNumerical), typeDiscriminator: "st ForNum")]
[JsonDerivedType(typeof(ForGeneric), typeDiscriminator: "st ForGen")]
[JsonDerivedType(typeof(Function), typeDiscriminator: "st Function")]
[JsonDerivedType(typeof(LocalFunction), typeDiscriminator: "st LocalFunction")]
[JsonDerivedType(typeof(Local), typeDiscriminator: "st Local")]
public abstract class StatNode
{
public class Semicolon(CodeRegion region) : StatNode
{
public CodeRegion region = region;
}
public class Assignment(VarlistNode lhs, ExplistNode rhs, CodeRegion startRegion, CodeRegion endRegion) : StatNode
{
public VarlistNode lhs = lhs;
public ExplistNode rhs = rhs;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class Functioncall(FunctioncallNode node) : StatNode
{
public FunctioncallNode node = node;
}
public class Label(string label, CodeRegion startRegion, CodeRegion endRegion) : StatNode
{
public CodeRegion startRegion = startRegion;
public CodeRegion endRegion = endRegion;
public string label = label;
}
public class Break(CodeRegion region) : StatNode
{
public CodeRegion region = region;
}
public class Goto(string label, CodeRegion startRegion, CodeRegion endRegion) : StatNode
{
public CodeRegion startRegion = startRegion;
public CodeRegion endRegion = endRegion;
public string label = label;
}
public class Do(BlockNode node, CodeRegion startRegion, CodeRegion endRegion) : StatNode
{
public CodeRegion startRegion = startRegion;
public CodeRegion endRegion = endRegion;
public BlockNode node = node;
}
public class While(ExpNode condition, BlockNode body, CodeRegion startRegion, CodeRegion endRegion) : StatNode
{
public ExpNode condition = condition;
public BlockNode body = body;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class Repeat(ExpNode condition, BlockNode body, CodeRegion startRegion, CodeRegion endRegion) : StatNode
{
public ExpNode condition = condition;
public BlockNode body = body;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class If(ExpNode condition, BlockNode body, List<ElseifNode> elseifs, CodeRegion startRegion, CodeRegion endRegion) : StatNode
{
public ExpNode condition = condition;
public BlockNode body = body;
public List<ElseifNode> elseifs = elseifs;
public BlockNode? else_;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class ForNumerical(string variable, ExpNode start, ExpNode end, ExpNode? change, BlockNode body, CodeRegion startRegion, CodeRegion endRegion) : StatNode
{
public string variable = variable;
public ExpNode start = start;
public ExpNode end = end;
public ExpNode? change = change;
public BlockNode body = body;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class ForGeneric(List<string> vars, ExplistNode exps, BlockNode body, CodeRegion startRegion, CodeRegion endRegion) : StatNode
{
public List<string> vars = vars;
public ExplistNode exps = exps;
public BlockNode body = body;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class Function(FunctionNode node) : StatNode
{
public FunctionNode node = node;
}
public class LocalFunction(string name, FuncbodyNode body, CodeRegion startRegion, CodeRegion endRegion) : StatNode
{
public string name = name;
public FuncbodyNode body = body;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class Local(AttnamelistNode attnames, ExplistNode? values, CodeRegion startRegion, CodeRegion endRegion) : StatNode
{
public AttnamelistNode attnames = attnames;
public ExplistNode? values = values;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
}
public class RetstatNode(ExplistNode? values, CodeRegion startRegion, CodeRegion endRegion)
{
public ExplistNode? values = values;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class FunctioncallNode(SuffixexpNode function, string? objectArg, ArgsNode args)
{
public SuffixexpNode function = function;
public string? objectArg = objectArg;
public ArgsNode args = args;
public CodeRegion startRegion = function.startRegion, endRegion = function.endRegion;
}
public class FunctionNode(FuncnameNode name, FuncbodyNode body, CodeRegion startRegion, CodeRegion endRegion)
{
public FuncnameNode name = name;
public FuncbodyNode body = body;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class ExplistNode(List<ExpNode> exps, CodeRegion startRegion, CodeRegion endRegion)
{
public List<ExpNode> exps = exps;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class VarlistNode(List<VarNode> vars, CodeRegion startRegion, CodeRegion endRegion)
{
public List<VarNode> vars = vars;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
[JsonDerivedType(typeof(Normal), typeDiscriminator: "s Normal")]
[JsonDerivedType(typeof(Functioncall), typeDiscriminator: "s Functioncall")]
public abstract class SuffixexpNode(CodeRegion startRegion, CodeRegion endRegion)
{
public CodeRegion startRegion = startRegion, endRegion = endRegion;
public class Normal(SuffixexpFirstPart firstPart, List<SuffixexpSuffix> suffixes, CodeRegion startRegion, CodeRegion endRegion) : SuffixexpNode(startRegion, endRegion)
{
public SuffixexpFirstPart firstPart = firstPart;
public List<SuffixexpSuffix> suffixes = suffixes;
}
public class Functioncall(FunctioncallNode node) : SuffixexpNode(node.startRegion, node.endRegion)
{
public FunctioncallNode node = node;
}
}
[JsonDerivedType(typeof(Bracketed), typeDiscriminator: "a Bracketed")]
[JsonDerivedType(typeof(Tableconstructor), typeDiscriminator: "a Tableconstructor")]
[JsonDerivedType(typeof(Literal), typeDiscriminator: "a Literal")]
public abstract class ArgsNode(CodeRegion startRegion, CodeRegion endRegion)
{
public CodeRegion startRegion = startRegion, endRegion = endRegion;
public class Bracketed(ExplistNode? node, CodeRegion startRegion, CodeRegion endRegion) : ArgsNode(startRegion, endRegion)
{
public ExplistNode? node = node;
}
public class Tableconstructor(TableconstructorNode node, CodeRegion startRegion, CodeRegion endRegion) : ArgsNode(startRegion, endRegion)
{
public TableconstructorNode node = node;
}
public class Literal(string name, CodeRegion startRegion, CodeRegion endRegion) : ArgsNode(startRegion, endRegion)
{
public string name = name;
}
}
[JsonDerivedType(typeof(Nil), typeDiscriminator: "e Nil")]
[JsonDerivedType(typeof(False), typeDiscriminator: "e True")]
[JsonDerivedType(typeof(True), typeDiscriminator: "e False")]
[JsonDerivedType(typeof(Numeral), typeDiscriminator: "e Numeral")]
[JsonDerivedType(typeof(LiteralString), typeDiscriminator: "e Literal")]
[JsonDerivedType(typeof(Varargs), typeDiscriminator: "e Varargs")]
[JsonDerivedType(typeof(Functiondef), typeDiscriminator: "e Functiondef")]
[JsonDerivedType(typeof(Suffixexp), typeDiscriminator: "e Suffixexp")]
[JsonDerivedType(typeof(Tableconstructor), typeDiscriminator: "e Tableconstructor")]
[JsonDerivedType(typeof(Unop), typeDiscriminator: "e Unop")]
[JsonDerivedType(typeof(Binop), typeDiscriminator: "e Binop")]
public abstract class ExpNode
{
public class Nil(CodeRegion region) : ExpNode
{
public CodeRegion region = region;
}
public class False(CodeRegion region) : ExpNode
{
public CodeRegion region = region;
}
public class True(CodeRegion region) : ExpNode
{
public CodeRegion region = region;
}
public class Numeral(INumeral value, CodeRegion region) : ExpNode
{
public CodeRegion region = region;
public INumeral value = value;
}
public class LiteralString(string value, CodeRegion region) : ExpNode
{
public CodeRegion region = region;
public string value = value;
}
public class Varargs(CodeRegion region) : ExpNode
{
public CodeRegion region = region;
}
public class Functiondef(FuncbodyNode node, CodeRegion startRegion, CodeRegion endRegion) : ExpNode
{
public CodeRegion startRegion = startRegion;
public CodeRegion endRegion = endRegion;
public FuncbodyNode node = node;
}
public class Suffixexp(SuffixexpNode node) : ExpNode
{
public SuffixexpNode node = node;
}
public class Tableconstructor(TableconstructorNode node) : ExpNode
{
public TableconstructorNode node = node;
}
public class Unop(UnopNode node) : ExpNode
{
public UnopNode node = node;
}
public class Binop(BinopNode node) : ExpNode
{
public BinopNode node = node;
}
}
public class ElseifNode(ExpNode condition, BlockNode body, CodeRegion startRegion, CodeRegion endRegion)
{
public ExpNode condition = condition;
public BlockNode body = body;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class FuncnameNode(string name, List<string> dottedNames, string? firstArg, CodeRegion startRegion, CodeRegion endRegion)
{
public string name = name;
public List<string> dottedNames = dottedNames;
public string? firstArg = firstArg;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class FuncbodyNode(ParlistNode? pars, BlockNode body, CodeRegion startRegion, CodeRegion endRegion)
{
public ParlistNode? pars = pars;
public BlockNode body = body;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class AttnamelistNode(List<AttnameNode> attnames, CodeRegion startRegion, CodeRegion endRegion)
{
public List<AttnameNode> attnames = attnames;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
[JsonDerivedType(typeof(Name), typeDiscriminator: "v Name")]
[JsonDerivedType(typeof(Indexed), typeDiscriminator: "v Indexed")]
[JsonDerivedType(typeof(Member), typeDiscriminator: "v Member")]
public abstract class VarNode(CodeRegion startRegion, CodeRegion endRegion)
{
public CodeRegion startRegion = startRegion, endRegion = endRegion;
public class Name(string name, CodeRegion startRegion, CodeRegion endRegion) : VarNode(startRegion, endRegion)
{
public string name = name;
}
public class Indexed(IndexedVarNode node, CodeRegion startRegion, CodeRegion endRegion) : VarNode(startRegion, endRegion)
{
public IndexedVarNode node = node;
}
public class Member(MemberVarNode node, CodeRegion startRegion, CodeRegion endRegion) : VarNode(startRegion, endRegion)
{
public MemberVarNode node = node;
}
}
public class TableconstructorNode(FieldlistNode? exps, CodeRegion startRegion, CodeRegion endRegion)
{
public FieldlistNode? exps = exps;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class UnopNode(UnopType type, ExpNode exp, CodeRegion startRegion, CodeRegion endRegion)
{
public UnopType type = type;
public ExpNode exp = exp;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public enum UnopType
{
Minus, LogicalNot, Length, BinaryNot,
}
public class BinopNode(ExpNode lhs, BinopType type, ExpNode rhs, CodeRegion startRegion, CodeRegion endRegion)
{
public ExpNode lhs = lhs;
public BinopType type = type;
public ExpNode rhs = rhs;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public enum BinopType
{
LogicalOr,
LogicalAnd,
Lt, Gt, LtEquals, GtEquals, NotEquals, Equals,
BinaryOr,
BinaryNot,
BinaryAnd,
Shl, Shr,
Concat,
Add, Sub,
Mul, Div, IntDiv, Mod,
Exp,
}
public class ParlistNode(List<string> names, bool hasVarargs, CodeRegion startRegion, CodeRegion endRegion)
{
public List<string> names = names;
public bool hasVarargs = hasVarargs;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class AttnameNode(string name, string? attribute, CodeRegion startRegion, CodeRegion endRegion)
{
public string name = name;
public string? attribute = attribute;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class IndexedVarNode(SuffixexpNode value, ExpNode index, CodeRegion startRegion, CodeRegion endRegion)
{
public SuffixexpNode value = value;
public ExpNode index = index;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class MemberVarNode(SuffixexpNode value, string name, CodeRegion startRegion, CodeRegion endRegion)
{
public SuffixexpNode value = value;
public string name = name;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
[JsonDerivedType(typeof(Name), typeDiscriminator: "sfp Name")]
[JsonDerivedType(typeof(BracketedExp), typeDiscriminator: "sfp BracketedExp")]
public abstract class SuffixexpFirstPart(CodeRegion startRegion, CodeRegion endRegion)
{
public CodeRegion startRegion = startRegion, endRegion = endRegion;
public class Name(string name, CodeRegion startRegion, CodeRegion endRegion) : SuffixexpFirstPart(startRegion, endRegion)
{
public string name = name;
}
public class BracketedExp(ExpNode node, CodeRegion startRegion, CodeRegion endRegion) : SuffixexpFirstPart(startRegion, endRegion)
{
public ExpNode node = node;
}
}
[JsonDerivedType(typeof(Dot))]
[JsonDerivedType(typeof(Indexed))]
[JsonDerivedType(typeof(Args))]
[JsonDerivedType(typeof(ArgsFirstArg))]
public abstract class SuffixexpSuffix(CodeRegion startRegion, CodeRegion endRegion)
{
public CodeRegion startRegion = startRegion, endRegion = endRegion;
public class Dot(string name, CodeRegion startRegion, CodeRegion endRegion) : SuffixexpSuffix(startRegion, endRegion)
{
public string name = name;
}
public class Indexed(ExpNode node, CodeRegion startRegion, CodeRegion endRegion) : SuffixexpSuffix(startRegion, endRegion)
{
public ExpNode node = node;
}
public class Args(ArgsNode node, CodeRegion startRegion, CodeRegion endRegion) : SuffixexpSuffix(startRegion, endRegion)
{
public ArgsNode node = node;
}
public class ArgsFirstArg(ArgsFirstArgNode node) : SuffixexpSuffix(node.startRegion, node.endRegion)
{
public ArgsFirstArgNode node = node;
}
}
public class FieldlistNode(List<FieldNode> exps, CodeRegion startRegion, CodeRegion endRegion)
{
public List<FieldNode> exps = exps;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class ArgsFirstArgNode(string name, ArgsNode rest, CodeRegion startRegion, CodeRegion endRegion)
{
public string name = name;
public ArgsNode rest = rest;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
[JsonDerivedType(typeof(IndexedAssignment))]
[JsonDerivedType(typeof(Assignment))]
[JsonDerivedType(typeof(Exp))]
public abstract class FieldNode(CodeRegion startRegion, CodeRegion endRegion)
{
public CodeRegion startRegion = startRegion, endRegion = endRegion;
public class IndexedAssignment(IndexedAssignmentNode node) : FieldNode(node.startRegion, node.endRegion)
{
public IndexedAssignmentNode node = node;
}
public class Assignment(FieldAssignmentNode node) : FieldNode(node.startRegion, node.endRegion)
{
public FieldAssignmentNode node = node;
}
public class Exp(ExpNode node, CodeRegion startRegion, CodeRegion endRegion) : FieldNode(startRegion, endRegion)
{
public ExpNode node = node;
}
}
public class IndexedAssignmentNode(ExpNode index, ExpNode rhs, CodeRegion startRegion, CodeRegion endRegion)
{
public ExpNode index = index;
public ExpNode rhs = rhs;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public class FieldAssignmentNode(string lhs, ExpNode rhs, CodeRegion startRegion, CodeRegion endRegion)
{
public string lhs = lhs;
public ExpNode rhs = rhs;
public CodeRegion startRegion = startRegion, endRegion = endRegion;
}
public int index;
public ChunkNode Parse(Token[] tokens)
{
return ParseChunk(tokens);
}
public ChunkNode ParseChunk(Token[] tokens)
{
BlockNode body = ParseBlock(tokens);
return new ChunkNode(block: body, startRegion: body.startRegion, endRegion: body.endRegion);
}
public BlockNode ParseBlock(Token[] tokens)
{
CodeRegion startRegion = tokens[index].region;
List<StatNode> stats = [];
while(index < tokens.Length &&
tokens[index].type != TokenType.Return &&
tokens[index].type != TokenType.End &&
tokens[index].type != TokenType.Elseif &&
tokens[index].type != TokenType.Else &&
tokens[index].type != TokenType.Until)
{
stats.Add(ParseStat(tokens));
}
BlockNode ret = new(stats: stats, startRegion: startRegion, endRegion: (stats.Count == 0 && index > 0) ? startRegion : tokens[index - 1].region);
if(index < tokens.Length && tokens[index].type == TokenType.Return)
{
ret.retstat = ParseRetstat(tokens);
}
return ret;
}
public StatNode ParseStat(Token[] tokens)
{
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}");
}
CodeRegion startRegion = tokens[index].region;
switch(tokens[index].type)
{
case TokenType.Semicolon:
{
index += 1;
return new StatNode.Semicolon(startRegion);
}
case TokenType.Break:
{
index += 1;
return new StatNode.Break(startRegion);
}
case TokenType.Goto:
{
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected name for goto at {startRegion}");
}
if(tokens[index].type != TokenType.Name)
{
throw new Exception($"{tokens[index].region}: Expected name for goto, got {tokens[index].type}");
}
StatNode.Goto ret = new(label: ((Token.StringData)tokens[index].data!).data, startRegion: startRegion, endRegion: tokens[index].region);
index += 1;
return ret;
}
case TokenType.Do:
{
index += 1;
BlockNode body = ParseBlock(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected end for `do` at {startRegion}");
}
if(tokens[index].type != TokenType.End)
{
throw new Exception($"{tokens[index].region}: Expected `end` to close `do` at {startRegion}, got {tokens[index].type}");
}
StatNode.Do ret = new(node: body, startRegion: startRegion, endRegion: tokens[index].region);
index += 1;
return ret;
}
case TokenType.While:
{
index += 1;
ExpNode condition = ParseExp(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `do` after condition of while loop starting at {startRegion}");
}
if(tokens[index].type != TokenType.Do)
{
throw new Exception($"{tokens[index].region}: Expected `do` after condition of while starting at {startRegion}, got {tokens[index].type}");
}
index += 1;
BlockNode body = ParseBlock(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `end` after body of while loop starting at {startRegion}");
}
CodeRegion endRegion = tokens[index].region;
index += 1;
return new StatNode.While(condition: condition, body: body, startRegion: startRegion, endRegion: endRegion);
}
case TokenType.Repeat:
{
index += 1;
BlockNode body = ParseBlock(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `until` after body of until loop starting at {startRegion}");
}
if(tokens[index].type != TokenType.Until)
{
throw new Exception($"{tokens[index].region}: Expected `until` after block of until loop starting at {startRegion}, got {tokens[index].type}");
}
index += 1;
ExpNode conditon = ParseExp(tokens);
return new StatNode.Repeat(condition: conditon, body: body, startRegion: startRegion, endRegion: tokens[index - 1].region);
}
case TokenType.If:
{
index += 1;
ExpNode condition = ParseExp(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `then` after condition of if starting at {startRegion}");
}
if(tokens[index].type != TokenType.Then)
{
throw new Exception($"{tokens[index].region}: Expected `then` after condition of if starting at {startRegion}, got {tokens[index].type}");
}
index += 1;
BlockNode body = ParseBlock(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `end` after body of if starting at {startRegion}");
}
List<ElseifNode> elseifs = [];
while(tokens[index].type == TokenType.Elseif)
{
CodeRegion elseifStartRegion = tokens[index].region;
index += 1;
ExpNode elseifCondition = ParseExp(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `then` after condition of elseif starting at {elseifStartRegion}");
}
if(tokens[index].type != TokenType.Then)
{
throw new Exception($"{tokens[index].region}: Expected `then` after condition of elseif starting at {elseifStartRegion}, got {tokens[index].type}");
}
index += 1;
BlockNode elseifBody = ParseBlock(tokens);
elseifs.Add(new(condition: elseifCondition, body: elseifBody, startRegion: elseifStartRegion, endRegion: elseifBody.endRegion));
}
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `end` after else-ifs of if starting at {startRegion}");
}
StatNode.If ret = new(condition: condition, body: body, elseifs: elseifs, startRegion: startRegion, endRegion: tokens[index - 1].region);
if(tokens[index].type == TokenType.Else)
{
index += 1;
ret.else_ = ParseBlock(tokens);
}
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `end` to close if starting at {startRegion}");
}
if(tokens[index].type != TokenType.End)
{
throw new Exception($"{tokens[index].region}: Expected `end` to close if starting at {startRegion}, got {tokens[index].type}");
}
ret.endRegion = tokens[index].region;
index += 1;
return ret;
}
case TokenType.For:
{
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected name after for at {startRegion}");
}
if(tokens[index].type != TokenType.Name)
{
throw new Exception($"{tokens[index].region}: Expected name after for at {startRegion}, got {tokens[index].type}");
}
string variable = ((Token.StringData)tokens[index].data!).data;
index += 1;
switch(tokens[index].type)
{
case TokenType.Equals:
{
index += 1;
ExpNode start = ParseExp(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `,` after start value of numerical for loop starting at {startRegion}");
}
if(tokens[index].type != TokenType.Comma)
{
throw new Exception($"{tokens[index].region}: Expected `,` after start value of for loop starting at {startRegion}, got {tokens[index].type}");
}
index += 1;
ExpNode end = ParseExp(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `do` or `,` after end value of numerical for loop starting at {startRegion}");
}
ExpNode? change = null;
if(tokens[index].type == TokenType.Comma)
{
index += 1;
change = ParseExp(tokens);
}
if(index >= tokens.Length)
{
string t = (change == null) ? "`do` or `,` after end value" : "`do` after change value";
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected {t} of numerical for loop starting at {startRegion}");
}
if(tokens[index].type != TokenType.Do)
{
string t = (change == null) ? "`do` or `,` after end value" : "`do` after change value";
throw new Exception($"{tokens[index].region}: Expected {t} of numerical for loop starting at {startRegion}, got {tokens[index].type}");
}
index += 1;
BlockNode body = ParseBlock(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `end` to close numerical for loop starting at {startRegion}");
}
if(tokens[index].type != TokenType.End)
{
throw new Exception($"{tokens[index].region}: Expected `end` to close numerical for loop starting at {startRegion}, got {tokens[index].type}");
}
CodeRegion endRegion = tokens[index].region;
index += 1;
return new StatNode.ForNumerical(variable: variable, start: start, end: end, change: change, body: body, startRegion: startRegion, endRegion: endRegion);
}
case TokenType.Comma:
{
List<string> names = [variable];
while(tokens[index].type == TokenType.Comma)
{
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected another name in name list of for-in loop starting at {startRegion}");
}
if(tokens[index].type != TokenType.Name)
{
throw new Exception($"{tokens[index].region}: Expected another name in name list of for-in loop starting at {startRegion}, got {tokens[index].type}");
}
names.Add(((Token.StringData)tokens[index].data!).data);
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `,` or `in` in for-in loop starting at {startRegion}");
}
}
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `in` after name list of for-in loop starting at {startRegion}");
}
if(tokens[index].type != TokenType.In)
{
throw new Exception($"{tokens[index].region}: Expected `in` after name list of for-in loop starting at {startRegion}, got {tokens[index].type}");
}
index += 1;
ExplistNode exps = ParseExplist(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `do` after exp list of for-in loop starting at {startRegion}");
}
if(tokens[index].type != TokenType.Do)
{
throw new Exception($"{tokens[index].region}: Expected `do` after exp list of for-in loop starting at {startRegion}, got {tokens[index].type}");
}
index += 1;
BlockNode body = ParseBlock(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `end` to close for-in loop starting at {startRegion}");
}
if(tokens[index].type != TokenType.End)
{
throw new Exception($"{tokens[index].region}: Expected `end` to close for-in loop starting at {startRegion}, got {tokens[index].type}");
}
CodeRegion endRegion = tokens[index].region;
index += 1;
return new StatNode.ForGeneric(vars: names, exps: exps, body: body, startRegion: startRegion, endRegion: endRegion);
}
case TokenType.In:
{
index += 1;
ExplistNode exps = ParseExplist(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `do` after exp list of for-in loop starting at {startRegion}");
}
if(tokens[index].type != TokenType.Do)
{
throw new Exception($"{tokens[index].region}: Expected `do` after exp list of for-in loop starting at {startRegion}, got {tokens[index].type}");
}
index += 1;
BlockNode body = ParseBlock(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `end` to close for-in loop starting at {startRegion}");
}
if(tokens[index].type != TokenType.End)
{
throw new Exception($"{tokens[index].region}: Expected `end` to close for-in loop starting at {startRegion}, got {tokens[index].type}");
}
CodeRegion endRegion = tokens[index].region;
index += 1;
return new StatNode.ForGeneric(vars: [variable], exps: exps, body: body, startRegion: startRegion, endRegion: endRegion);
}
default:
{
throw new Exception($"{tokens[index].type}: Expected either `=`, `,` or `in` after first name of for loop {startRegion}, got {tokens[index].type}");
}
}
}
case TokenType.Function:
{
index += 1;
FuncnameNode name = ParseFuncname(tokens);
FuncbodyNode body = ParseFuncbody(tokens);
return new StatNode.Function(new(name: name, body: body, startRegion: startRegion, endRegion: body.endRegion));
}
case TokenType.Local:
{
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length} after `local` at {startRegion}");
}
if(tokens[index].type == TokenType.Function)
{
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected name of local function starting at {startRegion}");
}
if(tokens[index].type != TokenType.Name)
{
throw new Exception($"{tokens[index].region}: Expected name of local function starting at {startRegion}, got {tokens[index].type}");
}
string name = ((Token.StringData)tokens[index].data!).data;
index += 1;
FuncbodyNode body = ParseFuncbody(tokens);
return new StatNode.LocalFunction(name: name, body: body, startRegion: startRegion, endRegion: body.endRegion);
}
else
{
AttnamelistNode attnames = ParseAttnamelist(tokens);
StatNode.Local ret = new(attnames: attnames, values: null, startRegion: startRegion, endRegion: attnames.endRegion);
if(index < tokens.Length && tokens[index].type == TokenType.Equals)
{
index += 1;
ret.values = ParseExplist(tokens);
ret.endRegion = ret.values.endRegion;
}
return ret;
}
}
case TokenType.ColonColon:
{
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected name of label starting at {startRegion}");
}
if(tokens[index].type != TokenType.Name)
{
throw new Exception($"{tokens[index].region}: Expected name of label starting at {startRegion}, got {tokens[index].type}");
}
string name = ((Token.StringData)tokens[index].data!).data;
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `::` after label name starting at {startRegion}");
}
if(tokens[index].type != TokenType.ColonColon)
{
throw new Exception($"{tokens[index].region}: Expected `::` after label name starting at {startRegion}, got {tokens[index].type}");
}
CodeRegion endRegion = tokens[index].region;
index += 1;
return new StatNode.Label(label: name, startRegion: startRegion, endRegion: endRegion);
}
case TokenType.Name:
case TokenType.RoundOpen:
{
SuffixexpNode suffixExp = ParseSuffixExp(tokens);
if(index >= tokens.Length)
{
if(suffixExp is SuffixexpNode.Normal)
{
throw new Exception($"{startRegion}: Expected function call, got normal suffix expression");
}
if(suffixExp is SuffixexpNode.Functioncall functioncall)
{
return new StatNode.Functioncall(node: functioncall.node);
}
}
else
{
switch(tokens[index].type)
{
case TokenType.Equals:
{
index += 1;
List<VarNode> lhs = [SuffixExpToVar(suffixExp)];
ExplistNode rhs = ParseExplist(tokens);
return new StatNode.Assignment(lhs: new(vars: lhs, startRegion: startRegion, endRegion: suffixExp.endRegion), rhs: rhs, startRegion: startRegion, endRegion: rhs.endRegion);
}
case TokenType.Comma:
{
List<VarNode> vars = [SuffixExpToVar(suffixExp)];
while(index < tokens.Length && tokens[index].type == TokenType.Comma)
{
index += 1;
vars.Add(ParseVar(tokens));
}
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `=` for assignment starting at {startRegion}");
}
if(tokens[index].type != TokenType.Equals)
{
throw new Exception($"{tokens[index].region}: Expected `=` for assignment starting at {startRegion}, got {tokens[index].type}");
}
index += 1;
VarlistNode varlistNode = new(vars: vars, startRegion: startRegion, endRegion: vars[^1].endRegion);
ExplistNode rhs = ParseExplist(tokens);
return new StatNode.Assignment(lhs: varlistNode, rhs: rhs, startRegion: startRegion, endRegion: rhs.endRegion);
}
}
if(suffixExp is SuffixexpNode.Normal)
{
throw new Exception($"{startRegion}: Expected function call, got normal suffix expression");
}
if(suffixExp is SuffixexpNode.Functioncall functioncall)
{
return new StatNode.Functioncall(node: functioncall.node);
}
}
}
break;
default:
{
throw new Exception($"Unexpected token {tokens[index]} at {startRegion}");
}
}
throw new NotImplementedException();
}
private VarNode ParseVar(Token[] tokens)
{
return SuffixExpToVar(ParseSuffixExp(tokens));
}
private static VarNode SuffixExpToVar(SuffixexpNode suffixExp)
{
if(suffixExp is not SuffixexpNode.Normal normal)
{
throw new Exception($"Expected a normal suffix expression to convert to var at {suffixExp.startRegion}-{suffixExp.endRegion}");
}
if(normal.suffixes.Count == 0)
{
if(normal.firstPart is not SuffixexpFirstPart.Name name)
{
throw new Exception($"Expected a name as first part of suffix expression to convert to var at {normal.firstPart.startRegion}-{normal.firstPart.endRegion}");
}
return new VarNode.Name(name: name.name, startRegion: suffixExp.startRegion, endRegion: suffixExp.endRegion);
}
SuffixexpSuffix last = normal.suffixes[^1];
_ = normal.suffixes.Remove(last);
return last switch
{
SuffixexpSuffix.Dot dot => new VarNode.Member(node: new(name: dot.name, value: normal, startRegion: suffixExp.startRegion, endRegion: suffixExp.endRegion), startRegion: suffixExp.startRegion, endRegion: dot.endRegion),
SuffixexpSuffix.Indexed indexed => new VarNode.Indexed(node: new(index: indexed.node, value: normal, startRegion: suffixExp.startRegion, endRegion: suffixExp.endRegion), startRegion: suffixExp.startRegion, endRegion: indexed.endRegion),
_ => throw new Exception($"Expected dot or indexed suffix expression to convert to var at {last.startRegion}-{last.endRegion}")
};
}
private SuffixexpNode ParseSuffixExp(Token[] tokens)
{
// primaryexp { '.' 'Name' | '[' exp']' | ':' 'Name' args | args }
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}");
}
CodeRegion startRegion = tokens[index].region;
SuffixexpFirstPart firstPart;
switch(tokens[index].type)
{
case TokenType.Name:
{
string name = ((Token.StringData)tokens[index].data!).data;
index += 1;
firstPart = new SuffixexpFirstPart.Name(name, startRegion, startRegion);
}
break;
case TokenType.RoundOpen:
{
index += 1;
ExpNode inner = ParseExp(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `)` to close bracketed expression starting at {startRegion}");
}
if(tokens[index].type != TokenType.RoundClosed)
{
throw new Exception($"{tokens[index].region}: Expected `)` to close bracketed expression at {startRegion}, got {tokens[index].type}");
}
firstPart = new SuffixexpFirstPart.BracketedExp(node: inner, startRegion: startRegion, endRegion: tokens[index].region);
index += 1;
}
break;
default:
throw new Exception($"{startRegion}: Expected either `)` or name as first part of suffix-expression, got {tokens[index].type}");
}
List<SuffixexpSuffix> suffixes = [];
bool shouldContinue = true;
while(shouldContinue && index < tokens.Length)
{
CodeRegion suffixStartRegion = tokens[index].region;
switch(tokens[index].type)
{
case TokenType.Dot:
{
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected name in dotted suffix of suffix expression starting at {startRegion}");
}
if(tokens[index].type != TokenType.Name)
{
throw new Exception($"{tokens[index].region}: Expected name in dotted suffix of suffix expression at {startRegion}, got {tokens[index].type}");
}
CodeRegion suffixEndRegion = tokens[index].region;
string name = ((Token.StringData)tokens[index].data!).data;
index += 1;
suffixes.Add(new SuffixexpSuffix.Dot(name, startRegion: suffixStartRegion, endRegion: suffixEndRegion));
}
break;
case TokenType.SquareOpen:
{
index += 1;
ExpNode inner = ParseExp(tokens);
suffixes.Add(new SuffixexpSuffix.Indexed(node: inner, startRegion: suffixStartRegion, endRegion: tokens[index - 1].region));
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `]` to close indexed suffix of suffix-expression starting at {suffixStartRegion}");
}
if(tokens[index].type != TokenType.SquareClosed)
{
throw new Exception($"{tokens[index].region}: Expected `]` to close indexed suffix of suffix-expression at {suffixStartRegion}, got {tokens[index].type}");
}
index += 1;
}
break;
case TokenType.Colon:
{
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected name as first arg after `:` in args suffix in suffix-expression starting at {suffixStartRegion}");
}
if(tokens[index].type != TokenType.Name)
{
throw new Exception($"{tokens[index].region}: Expected name as first arg after `:` in args suffix in suffix-expression at {suffixStartRegion}, got {tokens[index].type}");
}
string name = ((Token.StringData)tokens[index].data!).data;
index += 1;
ArgsNode args = ParseArgs(tokens);
suffixes.Add(new SuffixexpSuffix.ArgsFirstArg(new(name, rest: args, startRegion: suffixStartRegion, endRegion: args.endRegion)));
}
break;
case TokenType.RoundOpen:
case TokenType.CurlyOpen:
case TokenType.StringLiteral:
{
ArgsNode args = ParseArgs(tokens);
suffixes.Add(new SuffixexpSuffix.Args(node: args, startRegion: suffixStartRegion, endRegion: args.endRegion));
}
break;
default:
{
shouldContinue = false;
}
break;
}
}
CodeRegion endRegion;
if(suffixes.Count > 0)
{
endRegion = suffixes[^1].endRegion;
SuffixexpNode? ret = suffixes[^1] switch
{
SuffixexpSuffix.Args args => new SuffixexpNode.Functioncall(
node: new(
function: new SuffixexpNode.Normal(firstPart, suffixes[..^1], startRegion, args.endRegion),
args: args.node,
objectArg: null
)
),
SuffixexpSuffix.ArgsFirstArg node => new SuffixexpNode.Functioncall(
node: new(
function: new SuffixexpNode.Normal(firstPart: firstPart, suffixes: suffixes[..^1], startRegion, node.endRegion),
objectArg: node.node.name,
args: node.node.rest
)
),
_ => null,
};
if(ret is not null)
{
return ret;
}
}
else
{
endRegion = firstPart.endRegion;
}
return new SuffixexpNode.Normal(firstPart: firstPart, suffixes: suffixes, startRegion: startRegion, endRegion: endRegion);
}
private ArgsNode ParseArgs(Token[] tokens)
{
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `(`, `{{` or string to start args");
}
CodeRegion startRegion = tokens[index].region;
switch(tokens[index].type)
{
case TokenType.RoundOpen:
{
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected explist or `)` to continue args starting at {startRegion}");
}
if(tokens[index].type == TokenType.RoundClosed)
{
CodeRegion endRegion = tokens[index].region;
index += 1;
return new ArgsNode.Bracketed(null, startRegion: startRegion, endRegion: endRegion);
}
ExplistNode exps = ParseExplist(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `)` to close args starting at {startRegion}");
}
if(tokens[index].type != TokenType.RoundClosed)
{
throw new Exception($"{tokens[index].region}: Expected `)` to close args starting at {startRegion}, got {tokens[index].type}");
}
index += 1;
return new ArgsNode.Bracketed(node: exps, startRegion: startRegion, endRegion: exps.endRegion);
}
case TokenType.CurlyOpen:
{
TableconstructorNode node = ParseTableconstructor(tokens);
return new ArgsNode.Tableconstructor(node: node, startRegion: startRegion, endRegion: node.endRegion);
}
case TokenType.StringLiteral:
{
string value = ((Token.StringData)tokens[index].data!).data;
index += 1;
return new ArgsNode.Literal(name: value, startRegion: startRegion, endRegion: startRegion);
}
default:
throw new Exception($"{tokens[index].region}: Expected explist or `)` to continue args starting at {startRegion}");
}
}
private TableconstructorNode ParseTableconstructor(Token[] tokens)
{
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `{{` to start tableconstructor");
}
CodeRegion startRegion = tokens[index].region;
if(tokens[index].type != TokenType.CurlyOpen)
{
throw new Exception($"{startRegion}: Expected `{{` to start tableconstructor, got {tokens[index].type}");
}
index += 1;
if(index < tokens.Length && tokens[index].type == TokenType.CurlyClosed)
{
CodeRegion emptyEndRegion = tokens[index].region;
index += 1;
return new TableconstructorNode(exps: null, startRegion: startRegion, endRegion: emptyEndRegion);
}
FieldlistNode fields = ParseFieldlist(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `}}` to close tableconstructor starting at {startRegion}");
}
if(tokens[index].type != TokenType.CurlyClosed)
{
throw new Exception($"{tokens[index].region}: Expected `}}` to close tableconstructor starting at {startRegion}, got {tokens[index].type}");
}
CodeRegion endRegion = tokens[index].region;
index += 1;
return new TableconstructorNode(exps: fields, startRegion: startRegion, endRegion: endRegion);
}
private FieldlistNode ParseFieldlist(Token[] tokens)
{
List<FieldNode> fields = [ParseField(tokens)];
while(index < tokens.Length && IsFieldsep(tokens[index]))
{
index += 1;
if(index < tokens.Length && tokens[index].type is TokenType.SquareOpen or
TokenType.Name or TokenType.Nil or TokenType.True or TokenType.False or TokenType.Numeral or TokenType.StringLiteral or
TokenType.DotDotDot or TokenType.CurlyOpen or TokenType.Function or TokenType.Minus or TokenType.Hash or TokenType.Not or TokenType.Nil or TokenType.RoundOpen)
{
fields.Add(ParseField(tokens));
}
}
// NOTE: Since at least 1 field is parsed the list accesses are safe
return new FieldlistNode(exps: fields, startRegion: fields[0].startRegion, endRegion: fields[^1].endRegion);
}
private static bool IsFieldsep(Token token) => token.type is TokenType.Comma or TokenType.Semicolon;
private FieldNode ParseField(Token[] tokens)
{
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `[` or name to start field");
}
CodeRegion startRegion = tokens[index].region;
switch(tokens[index].type)
{
case TokenType.SquareOpen:
{
index += 1;
ExpNode indexNode = ParseExp(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `]` to close indexed field in indexed field assignment starting at {startRegion}");
}
if(tokens[index].type != TokenType.SquareClosed)
{
throw new Exception($"{tokens[index].region}: Expected `]` to close indexed field starting in indexed field assignment at {startRegion}, got {tokens[index].type}");
}
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `=` to continue indexed field assignment starting at {startRegion}");
}
if(tokens[index].type != TokenType.Equals)
{
throw new Exception($"{tokens[index].region}: Expected `=` to continue indexed field assignment starting at {startRegion}, got {tokens[index].type}");
}
index += 1;
ExpNode rhs = ParseExp(tokens);
return new FieldNode.IndexedAssignment(node: new(index: indexNode, rhs: rhs, startRegion: startRegion, endRegion: tokens[index - 1].region));
}
case TokenType.Name:
{
if(index + 1 < tokens.Length && tokens[index + 1].type == TokenType.Equals)
{
string name = ((Token.StringData)tokens[index].data!).data;
index += 2;
ExpNode rhs = ParseExp(tokens);
return new FieldNode.Assignment(node: new(lhs: name, rhs: rhs, startRegion: startRegion, endRegion: tokens[index - 1].region));
}
ExpNode exp = ParseExp(tokens);
return new FieldNode.Exp(node: exp, startRegion: startRegion, endRegion: tokens[index - 1].region);
}
default:
{
ExpNode exp = ParseExp(tokens);
return new FieldNode.Exp(node: exp, startRegion: startRegion, endRegion: tokens[index - 1].region);
}
}
}
private AttnamelistNode ParseAttnamelist(Token[] tokens)
{
List<AttnameNode> attnames = [ParseAttname(tokens)];
while(index < tokens.Length && tokens[index].type == TokenType.Comma)
{
index += 1;
attnames.Add(ParseAttname(tokens));
}
// NOTE: Since at least 1 attname is parsed the list accesses are safe
return new(attnames: attnames, startRegion: attnames[0].startRegion, endRegion: attnames[^1].endRegion);
}
private AttnameNode ParseAttname(Token[] tokens)
{
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected name to start attname");
}
CodeRegion startRegion = tokens[index].region;
if(tokens[index].type != TokenType.Name)
{
throw new Exception($"{tokens[index].region}: Expected name to start attname at {startRegion}, got {tokens[index].type}");
}
string name = ((Token.StringData)tokens[index].data!).data;
index += 1;
if(index < tokens.Length && tokens[index].type == TokenType.Lt)
{
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected attribute name of attname starting at {startRegion}");
}
if(tokens[index].type != TokenType.Name)
{
throw new Exception($"{tokens[index].region}: Expected attribute name of attname at {startRegion}, got {tokens[index].type}");
}
string attribute = ((Token.StringData)tokens[index].data!).data;
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `>` to close attribute of attname starting at {startRegion}");
}
if(tokens[index].type != TokenType.Gt)
{
throw new Exception($"{tokens[index].region}: Expected `>` to close attribute of attname starting at {startRegion}, got {tokens[index].type}");
}
CodeRegion endRegion = tokens[index].region;
index += 1;
return new AttnameNode(name: name, attribute: attribute, startRegion: startRegion, endRegion: endRegion);
}
return new AttnameNode(name: name, attribute: null, startRegion: startRegion, endRegion: startRegion);
}
private FuncbodyNode ParseFuncbody(Token[] tokens)
{
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `(` to start funcbody");
}
CodeRegion startRegion = tokens[index].region;
if(tokens[index].type != TokenType.RoundOpen)
{
throw new Exception($"{tokens[index].region}: Expected `(` to start funcbody at {startRegion}, got {tokens[index].type}");
}
index += 1;
ParlistNode? pars;
if(index < tokens.Length && tokens[index].type == TokenType.RoundClosed)
{
index += 1;
pars = null;
}
else
{
pars = ParseParlist(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `)` to close parlist of funcbody starting at {startRegion}");
}
if(tokens[index].type != TokenType.RoundClosed)
{
throw new Exception($"{tokens[index].region}: Expected `)` to close parlist of funcbody at {startRegion}, got {tokens[index].type}");
}
index += 1;
}
BlockNode body = ParseBlock(tokens);
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `end` to close funcbody starting at {startRegion}");
}
if(tokens[index].type != TokenType.End)
{
throw new Exception($"{tokens[index].region}: Expected `end` to close funcbody starting at {startRegion}, got {tokens[index].type}");
}
CodeRegion endRegion = tokens[index].region;
index += 1;
return new FuncbodyNode(pars: pars, body: body, startRegion: startRegion, endRegion: endRegion);
}
private ParlistNode ParseParlist(Token[] tokens)
{
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `...` or name to start parlist");
}
CodeRegion startRegion = tokens[index].region;
if(tokens[index].type == TokenType.DotDotDot)
{
index += 1;
return new ParlistNode(names: [], hasVarargs: true, startRegion: startRegion, endRegion: startRegion);
}
if(tokens[index].type != TokenType.Name)
{
throw new Exception($"{startRegion}: Expected `...` or name to start parlist, got {tokens[index].type}");
}
List<string> names = [((Token.StringData)tokens[index].data!).data];
index += 1;
while(index < tokens.Length && tokens[index].type == TokenType.Comma)
{
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `...` or name to continue parlist starting at {startRegion}");
}
switch(tokens[index].type)
{
case TokenType.Name:
{
names.Add(((Token.StringData)tokens[index].data!).data);
index += 1;
}
break;
case TokenType.DotDotDot:
{
CodeRegion endRegion = tokens[index].region;
index += 1;
return new ParlistNode(names: names, hasVarargs: true, startRegion: startRegion, endRegion: endRegion);
};
default:
{
throw new Exception($"{tokens[index].region}: Expected `...` or name to continue parlist starting at {startRegion}, got {tokens[index].type}");
}
}
}
return new ParlistNode(names: names, hasVarargs: false, startRegion: startRegion, endRegion: tokens[index - 1].region);
}
private FuncnameNode ParseFuncname(Token[] tokens)
{
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected name to start funcname");
}
CodeRegion startRegion = tokens[index].region;
if(tokens[index].type != TokenType.Name)
{
throw new Exception($"{startRegion}: Expected name to start funcname, got {tokens[index].type}");
}
string name = ((Token.StringData)tokens[index].data!).data;
index += 1;
List<string> dottedNames = [];
while(index < tokens.Length && tokens[index].type == TokenType.Dot)
{
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected name in dotted funcname starting at {startRegion}");
}
if(tokens[index].type != TokenType.Name)
{
throw new Exception($"{tokens[index].region}: Expected name in dotted funcname starting at {startRegion}, got {tokens[index].type}");
}
dottedNames.Add(((Token.StringData)tokens[index].data!).data);
index += 1;
}
if(index < tokens.Length && tokens[index].type == TokenType.Colon)
{
index += 1;
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected name as first arg name after `:` in funcname starting at {startRegion}");
}
if(tokens[index].type != TokenType.Name)
{
throw new Exception($"{tokens[index].region}: Expected name as first arg name after `:` in funcname starting at {startRegion}, got {tokens[index].type}");
}
string firstArg = ((Token.StringData)tokens[index].data!).data;
CodeRegion endRegion = tokens[index].region;
index += 1;
return new FuncnameNode(name: name, dottedNames: dottedNames, firstArg: firstArg, startRegion: startRegion, endRegion: endRegion);
}
return new FuncnameNode(name: name, dottedNames: dottedNames, firstArg: null, startRegion: startRegion, endRegion: tokens[index - 1].region);
}
private ExplistNode ParseExplist(Token[] tokens)
{
CodeRegion startRegion = tokens[index].region;
List<ExpNode> exps = [ParseExp(tokens)];
while(index < tokens.Length && tokens[index].type == TokenType.Comma)
{
index += 1;
exps.Add(ParseExp(tokens));
}
return new ExplistNode(exps: exps, startRegion: startRegion, endRegion: tokens[index - 1].region);
}
private ExpNode ParseExp(Token[] tokens)
{
ExpNode lhs = ParseExpPrimary(tokens);
return ParseExpPrecedence(tokens, lhs, 0);
}
private ExpNode ParseExpPrecedence(Token[] tokens, ExpNode lhs, int minPrecedence)
{
ExpNode currentLhs = lhs;
while(index < tokens.Length && IsBinop(tokens[index]))
{
CodeRegion startRegion = tokens[index].region;
int precedence = GetPrecedence(tokens[index]);
if(precedence < minPrecedence)
{
break;
}
BinopType op = GetBinopType(tokens[index]);
index += 1;
ExpNode rhs = ParseExpPrimary(tokens);
while(index < tokens.Length && IsBinop(tokens[index]) && (GetPrecedence(tokens[index]) > precedence || (GetPrecedence(tokens[index]) == precedence && IsRightAssociative(tokens[index]))))
{
int associativityBoost = (GetPrecedence(tokens[index]) == precedence) ? 0 : 1;
rhs = ParseExpPrecedence(tokens, lhs: rhs, minPrecedence: precedence + associativityBoost);
}
currentLhs = new ExpNode.Binop(node: new(lhs: currentLhs, type: op, rhs: rhs, startRegion: startRegion, endRegion: tokens[index - 1].region));
}
return currentLhs;
}
private static bool IsRightAssociative(Token token) => token.type is TokenType.DotDot or TokenType.Caret;
private static BinopType GetBinopType(Token token) => token.type switch
{
TokenType.Or => BinopType.LogicalOr,
TokenType.And => BinopType.LogicalAnd,
TokenType.Lt => BinopType.Lt,
TokenType.Gt => BinopType.Gt,
TokenType.LtEquals => BinopType.LtEquals,
TokenType.GtEquals => BinopType.GtEquals,
TokenType.LtLt => BinopType.Shl,
TokenType.GtGt => BinopType.Shr,
TokenType.TildeEquals => BinopType.NotEquals,
TokenType.EqualsEquals => BinopType.Equals,
TokenType.Pipe => BinopType.BinaryOr,
TokenType.Tilde => BinopType.BinaryNot,
TokenType.Ampersand => BinopType.BinaryAnd,
TokenType.DotDot => BinopType.Concat,
TokenType.Plus => BinopType.Add,
TokenType.Minus => BinopType.Sub,
TokenType.Star => BinopType.Mul,
TokenType.Slash => BinopType.Div,
TokenType.SlashSlash => BinopType.IntDiv,
TokenType.Percent => BinopType.Mod,
TokenType.Caret => BinopType.Exp,
_ => throw new Exception($"{token.region}: Expected binary operator with precedence, got {token.type}"),
};
private static int GetPrecedence(Token token) => token.type switch
{
TokenType.Or => 2,
TokenType.And => 4,
TokenType.Lt or TokenType.Gt or TokenType.LtEquals or TokenType.GtEquals or TokenType.TildeEquals or TokenType.EqualsEquals => 6,
TokenType.Pipe => 8,
TokenType.Tilde => 10,
TokenType.Ampersand => 12,
TokenType.LtLt or TokenType.GtGt => 14,
TokenType.DotDot => 16,
TokenType.Plus or TokenType.Minus => 18,
TokenType.Star or TokenType.Slash or TokenType.SlashSlash or TokenType.Percent => 20,
TokenType.Caret => 22,
_ => throw new Exception($"{token.region}: Expected binary operator with precedence, got {token.type}"),
};
private static bool IsBinop(Token token) => token.type switch
{
TokenType.Or or TokenType.And or TokenType.Lt or TokenType.Gt or TokenType.LtEquals or TokenType.GtEquals or TokenType.TildeEquals or TokenType.EqualsEquals or
TokenType.Pipe or TokenType.Tilde or TokenType.Ampersand or TokenType.LtLt or TokenType.GtGt or TokenType.DotDot or TokenType.Plus or TokenType.Minus or
TokenType.Star or TokenType.Slash or TokenType.SlashSlash or TokenType.Percent or TokenType.Caret => true,
_ => false
};
private ExpNode ParseExpPrimary(Token[] tokens)
{
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected primary expression (`nil`, `true`, `false`, numeral, string, `...`, `function`, `{{`, `#`, `not`, `~`)");
}
CodeRegion startRegion = tokens[index].region;
switch(tokens[index].type)
{
case TokenType.Nil:
{
index += 1;
return new ExpNode.Nil(region: startRegion);
}
case TokenType.True:
{
index += 1;
return new ExpNode.True(region: startRegion);
}
case TokenType.False:
{
index += 1;
return new ExpNode.False(region: startRegion);
}
case TokenType.Numeral:
{
INumeral numeral = ((Token.NumeralData)tokens[index].data!).numeral;
index += 1;
return new ExpNode.Numeral(value: numeral, region: startRegion);
}
case TokenType.StringLiteral:
{
string value = ((Token.StringData)tokens[index].data!).data;
index += 1;
return new ExpNode.LiteralString(value: value, region: startRegion);
}
case TokenType.DotDotDot:
{
index += 1;
return new ExpNode.Varargs(region: startRegion);
}
case TokenType.CurlyOpen:
{
TableconstructorNode inner = ParseTableconstructor(tokens);
return new ExpNode.Tableconstructor(node: inner);
}
case TokenType.Function:
{
index += 1;
FuncbodyNode body = ParseFuncbody(tokens);
return new ExpNode.Functiondef(node: body, startRegion: startRegion, endRegion: body.endRegion);
}
case TokenType.Minus:
{
index += 1;
ExpNode unop = ParseExp(tokens);
return new ExpNode.Unop(node: new(type: UnopType.Minus, exp: unop, startRegion: startRegion, endRegion: tokens[index - 1].region));
}
case TokenType.Hash:
{
index += 1;
ExpNode unop = ParseExp(tokens);
return new ExpNode.Unop(node: new(type: UnopType.Length, exp: unop, startRegion: startRegion, endRegion: tokens[index - 1].region));
}
case TokenType.Not:
{
index += 1;
ExpNode unop = ParseExp(tokens);
return new ExpNode.Unop(node: new(type: UnopType.LogicalNot, exp: unop, startRegion: startRegion, endRegion: tokens[index - 1].region));
}
case TokenType.Tilde:
{
index += 1;
ExpNode unop = ParseExp(tokens);
return new ExpNode.Unop(node: new(type: UnopType.BinaryNot, exp: unop, startRegion: startRegion, endRegion: tokens[index - 1].region));
}
default:
{
SuffixexpNode suffixexp = ParseSuffixExp(tokens);
return new ExpNode.Suffixexp(node: suffixexp);
}
}
}
private RetstatNode ParseRetstat(Token[] tokens)
{
if(index >= tokens.Length)
{
throw new Exception($"Index {index} out of bounds of {tokens.Length}, expected `return` to start retstat");
}
CodeRegion startRegion = tokens[index].region;
if(tokens[index].type != TokenType.Return)
{
throw new Exception($"{startRegion}: Expected `return` to start retstat, got {tokens[index].type}");
}
index += 1;
if(index >= tokens.Length)
{
return new RetstatNode(values: null, startRegion: startRegion, endRegion: startRegion);
}
if(tokens[index].type is TokenType.Semicolon or TokenType.Else or TokenType.Elseif or TokenType.End)
{
CodeRegion emptyEndRegion;
if(tokens[index].type == TokenType.Semicolon)
{
emptyEndRegion = tokens[index].region;
index += 1;
}
else
{
emptyEndRegion = startRegion;
}
return new RetstatNode(values: null, startRegion: startRegion, endRegion: emptyEndRegion);
}
ExplistNode values = ParseExplist(tokens);
CodeRegion endRegion;
if(index < tokens.Length && tokens[index].type == TokenType.Semicolon)
{
endRegion = tokens[index].region;
index += 1;
}
else
{
endRegion = values.endRegion;
}
return new RetstatNode(values: values, startRegion: startRegion, endRegion: endRegion);
}
}

94
Program.cs
View File

@ -1,94 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text.Json;
namespace luaaaaah;
public class Program
{
internal static JsonSerializerOptions options = new()
{
IncludeFields = true,
WriteIndented = true,
};
public static void Main(string[] args)
{
switch(args[0])
{
case "test":
{
Test(args[1]);
}
break;
case "run":
{
Run(args[1], true);
}
break;
}
}
public static void Run(string file, bool debug)
{
string content = File.ReadAllText(file).ReplaceLineEndings();
Token[] tokens = new Tokenizer().Tokenize(content);
if(debug)
{
foreach(Token token in tokens)
{
Console.WriteLine($"{token.region}: {token.type} {{{token.data}}}");
}
}
if(tokens.Length == 0)
{
return;
}
if(Path.GetFileName(file).StartsWith("tokenizer"))
{
Console.WriteLine($"Skipping parsing of `{file}`");
}
else
{
Parser.ChunkNode root = new Parser().Parse(tokens);
if(debug)
{
Console.WriteLine("Parsed tree:");
Console.WriteLine(JsonSerializer.Serialize(root, options: options));
}
}
}
static readonly Dictionary<string, string> failedFiles = [];
public static void Test(string directory)
{
TestRecursive(directory);
Console.WriteLine("===FAILED===");
foreach(KeyValuePair<string, string> entry in failedFiles)
{
Console.WriteLine($"{entry.Key}: {entry.Value}");
}
Console.WriteLine($"==={failedFiles.Count}===");
}
public static void TestRecursive(string directory)
{
foreach(string file in Directory.EnumerateFiles(directory))
{
if(file.EndsWith(".lua"))
{
try
{
Run(file, false);
}
catch(Exception e)
{
Console.WriteLine($"{file}: {e}");
failedFiles.Add(file, e.ToString());
}
}
}
foreach(string dir in Directory.EnumerateDirectories(directory))
{
TestRecursive(dir);
}
}
}

2843
Tokenizer.cs
View File

@ -1,2843 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace luaaaaah;
class Tokenizer
{
private readonly List<Token> tokens = [];
private State state = State.Start;
int? lastIndex;
int index;
int openingLongBracketLevel;
int closingLongBracketLevel;
Token? currentToken;
CodeLocation currentLocation = new(line: 0, col: 0);
long escapeSequenceNumber;
public Token[] Tokenize(string content)
{
if(content.StartsWith('#'))
{
content = content[content.IndexOf('\n')..];
}
while(index < content.Length)
{
TokenizeChar(content[index]);
if(content[index] == '\n')
{
currentLocation.line += 1;
currentLocation.col = 0;
}
else
{
currentLocation.col += 1;
}
index += 1;
}
TokenizeChar('\n');
return [.. tokens];
}
private void AppendDataChar(char ch)
{
if((Token.StringData?)currentToken!.data == null)
{
currentToken!.data = new Token.StringData($"{ch}");
}
else
{
((Token.StringData?)currentToken!.data!).data += ch;
}
currentToken.region.end = new(currentLocation);
}
private void AppendDataInt(char ch)
{
if((Token.NumeralData?)currentToken!.data == null)
{
currentToken!.data = new Token.NumeralData(new INumeral.Integer(ch - '0'));
}
else
{
((INumeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value *= 10;
((INumeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value += ch - '0';
}
currentToken.region.end = new(currentLocation);
}
private void AppendDataIntHex(char ch)
{
int v = char.IsAsciiDigit(ch) ? ch - '0' : 10 + char.ToLower(ch) - 'a';
if((Token.NumeralData?)currentToken!.data == null)
{
currentToken!.data = new Token.NumeralData(new INumeral.Integer(v));
}
else
{
((INumeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value *= 16;
((INumeral.Integer)((Token.NumeralData?)currentToken!.data!).numeral).value += v;
}
currentToken.region.end = new(currentLocation);
}
private void TokenizeTerminal(State newState, TokenType type)
{
lastIndex = index;
state = newState;
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: type);
}
private void TokenizeTerminalName(State newState, char ch)
{
lastIndex = index;
state = newState;
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}"));
}
private void Backtrack(TokenType newType)
{
if(currentToken == null || currentToken.type == null)
{
throw new Exception($"Lexer error at {currentLocation}");
}
currentToken.type = newType;
currentToken.data = null;
currentLocation = new(currentToken.region.end);
tokens.Add(currentToken);
currentToken = null;
index = lastIndex!.Value;
lastIndex = null;
state = State.Start;
}
private void BacktrackNoClear(TokenType newType)
{
if(currentToken == null || currentToken.type == null)
{
throw new Exception($"Lexer error at {currentLocation}");
}
currentToken.type = newType;
currentLocation = new(currentToken.region.end);
tokens.Add(currentToken);
currentToken = null;
index = lastIndex!.Value;
lastIndex = null;
state = State.Start;
}
private void BacktrackNoTypeChange()
{
if(currentToken == null || currentToken.type == null)
{
throw new Exception($"Lexer error at {currentLocation}");
}
currentLocation = new(currentToken.region.end);
tokens.Add(currentToken);
currentToken = null;
index = lastIndex!.Value;
lastIndex = null;
state = State.Start;
}
private void TokenizeChar(char ch)
{
switch(state)
{
case State.Start:
{
switch(ch)
{
case '-':
TokenizeTerminal(State.Minus, TokenType.Minus);
break;
case ',':
TokenizeTerminal(State.Comma, TokenType.Comma);
break;
case '=':
TokenizeTerminal(State.Equals, TokenType.Equals);
break;
case '(':
TokenizeTerminal(State.RoundOpen, TokenType.RoundOpen);
break;
case ')':
TokenizeTerminal(State.RoundClosed, TokenType.RoundClosed);
break;
case '.':
TokenizeTerminal(State.Dot, TokenType.Dot);
break;
case ':':
TokenizeTerminal(State.Colon, TokenType.Colon);
break;
case '{':
TokenizeTerminal(State.CurlyOpen, TokenType.CurlyOpen);
break;
case '}':
TokenizeTerminal(State.CurlyClosed, TokenType.CurlyClosed);
break;
case '[':
TokenizeTerminal(State.SquareOpen, TokenType.SquareOpen);
break;
case ']':
TokenizeTerminal(State.SquareClosed, TokenType.SquareClosed);
break;
case '+':
TokenizeTerminal(State.Plus, TokenType.Plus);
break;
case '~':
TokenizeTerminal(State.Tilde, TokenType.Tilde);
break;
case '>':
TokenizeTerminal(State.Gt, TokenType.Gt);
break;
case '<':
TokenizeTerminal(State.Lt, TokenType.Lt);
break;
case '#':
TokenizeTerminal(State.Hash, TokenType.Hash);
break;
case '|':
TokenizeTerminal(State.Pipe, TokenType.Pipe);
break;
case '&':
TokenizeTerminal(State.Ampersand, TokenType.Ampersand);
break;
case '%':
TokenizeTerminal(State.Percent, TokenType.Percent);
break;
case '*':
TokenizeTerminal(State.Star, TokenType.Star);
break;
case '/':
TokenizeTerminal(State.Slash, TokenType.Slash);
break;
case ';':
TokenizeTerminal(State.Semicolon, TokenType.Semicolon);
break;
case '^':
TokenizeTerminal(State.Caret, TokenType.Caret);
break;
case 'a':
TokenizeTerminalName(State.A, ch);
break;
case 'b':
TokenizeTerminalName(State.B, ch);
break;
case 'd':
TokenizeTerminalName(State.D, ch);
break;
case 'e':
TokenizeTerminalName(State.E, ch);
break;
case 'f':
TokenizeTerminalName(State.F, ch);
break;
case 'i':
TokenizeTerminalName(State.I, ch);
break;
case 'g':
TokenizeTerminalName(State.G, ch);
break;
case 'l':
TokenizeTerminalName(State.L, ch);
break;
case 'n':
TokenizeTerminalName(State.N, ch);
break;
case 'o':
TokenizeTerminalName(State.O, ch);
break;
case 'r':
TokenizeTerminalName(State.R, ch);
break;
case 't':
TokenizeTerminalName(State.T, ch);
break;
case 'u':
TokenizeTerminalName(State.U, ch);
break;
case 'w':
TokenizeTerminalName(State.W, ch);
break;
case '0':
{
lastIndex = index;
state = State.Zero;
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Numeral, data: new Token.NumeralData(new INumeral.Integer(0)));
} /* tokenizeTerminalIntNum(TokenType.Numeral, TokenizerState.Zero, tokenNumeral, ch); */
break;
case '"':
{
state = State.Quote;
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral);
}
break;
case '\'':
{
state = State.SingleQuote;
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral);
}
break;
default:
{
if(char.IsWhiteSpace(ch)) { }
else if(char.IsAsciiLetter(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Name, data: new Token.StringData($"{ch}"));
}
else if(char.IsDigit(ch))
{
lastIndex = index;
state = State.Integer;
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Numeral, data: new Token.NumeralData(new INumeral.Integer(ch - '0')));
}
else
{
throw new NotImplementedException($"{ch} at {currentLocation}");
}
}
break;
}
}
break;
case State.Quote:
{
if(ch == '\\')
{
state = State.QuoteBackslash;
}
else if(ch == '"')
{
lastIndex = index;
state = State.String;
if(currentToken == null || currentToken.type == null)
{
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral);
}
else
{
currentToken.type = TokenType.StringLiteral;
currentToken.region.end = new(currentLocation);
currentToken.data ??= new Token.StringData("");
}
}
else
{
AppendDataChar(ch);
}
}
break;
case State.QuoteBackslash:
{
switch(ch)
{
case 'a':
{
AppendDataChar('\u0007');
state = State.Quote;
}
break;
case 'b':
{
AppendDataChar('\u0008');
state = State.Quote;
}
break;
case 't':
{
AppendDataChar('\t');
state = State.Quote;
}
break;
case 'n':
case '\n':
{
AppendDataChar('\n');
state = State.Quote;
}
break;
case 'v':
{
AppendDataChar('\u000b');
state = State.Quote;
}
break;
case 'f':
{
AppendDataChar('\u000c');
state = State.Quote;
}
break;
case 'r':
{
AppendDataChar('\r');
state = State.Quote;
}
break;
case '\\':
{
AppendDataChar('\\');
state = State.Quote;
}
break;
case '"':
{
AppendDataChar('"');
state = State.Quote;
}
break;
case '\'':
{
AppendDataChar('\'');
state = State.Quote;
}
break;
case 'z':
{
state = State.QuoteBackslashZ;
}
break;
case 'x':
{
state = State.QuoteBackslashX;
throw new NotImplementedException($"\\x escape sequences are broken right now");
}
case 'u':
{
state = State.QuoteBackslashU;
throw new NotImplementedException($"\\u escape sequences are broken right now");
}
default: throw new Exception($"Unknown escape sequence: \\{ch} at {currentLocation}");
}
}
break;
case State.QuoteBackslashU:
{
if(ch == '{')
{
state = State.QuoteBackslashUBracket;
}
else
{
throw new Exception($"Expected `{{` to continue \\u escape sequence at {currentLocation}, got {ch}");
}
}
break;
case State.QuoteBackslashUBracket:
{
if(char.IsAsciiHexDigit(ch))
{
state = State.QuoteBackslashUBracketHex;
escapeSequenceNumber = char.IsAsciiDigit(ch) ? ch - '0' : 10 + char.ToLower(ch) - 'a';
}
else
{
throw new Exception($"Expected hex digit to continue \\u escape sequence at {currentLocation}, got {ch}");
}
}
break;
case State.QuoteBackslashUBracketHex:
{
if(char.IsAsciiHexDigit(ch))
{
escapeSequenceNumber = (escapeSequenceNumber * 16) + (char.IsAsciiDigit(ch) ? ch - '0' : 10 + char.ToLower(ch) - 'a');
if(escapeSequenceNumber > uint.MaxValue)
{
throw new Exception($"{currentLocation}: \\u escape sequence has a value > 2^31 which is not permitted");
}
}
else if(ch == '}')
{
state = State.Quote;
// TODO: THIS IS WRONG, there is zero padding due to the fixed size array
char[] chars = Encoding.UTF8.GetChars(BitConverter.GetBytes((uint)escapeSequenceNumber));
for(int i = 0; i < chars.Length; i++)
{
AppendDataChar(chars[i]);
}
escapeSequenceNumber = 0;
}
else
{
throw new Exception($"Expected second hex digit to continue \\u escape sequence at {currentLocation}, got {ch}");
}
}
break;
case State.QuoteBackslashZ:
{
if(ch == '\\')
{
state = State.QuoteBackslash;
}
else if(ch == '"')
{
lastIndex = index;
state = State.String;
if(currentToken == null || currentToken.type == null)
{
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral);
}
else
{
currentToken.type = TokenType.StringLiteral;
currentToken.region.end = new(currentLocation);
currentToken.data = new Token.StringData("");
}
}
else if(!char.IsWhiteSpace(ch))
{
AppendDataChar(ch);
state = State.Quote;
}
else
{
// Noop, https://www.lua.org/manual/5.4/manual.html#3.1:
// "The escape sequence '\z' skips the following span of whitespace characters, including line breaks;"
}
}
break;
case State.SingleQuote:
{
if(ch == '\\')
{
state = State.SingleQuoteBackslash;
}
else if(ch == '\'')
{
lastIndex = index;
state = State.String;
if(currentToken == null || currentToken.type == null)
{
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral);
}
else
{
currentToken.type = TokenType.StringLiteral;
currentToken.region.end = new(currentLocation);
currentToken.data ??= new Token.StringData("");
}
}
else
{
AppendDataChar(ch);
}
}
break;
case State.SingleQuoteBackslash:
{
switch(ch)
{
case 'a':
{
AppendDataChar('\u0007');
state = State.SingleQuote;
}
break;
case 'b':
{
AppendDataChar('\u0008');
state = State.SingleQuote;
}
break;
case 't':
{
AppendDataChar('\t');
state = State.SingleQuote;
}
break;
case 'n':
case '\n':
{
AppendDataChar('\n');
state = State.SingleQuote;
}
break;
case 'v':
{
AppendDataChar('\u000b');
state = State.SingleQuote;
}
break;
case 'f':
{
AppendDataChar('\u000c');
state = State.SingleQuote;
}
break;
case 'r':
{
AppendDataChar('\r');
state = State.SingleQuote;
}
break;
case '\\':
{
AppendDataChar('\\');
state = State.SingleQuote;
}
break;
case '"':
{
AppendDataChar('"');
state = State.SingleQuote;
}
break;
case '\'':
{
AppendDataChar('\'');
state = State.SingleQuote;
}
break;
case 'z':
state = State.SingleQuoteBackslashZ;
break;
case 'x':
state = State.SingleQuoteBackslashX;
break;
case 'u':
state = State.SingleQuoteBackslashU;
break;
default: throw new Exception($"Unknown escape sequence: \\{ch}");
}
}
break;
case State.SingleQuoteBackslashU:
state = ch == '{'
? State.SingleQuoteBackslashUBracket
: throw new Exception($"Expected `{{` to continue \\u escape sequence at {currentLocation}, got {ch}");
break;
case State.SingleQuoteBackslashUBracket:
{
if(char.IsAsciiHexDigit(ch))
{
state = State.SingleQuoteBackslashUBracketHex;
escapeSequenceNumber = char.IsAsciiDigit(ch) ? ch - '0' : 10 + char.ToLower(ch) - 'a';
}
else
{
throw new Exception($"Expected hex digit to continue \\u escape sequence at {currentLocation}, got {ch}");
}
}
break;
case State.SingleQuoteBackslashUBracketHex:
{
if(char.IsAsciiHexDigit(ch))
{
escapeSequenceNumber = (escapeSequenceNumber * 16) + (char.IsAsciiDigit(ch) ? ch - '0' : 10 + char.ToLower(ch) - 'a');
if(escapeSequenceNumber > uint.MaxValue)
{
throw new Exception($"{currentLocation}: \\u escape sequence has a value > 2^31 which is not permitted");
}
}
else if(ch == '}')
{
state = State.SingleQuote;
// TODO: THIS IS WRONG, there is zero padding due to the fixed size array
char[] chars = Encoding.UTF8.GetChars(BitConverter.GetBytes((uint)escapeSequenceNumber));
for(int i = 0; i < chars.Length; i++)
{
AppendDataChar(chars[i]);
}
escapeSequenceNumber = 0;
}
else
{
throw new Exception($"Expected second hex digit to continue \\u escape sequence at {currentLocation}, got {ch}");
}
}
break;
case State.SingleQuoteBackslashZ:
{
if(ch == '\\')
{
state = State.SingleQuoteBackslash;
}
else if(ch == '\'')
{
lastIndex = index;
state = State.String;
if(currentToken == null || currentToken.type == null)
{
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral);
}
else
{
currentToken.type = TokenType.StringLiteral;
currentToken.region.end = new(currentLocation);
}
}
else if(!char.IsWhiteSpace(ch))
{
AppendDataChar(ch);
state = State.SingleQuote;
}
else
{
// Noop, https://www.lua.org/manual/5.4/manual.html#3.1:
// "The escape sequence '\z' skips the following span of whitespace characters, including line breaks;"
}
}
break;
case State.SingleQuoteBackslashX:
{
if(char.IsAsciiHexDigit(ch))
{
state = State.SingleQuoteBackslashXHex;
escapeSequenceNumber = char.IsAsciiDigit(ch) ? ch - '0' : 10 + char.ToLower(ch) - 'a';
}
else
{
throw new Exception($"{currentLocation}: Expected hex digit in \\x escape sequence, got {ch}");
}
}
break;
case State.SingleQuoteBackslashXHex:
{
if(char.IsAsciiHexDigit(ch))
{
state = State.SingleQuote;
escapeSequenceNumber = (escapeSequenceNumber * 16) + (char.IsAsciiDigit(ch) ? ch - '0' : 10 + char.ToLower(ch) - 'a');
// TODO: THIS IS WRONG, there is zero padding due to the fixed size array
foreach(char c in Encoding.UTF8.GetChars(BitConverter.GetBytes(escapeSequenceNumber)))
{
AppendDataChar(c);
}
escapeSequenceNumber = 0;
}
else
{
throw new Exception($"{currentLocation}: Expected second hex digit in \\x escape sequence, got {ch}");
}
}
break;
case State.QuoteBackslashX:
{
if(char.IsAsciiHexDigit(ch))
{
state = State.QuoteBackslashXHex;
escapeSequenceNumber = char.IsAsciiDigit(ch) ? ch - '0' : 10 + char.ToLower(ch) - 'a';
}
else
{
throw new Exception($"{currentLocation}: Expected hex digit in \\x escape sequence, got {ch}");
}
}
break;
case State.QuoteBackslashXHex:
{
if(char.IsAsciiHexDigit(ch))
{
state = State.Quote;
escapeSequenceNumber = (escapeSequenceNumber * 16) + (char.IsAsciiDigit(ch) ? ch - '0' : 10 + char.ToLower(ch) - 'a');
// TODO: THIS IS WRONG, there is zero padding due to the fixed size array
foreach(char c in Encoding.UTF8.GetChars(BitConverter.GetBytes(escapeSequenceNumber)))
{
AppendDataChar(c);
}
escapeSequenceNumber = 0;
}
else
{
throw new Exception($"{currentLocation}: Expected second hex digit in \\x escape sequence, got {ch}");
}
}
break;
case State.String:
{
BacktrackNoClear(TokenType.StringLiteral);
}
break;
case State.Name:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Zero:
{
if(ch is 'x' or 'X')
{
currentToken!.type = null;
state = State.HexNumberX;
}
else if(ch == '.')
{
state = State.Float;
currentToken!.type = null;
currentToken!.data = null;
AppendDataChar('0');
AppendDataChar('.');
}
else if(char.IsAsciiDigit(ch))
{
lastIndex = index;
AppendDataInt(ch);
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.Float:
{
if(char.IsAsciiDigit(ch))
{
lastIndex = index;
AppendDataChar(ch);
}
else
{
if(currentToken == null)
{
throw new Exception($"Lexer error at {currentLocation}");
}
currentLocation = new(currentToken.region.end);
currentToken.type = TokenType.Numeral;
currentToken.data = new Token.NumeralData(new INumeral.Float(float.Parse(((Token.StringData)currentToken.data!).data)));
tokens.Add(currentToken);
currentToken = null;
index = lastIndex!.Value;
lastIndex = null;
state = State.Start;
}
}
break;
case State.HexNumberX:
{
if(char.IsAsciiHexDigit(ch))
{
lastIndex = index;
currentToken!.type = TokenType.Numeral;
AppendDataIntHex(ch);
state = State.HexNumber;
}
else if(ch == '.')
{
throw new NotImplementedException($"{currentLocation}: Hex floats at are not implemented");
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.HexNumber:
{
if(ch == 'p')
{
currentToken!.type = null;
state = State.HexExpNumber;
}
else if(char.IsAsciiHexDigit(ch))
{
lastIndex = index;
currentToken!.type = TokenType.Numeral;
AppendDataIntHex(ch);
}
else if(ch == '.')
{
throw new NotImplementedException($"{currentLocation}: Hex floats at are not implemented");
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.Integer:
{
if(ch == 'e')
{
currentToken!.type = null;
state = State.ExpNumber;
}
else if(ch == '.')
{
currentToken!.type = null;
currentToken.data = new Token.StringData($"{((INumeral.Integer)((Token.NumeralData)currentToken!.data!).numeral).value}.");
state = State.Float;
}
else if(char.IsAsciiDigit(ch))
{
lastIndex = index;
currentToken!.type = TokenType.Numeral;
AppendDataInt(ch);
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.SquareOpen:
{
if(ch == '[')
{
currentToken = new Token(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral);
state = State.StringWithLongBracket;
}
else if(ch == '=')
{
openingLongBracketLevel = 1;
state = State.StringStartLongBracket;
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.Comma:
case State.RoundOpen:
case State.RoundClosed:
case State.CurlyOpen:
case State.CurlyClosed:
case State.Plus:
case State.TildeEquals:
case State.EqualsEquals:
case State.Hash:
case State.GtEquals:
case State.LtEquals:
case State.SquareClosed:
case State.Pipe:
case State.Ampersand:
case State.Percent:
case State.Star:
case State.Semicolon:
case State.Caret:
case State.DotDotDot:
case State.GtGt:
case State.LtLt:
case State.ColonColon:
case State.SlashSlash:
{
BacktrackNoTypeChange();
}
break;
case State.Tilde:
{
if(ch == '=')
{
lastIndex = index;
state = State.TildeEquals;
currentToken!.type = TokenType.TildeEquals;
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.Gt:
{
if(ch == '=')
{
lastIndex = index;
state = State.GtEquals;
currentToken!.type = TokenType.GtEquals;
}
else if(ch == '>')
{
lastIndex = index;
state = State.GtGt;
currentToken!.type = TokenType.GtGt;
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.Lt:
{
if(ch == '=')
{
lastIndex = index;
state = State.LtEquals;
currentToken!.type = TokenType.LtEquals;
}
else if(ch == '<')
{
lastIndex = index;
state = State.LtLt;
currentToken!.type = TokenType.LtLt;
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.Slash:
{
if(ch == '/')
{
lastIndex = index;
state = State.SlashSlash;
currentToken!.type = TokenType.SlashSlash;
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.Dot:
{
if(ch == '.')
{
lastIndex = index;
state = State.DotDot;
currentToken!.type = TokenType.DotDot;
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.DotDot:
{
if(ch == '.')
{
lastIndex = index;
state = State.DotDotDot;
currentToken!.type = TokenType.DotDotDot;
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.Colon:
{
if(ch == ':')
{
lastIndex = index;
state = State.ColonColon;
currentToken!.type = TokenType.ColonColon;
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.Equals:
{
if(ch == '=')
{
lastIndex = index;
state = State.EqualsEquals;
currentToken!.type = TokenType.EqualsEquals;
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.Minus:
{
if(ch == '-')
{
lastIndex = index;
state = State.SmallCommentStart;
currentToken = null;
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.SmallCommentStart:
{
if(ch == '[')
{
state = State.BigCommentStartLongBracket;
}
else if(ch == '\n')
{
state = State.Start;
lastIndex = null;
}
else
{
state = State.SmallComment;
}
}
break;
case State.SmallComment:
{
if(ch == '\n')
{
state = State.Start;
lastIndex = null;
}
}
break;
case State.BigCommentStartLongBracket:
{
if(ch == '=')
{
openingLongBracketLevel += 1;
}
else if(ch == '[')
{
state = State.BigComment;
}
else if(ch == '\n')
{
state = State.Start;
}
else
{
state = State.SmallComment;
}
}
break;
case State.BigComment:
{
if(ch == ']')
{
state = State.BigCommentEndLongBracket;
closingLongBracketLevel = 0;
}
}
break;
case State.BigCommentEndLongBracket:
{
if(ch == '=')
{
closingLongBracketLevel += 1;
if(openingLongBracketLevel < closingLongBracketLevel)
{
state = State.BigComment;
}
}
else if(ch == ']' && openingLongBracketLevel == closingLongBracketLevel)
{
state = State.Start;
openingLongBracketLevel = 0;
closingLongBracketLevel = 0;
}
else
{
closingLongBracketLevel = 0;
state = State.BigComment;
}
}
break;
case State.StringStartLongBracket:
{
if(ch == '=')
{
openingLongBracketLevel += 1;
}
else if(ch == '[')
{
state = State.StringWithLongBracket;
}
else
{
BacktrackNoTypeChange();
}
}
break;
case State.StringWithLongBracket:
{
if(ch == ']')
{
state = State.StringEndLongBracket;
closingLongBracketLevel = 0;
}
else
{
AppendDataChar(ch);
}
}
break;
case State.StringEndLongBracket:
{
if(ch == '=')
{
closingLongBracketLevel += 1;
if(openingLongBracketLevel < closingLongBracketLevel)
{
state = State.StringWithLongBracket;
}
AppendDataChar(ch);
}
else if(ch == ']' && openingLongBracketLevel == closingLongBracketLevel)
{
if(currentToken == null || currentToken.type == null)
{
throw new Exception($"Lexer error at {currentLocation}");
}
if((Token.StringData?)currentToken.data == null)
{
currentToken.data = new Token.StringData("");
}
currentToken.type = TokenType.StringLiteral;
((Token.StringData)currentToken.data).data = ((Token.StringData)currentToken.data).data.Remove(((Token.StringData)currentToken.data).data.Length - closingLongBracketLevel);
currentLocation = new(currentToken.region.end);
tokens.Add(currentToken);
currentToken = null;
lastIndex = null;
state = State.Start;
openingLongBracketLevel = 0;
closingLongBracketLevel = 0;
}
else
{
closingLongBracketLevel = 0;
AppendDataChar(ch);
state = State.StringWithLongBracket;
}
}
break;
case State.A:
{
if(ch == 'n')
{
lastIndex = index;
state = State.An;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.An:
{
if(ch == 'd')
{
lastIndex = index;
state = State.And;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.And:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.And);
}
}
break;
case State.W:
{
if(ch == 'h')
{
lastIndex = index;
state = State.Wh;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Wh:
{
if(ch == 'i')
{
lastIndex = index;
state = State.Whi;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Whi:
{
if(ch == 'l')
{
lastIndex = index;
state = State.Whil;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Whil:
{
if(ch == 'e')
{
lastIndex = index;
state = State.While;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.While:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.While);
}
}
break;
case State.B:
{
if(ch == 'r')
{
lastIndex = index;
state = State.Br;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Br:
{
if(ch == 'e')
{
lastIndex = index;
state = State.Bre;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Bre:
{
if(ch == 'a')
{
lastIndex = index;
state = State.Brea;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Brea:
{
if(ch == 'k')
{
lastIndex = index;
state = State.Break;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Break:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Break);
}
}
break;
case State.G:
{
if(ch == 'o')
{
lastIndex = index;
state = State.Go;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Go:
{
if(ch == 't')
{
lastIndex = index;
state = State.Got;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Got:
{
if(ch == 'o')
{
lastIndex = index;
state = State.Goto;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Goto:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Goto);
}
}
break;
case State.R:
{
if(ch == 'e')
{
lastIndex = index;
state = State.Re;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Re:
{
if(ch == 't')
{
lastIndex = index;
state = State.Ret;
AppendDataChar(ch);
}
else if(ch == 'p')
{
lastIndex = index;
state = State.Rep;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Ret:
{
if(ch == 'u')
{
lastIndex = index;
state = State.Retu;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Retu:
{
if(ch == 'r')
{
lastIndex = index;
state = State.Retur;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Retur:
{
if(ch == 'n')
{
lastIndex = index;
state = State.Return;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Return:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Return);
}
}
break;
case State.Rep:
{
if(ch == 'e')
{
lastIndex = index;
state = State.Repe;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Repe:
{
if(ch == 'a')
{
lastIndex = index;
state = State.Repea;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Repea:
{
if(ch == 't')
{
lastIndex = index;
state = State.Repeat;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Repeat:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Repeat);
}
}
break;
case State.N:
{
if(ch == 'i')
{
lastIndex = index;
state = State.Ni;
AppendDataChar(ch);
}
else if(ch == 'o')
{
lastIndex = index;
state = State.No;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Ni:
{
if(ch == 'l')
{
lastIndex = index;
state = State.Nil;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Nil:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Nil);
}
}
break;
case State.No:
{
if(ch == 't')
{
lastIndex = index;
state = State.Not;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Not:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Not);
}
}
break;
case State.T:
{
if(ch == 'h')
{
lastIndex = index;
state = State.Th;
AppendDataChar(ch);
}
else if(ch == 'r')
{
lastIndex = index;
state = State.Tr;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Th:
{
if(ch == 'e')
{
lastIndex = index;
state = State.The;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.The:
{
if(ch == 'n')
{
lastIndex = index;
state = State.Then;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Then:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Then);
}
}
break;
case State.Tr:
{
if(ch == 'u')
{
lastIndex = index;
state = State.Tru;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Tru:
{
if(ch == 'e')
{
lastIndex = index;
state = State.True;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.True:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.True);
}
}
break;
case State.E:
{
if(ch == 'l')
{
lastIndex = index;
state = State.El;
AppendDataChar(ch);
}
else if(ch == 'n')
{
lastIndex = index;
state = State.En;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.El:
{
if(ch == 's')
{
lastIndex = index;
state = State.Els;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Els:
{
if(ch == 'e')
{
lastIndex = index;
state = State.Else;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Else:
{
if(ch == 'i')
{
lastIndex = index;
state = State.Elsei;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Else);
}
}
break;
case State.Elsei:
{
if(ch == 'f')
{
lastIndex = index;
state = State.Elseif;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Elseif:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Elseif);
}
}
break;
case State.En:
{
if(ch == 'd')
{
lastIndex = index;
state = State.End;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.End:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.End);
}
}
break;
case State.O:
{
if(ch == 'r')
{
lastIndex = index;
state = State.Or;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Or:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Or);
}
}
break;
case State.D:
{
if(ch == 'o')
{
lastIndex = index;
state = State.Do;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Do:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Do);
}
}
break;
case State.I:
{
if(ch == 'f')
{
lastIndex = index;
state = State.If;
AppendDataChar(ch);
}
else if(ch == 'n')
{
lastIndex = index;
state = State.In;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.In:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.In);
}
}
break;
case State.If:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.If);
}
}
break;
case State.F:
{
if(ch == 'u')
{
lastIndex = index;
state = State.Fu;
AppendDataChar(ch);
}
else if(ch == 'a')
{
lastIndex = index;
state = State.Fa;
AppendDataChar(ch);
}
else if(ch == 'o')
{
lastIndex = index;
state = State.Fo;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Fu:
{
if(ch == 'n')
{
lastIndex = index;
state = State.Fun;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Fun:
{
if(ch == 'c')
{
lastIndex = index;
state = State.Func;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Func:
{
if(ch == 't')
{
lastIndex = index;
state = State.Funct;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Funct:
{
if(ch == 'i')
{
lastIndex = index;
state = State.Functi;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Functi:
{
if(ch == 'o')
{
lastIndex = index;
state = State.Functio;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Functio:
{
if(ch == 'n')
{
lastIndex = index;
state = State.Function;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Function:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Function);
}
}
break;
case State.Fa:
{
if(ch == 'l')
{
lastIndex = index;
state = State.Fal;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Fal:
{
if(ch == 's')
{
lastIndex = index;
state = State.Fals;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Fals:
{
if(ch == 'e')
{
lastIndex = index;
state = State.False;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.False:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.False);
}
}
break;
case State.Fo:
{
if(ch == 'r')
{
lastIndex = index;
state = State.For;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.For:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.For);
}
}
break;
case State.L:
{
if(ch == 'o')
{
lastIndex = index;
state = State.Lo;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Lo:
{
if(ch == 'c')
{
lastIndex = index;
state = State.Loc;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Loc:
{
if(ch == 'a')
{
lastIndex = index;
state = State.Loca;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Loca:
{
if(ch == 'l')
{
lastIndex = index;
state = State.Local;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Local:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Local);
}
}
break;
case State.U:
{
if(ch == 'n')
{
lastIndex = index;
state = State.Un;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Un:
{
if(ch == 't')
{
lastIndex = index;
state = State.Unt;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Unt:
{
if(ch == 'i')
{
lastIndex = index;
state = State.Unti;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Unti:
{
if(ch == 'l')
{
lastIndex = index;
state = State.Until;
AppendDataChar(ch);
}
else if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
BacktrackNoClear(TokenType.Name);
}
}
break;
case State.Until:
{
if(char.IsAsciiLetterOrDigit(ch) || ch == '_')
{
lastIndex = index;
state = State.Name;
currentToken!.type = TokenType.Name;
AppendDataChar(ch);
}
else
{
Backtrack(TokenType.Until);
}
}
break;
default:
throw new NotImplementedException(state.ToString());
}
}
private enum State
{
Start,
Quote, SingleQuote, Name, Integer, Float, Zero,
A, B, D, E, F, G, I, L, N, O, R, T, U, W,
Plus, Minus, Star, Slash, Percent, Caret, Hash,
Ampersand, Tilde, Pipe, Lt, Gt, Equals, RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, StringStartLongBracket, StringWithLongBracket, StringEndLongBracket,
Colon, Semicolon, Comma, Dot,
An, Br, Do, El, En, Fa, Fo, Fu, Go, If, In, Lo, Ni, No, Or, Re, Th, Tr, Un, Wh,
LtLt, GtGt, SlashSlash, EqualsEquals, TildeEquals, LtEquals, GtEquals, ColonColon, DotDot,
SmallCommentStart, QuoteBackslash, SingleQuoteBackslash, String, HexNumberX, ExpNumber,
And, Bre, Els, End, Fal, For, Fun, Got, Loc, Nil, Not, Rep, Ret, The, Tru, Unt, Whi,
DotDotDot, HexNumber, QuoteBackslashZ, SingleQuoteBackslashZ, QuoteBackslashX, SingleQuoteBackslashX, QuoteBackslashXHex, SingleQuoteBackslashXHex,
SingleQuoteBackslashU, SingleQuoteBackslashUBracket, SingleQuoteBackslashUBracketHex,
QuoteBackslashU, QuoteBackslashUBracket, QuoteBackslashUBracketHex,
SmallComment, BigComment, BigCommentStartLongBracket, BigCommentEndLongBracket,
Brea, Else, Fals, Func, Goto, Loca, Repe, Retu, Then, True, Unti, Whil, HexExpNumber,
Break, Elsei, False, Funct, Local, Repea, Retur, Until, While,
Elseif, Functi, Repeat, Return,
Functio,
Function,
}
}
internal class Token(CodeRegion region, TokenType? type = null, Token.IData? data = null)
{
public CodeRegion region = region;
public IData? data = data;
public TokenType? type = type;
public interface IData { }
public class NumeralData(INumeral numeral) : IData
{
public INumeral numeral = numeral;
public override string ToString()
{
return $"NumeralData {numeral}";
}
}
public class StringData(string data) : IData
{
public string data = data;
public override string ToString()
{
return $"StringData \"{data}\"";
}
}
}
public enum TokenType
{
Name,
And, Break, Do, Else, Elseif, End,
False, For, Function, Goto, If, In,
Local, Nil, Not, Or, Repeat, Return,
Then, True, Until, While,
Plus, Minus, Star, Slash, Percent, Caret, Hash,
Ampersand, Tilde, Pipe, LtLt, GtGt, SlashSlash,
EqualsEquals, TildeEquals, LtEquals, GtEquals, Lt, Gt, Equals,
RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, ColonColon,
Semicolon, Colon, Comma, Dot, DotDot, DotDotDot,
Numeral,
StringLiteral,
}

83
build.zig Normal file
View File

@ -0,0 +1,83 @@
const std = @import("std");
// Although this function looks imperative, note that its job is to
// declaratively construct a build graph that will be executed by an external
// runner.
pub fn build(b: *std.Build) void {
// Standard target options allows the person running `zig build` to choose
// what target to build for. Here we do not override the defaults, which
// means any target is allowed, and the default is native. Other options
// for restricting supported target set are available.
const target = b.standardTargetOptions(.{});
// Standard optimization options allow the person running `zig build` to select
// between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall. Here we do not
// set a preferred release mode, allowing the user to decide how to optimize.
const optimize = b.standardOptimizeOption(.{});
const exe = b.addExecutable(.{
.name = "luaaaaah_zig",
// In this case the main source file is merely a path, however, in more
// complicated build scripts, this could be a generated file.
.root_source_file = .{ .path = "src/main.zig" },
.target = target,
.optimize = optimize,
});
exe.addModule("types", b.addModule("types", .{
.source_file = .{ .path = "src/types.zig" }
}));
exe.addModule("tokenizer", b.addModule("tokenizer", .{
.source_file = .{ .path = "src/tokenizer.zig" },
}));
exe.addModule("parser", b.addModule("parser", .{
.source_file = .{ .path = "src/parser.zig" },
}));
exe.addModule("treewalker", b.addModule("treewalker", .{
.source_file = .{ .path = "src/treewalker.zig" },
}));
// This declares intent for the executable to be installed into the
// standard location when the user invokes the "install" step (the default
// step when running `zig build`).
b.installArtifact(exe);
// This *creates* a Run step in the build graph, to be executed when another
// step is evaluated that depends on it. The next line below will establish
// such a dependency.
const run_cmd = b.addRunArtifact(exe);
// By making the run step depend on the install step, it will be run from the
// installation directory rather than directly from within the cache directory.
// This is not necessary, however, if the application depends on other installed
// files, this ensures they will be present and in the expected location.
run_cmd.step.dependOn(b.getInstallStep());
// This allows the user to pass arguments to the application in the build
// command itself, like this: `zig build run -- arg1 arg2 etc`
if (b.args) |args| {
run_cmd.addArgs(args);
}
// This creates a build step. It will be visible in the `zig build --help` menu,
// and can be selected like this: `zig build run`
// This will evaluate the `run` step rather than the default, which is "install".
const run_step = b.step("run", "Run the app");
run_step.dependOn(&run_cmd.step);
// Creates a step for unit testing. This only builds the test executable
// but does not run it.
const unit_tests = b.addTest(.{
.root_source_file = .{ .path = "src/main.zig" },
.target = target,
.optimize = optimize,
});
const run_unit_tests = b.addRunArtifact(unit_tests);
// Similar to creating the run step earlier, this exposes a `test` step to
// the `zig build --help` menu, providing a way for the user to request
// running the unit tests.
const test_step = b.step("test", "Run unit tests");
test_step.dependOn(&run_unit_tests.step);
}

9
luaaaaah.csproj
View File

@ -1,9 +0,0 @@
<Project
Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>disable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

40
src/main.zig Normal file
View File

@ -0,0 +1,40 @@
const std = @import("std");
const tokenize = @import("tokenizer.zig").tokenize;
const parse = @import("parser.zig").parse;
const treewalk = @import("treewalker.zig").interpret;
pub fn main() !void
{
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
defer _ = gpa.deinit();
const allocator = gpa.allocator();
const args = try std.process.argsAlloc(allocator);
defer std.process.argsFree(allocator, args);
const file = try std.fs.cwd().openFile(args[1], .{});
defer file.close();
const content = try file.readToEndAlloc(allocator, 13000);
defer allocator.free(content);
const tokens = try tokenize(content, allocator);
defer
{
var i: usize = 0;
while(i < tokens.len)
{
switch(tokens[i].tokenData)
{
.string => |*data|
{
allocator.free(data.*);
},
else => {}
}
i += 1;
}
allocator.free(tokens);
}
var parserAllocator = std.heap.ArenaAllocator.init(std.heap.page_allocator);
defer parserAllocator.deinit();
const root = try parse(tokens, &parserAllocator);
try treewalk(root, allocator);
}

2572
src/parser.zig Normal file
View File

@ -0,0 +1,2572 @@
const Token = @import("tokenizer.zig").Token;
const TokenType = @import("tokenizer.zig").TokenType;
const std = @import("std");
const types = @import("types.zig");
const CodeRegion = @import("types.zig").CodeRegion;
pub const ChunkNode = struct
{
block: BlockNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
pub fn dump(self: *const ChunkNode, indent: usize) void
{
std.debug.print("ChunkNode ({} - {}):\n", .{self.startRegion, self.endRegion});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("block: ", .{});
self.block.dump(indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const BlockNode = struct
{
stats: std.ArrayList(StatNode),
retstat: ?RetstatNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const BlockNode, indent: usize) void
{
std.debug.print("BlockNode:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("stats:\n", .{});
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("[\n", .{});
for(self.stats.items) |stat|
{
for (0..indent + 2) |_|
{
std.debug.print("\t", .{});
}
dumpStatNode(stat, indent + 2);
}
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("]\n", .{});
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("retstat: ", .{});
if(self.retstat == null)
{
std.debug.print("null\n", .{});
}
else
{
self.retstat.?.dump(indent + 1);
}
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const RetstatNode = struct
{
values: ?ExplistNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const RetstatNode, indent: usize) void
{
std.debug.print("Retstat Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("values: ", .{});
if(self.values == null)
{
std.debug.print("null\n", .{});
}
else
{
self.values.?.dump(indent + 1);
}
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const StatNode = union(enum)
{
Semicolon,
Assignment: AssignmentNode,
Functioncall: FunctioncallNode,
Label: []u8,
Break,
Goto: []u8,
Do: BlockNode,
While: WhileNode,
Repeat: RepeatNode,
If: IfNode,
ForNumerical: ForNumericalNode,
ForGeneric: ForGenericNode,
Function: FunctionNode,
LocalFunction: LocalFunctionNode,
Local: LocalNode,
};
fn dumpStatNode(stat: StatNode, indent: usize) void
{
switch(stat)
{
.Semicolon => std.debug.print("Semicolon\n", .{}),
.Assignment => |*node| node.dump(indent),
.Functioncall => |*node| node.dump(indent),
.Label => |*label| std.debug.print("Label: '{s}'\n", .{label.*}),
.Break => std.debug.print("Break\n", .{}),
.Goto => |*label| std.debug.print("Goto: '{s}'\n", .{label.*}),
.Do => |*node| node.dump(indent),
.While => |*node| node.dump(indent),
.Repeat => |*node| node.dump(indent),
.If => |*node| node.dump(indent),
.ForNumerical => |*node| node.dump(indent),
.ForGeneric => |*node| node.dump(indent),
.Function => |*node| node.dump(indent),
.LocalFunction => |*node| node.dump(indent),
.Local => |*node| node.dump(indent),
}
}
pub const AssignmentNode = struct
{
lhs: VarlistNode, rhs: ExplistNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const AssignmentNode, indent: usize) void
{
std.debug.print("Assignment Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("lhs: ", .{});
self.lhs.dump(indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("rhs: ", .{});
self.rhs.dump(indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const WhileNode = struct
{
condition: ExpNode,
body: BlockNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const WhileNode, indent: usize) void
{
std.debug.print("While Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("condition: ", .{});
dumpExpNode(self.condition, indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("body: ", .{});
self.body.dump(indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const RepeatNode = struct
{
condition: ExpNode,
body: BlockNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const RepeatNode, indent: usize) void
{
std.debug.print("Repeat Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("condition: ", .{});
dumpExpNode(self.condition, indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("body: ", .{});
self.body.dump(indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const IfNode = struct
{
condition: ExpNode,
body: BlockNode,
elseifs: std.ArrayList(ElseifNode),
else_: ?BlockNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const IfNode, indent: usize) void
{
std.debug.print("If Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("condition: ", .{});
dumpExpNode(self.condition, indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("body: ", .{});
self.body.dump(indent + 1);
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("elseifs:\n", .{});
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("[\n", .{});
for(self.elseifs.items) |elseif|
{
for (0..indent + 2) |_|
{
std.debug.print("\t", .{});
}
elseif.dump(indent + 2);
}
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("]\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("else: ", .{});
if(self.else_ == null)
{
std.debug.print("null\n", .{});
}
else
{
self.else_.?.dump(indent + 1);
}
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const ForNumericalNode = struct
{
variable: []u8,
start: ExpNode,
end: ExpNode,
change: ?ExpNode,
body: BlockNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const ForNumericalNode, indent: usize) void
{
std.debug.print("For Numerical Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("variable: '{s}'\n", .{self.variable});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("start: ", .{});
dumpExpNode(self.start, indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("end: ", .{});
dumpExpNode(self.end, indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("change: ", .{});
if(self.change == null)
{
std.debug.print("null\n", .{});
}
else
{
dumpExpNode(self.start, indent + 1);
}
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("body: ", .{});
self.body.dump(indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const ForGenericNode = struct
{
vars: std.ArrayList([]u8),
exps: ExplistNode,
body: BlockNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const ForGenericNode, indent: usize) void
{
std.debug.print("For Generic Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("vars:\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("[\n", .{});
for(self.vars.items) |v|
{
for (0..(indent + 2)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("'{s}'\n", .{v});
}
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("]\n", .{});
std.debug.print("exps: ", .{});
self.exps.dump(indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("body: ", .{});
self.body.dump(indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const FunctionNode = struct
{
name: FuncnameNode,
body: FuncbodyNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const FunctionNode, indent: usize) void
{
std.debug.print("Function Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("name: ", .{});
self.name.dump(indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("body: ", .{});
self.body.dump(indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const LocalFunctionNode = struct
{
name: []u8,
body: FuncbodyNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const LocalFunctionNode, indent: usize) void
{
std.debug.print("Local Function Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("name: '{s}'\n", .{self.name});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("body: ", .{});
self.body.dump(indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const LocalNode = struct
{
attnames: AttnamelistNode,
values: ?ExplistNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const LocalNode, indent: usize) void
{
std.debug.print("Local Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("attnames: ", .{});
self.attnames.dump(indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("values: ", .{});
if(self.values == null)
{
std.debug.print("null\n", .{});
}
else
{
self.values.?.dump(indent + 1);
}
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const FunctioncallNode = struct
{
function: SuffixexpNode,
objectArg: ?[]u8,
args: ArgsNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const FunctioncallNode, indent: usize) void
{
std.debug.print("Functioncall Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("function: ", .{});
dumpSuffixExpNode(self.function, indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("object arg: ", .{});
if(self.objectArg == null)
{
std.debug.print("null\n", .{});
}
else
{
std.debug.print("'{s}'\n", .{self.objectArg.?});
}
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("args: ", .{});
dumpArgsNode(self.args, indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const VarlistNode = struct
{
vars: std.ArrayList(VarNode),
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const VarlistNode, indent: usize) void
{
std.debug.print("Varlist Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("vars:\n", .{});
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("[\n", .{});
for(self.vars.items) |item|
{
for (0..indent + 2) |_|
{
std.debug.print("\t", .{});
}
dumpVarNode(item, indent + 2);
}
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("]\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const ExplistNode = struct
{
exps: std.ArrayList(ExpNode),
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const ExplistNode, indent: usize) void
{
std.debug.print("Explist Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("exps:\n", .{});
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("[\n", .{});
for(self.exps.items) |item|
{
for (0..indent + 2) |_|
{
std.debug.print("\t", .{});
}
dumpExpNode(item, indent + 2);
}
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("]\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const ExpNode = union(enum)
{
Nil,
False,
True,
Numeral: types.Numeral,
LiteralString: []u8,
Varargs,
Functiondef: FuncbodyNode,
Suffixexp: *SuffixexpNode,
Tableconstructor: TableconstructorNode,
Unop: UnopNode,
Binop: *BinopNode,
};
fn dumpExpNode(expNode: ExpNode, indent: usize) void
{
switch(expNode)
{
.Nil => std.debug.print("Nil\n", .{}),
.False => std.debug.print("False\n", .{}),
.True => std.debug.print("True\n", .{}),
.Numeral => |*numeral| std.debug.print("{}\n", .{numeral.*}),
.LiteralString => |*string| std.debug.print("LiteralString: '{s}'\n", .{string.*}),
.Varargs => std.debug.print("Varargs", .{}),
.Functiondef => |*node| node.dump(indent),
.Suffixexp => |*node| dumpSuffixExpNode(node.*.*, indent),
.Tableconstructor => |*node| node.dump(indent),
.Unop => |*node| node.dump(indent),
.Binop => |*node| node.*.dump(indent),
}
}
pub const UnopNode = struct
{
unopType: UnopType,
exp: *ExpNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const UnopNode, indent: usize) void
{
std.debug.print("Unop Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("unop type: {}\n", .{self.unopType});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("exp: ", .{});
dumpExpNode(self.exp.*, indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const BinopNode = struct
{
lhs: ExpNode,
op: BinopType,
rhs: ExpNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const BinopNode, indent: usize) void
{
std.debug.print("Binop Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("lhs: ", .{});
dumpExpNode(self.lhs, indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("op: {}\n", .{self.op});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("rhs: ", .{});
dumpExpNode(self.rhs, indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const ElseifNode = struct
{
condition: ExpNode,
body: BlockNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const ElseifNode, indent: usize) void
{
std.debug.print("Elseif Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("condition: ", .{});
dumpExpNode(self.condition, indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("body: ", .{});
self.body.dump(indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const FuncnameNode = struct
{
name: []u8,
dottedNames: std.ArrayList([]u8),
firstArg: ?[]u8,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const FuncnameNode, indent: usize) void
{
std.debug.print("Funcname Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("name: '{s}'\n", .{self.name});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("dottedNames:\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("[\n", .{});
for(self.dottedNames.items) |dottedName|
{
for (0..(indent + 2)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("'{s}'\n", .{dottedName});
}
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("]\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("firstArg: ", .{});
if(self.firstArg == null)
{
std.debug.print("null\n", .{});
}
else
{
std.debug.print("'{s}'\n", .{self.firstArg.?});
}
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const FuncbodyNode = struct
{
pars: ?ParlistNode,
body: BlockNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const FuncbodyNode, indent: usize) void
{
std.debug.print("Funcbody Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("pars: ", .{});
if(self.pars == null)
{
std.debug.print("null\n", .{});
}
else
{
self.pars.?.dump(indent + 1);
}
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
self.body.dump(indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const AttnamelistNode = struct
{
attnames: std.ArrayList(AttnameNode),
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const AttnamelistNode, indent: usize) void
{
std.debug.print("Attnamelist Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("attNames:\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("[\n", .{});
for(self.attnames.items) |attNames|
{
for (0..(indent + 2)) |_|
{
std.debug.print("\t", .{});
}
attNames.dump(indent + 2);
}
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("]\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const AttnameNode = struct
{
name: []u8,
attribute: ?[]u8,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const AttnameNode, indent: usize) void
{
std.debug.print("Funcname Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("name: '{s}'\n", .{self.name});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("attribute: ", .{});
if(self.attribute == null)
{
std.debug.print("null\n", .{});
}
else
{
std.debug.print("'{s}'\n", .{self.attribute.?});
}
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const SuffixexpNode = union(enum)
{
Normal: NormalSuffixNode,
Functioncall: *FunctioncallNode,
};
fn dumpSuffixExpNode(suffixexpNode: SuffixexpNode, indent: usize) void
{
switch(suffixexpNode)
{
.Normal => |*node| node.dump(indent),
.Functioncall => |*node| node.*.dump(indent),
}
}
pub const ArgsNode = union(enum)
{
Bracketed: ?ExplistNode,
Tableconstructor: TableconstructorNode,
Literal: []u8,
};
fn dumpArgsNode(argsNode: ArgsNode, indent: usize) void
{
switch(argsNode)
{
.Bracketed => |*name|
{
if(name.* == null)
{
std.debug.print("null\n", .{});
}
else
{
name.*.?.dump(indent);
}
},
.Tableconstructor => |*node| node.dump(indent),
.Literal => |*string| std.debug.print("Literal: '{s}'\n", .{string}),
}
}
pub const VarNode = union(enum)
{
Name: []u8,
Indexed: IndexedVarNode,
Member: MemberVarNode,
};
fn dumpVarNode(varNode: VarNode, indent: usize) void
{
switch(varNode)
{
.Name => |*name| std.debug.print("Name: '{s}'\n", .{name.*}),
.Indexed => |*node| node.dump(indent),
.Member => |*node| node.dump(indent),
}
}
pub const IndexedVarNode = struct
{
value: SuffixexpNode,
index: ExpNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const IndexedVarNode, indent: usize) void
{
std.debug.print("Indexed Var Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("value: ", .{});
dumpSuffixExpNode(self.value, indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("index: ", .{});
dumpExpNode(self.index, indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const MemberVarNode = struct
{
value: SuffixexpNode,
name: []u8,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const MemberVarNode, indent: usize) void
{
std.debug.print("Member Var Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("value: ", .{});
dumpSuffixExpNode(self.value, indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("name: '{s}'\n", .{self.name});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const TableconstructorNode = struct
{
exps: ?FieldlistNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const TableconstructorNode, indent: usize) void
{
std.debug.print("Tableconstructor Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("exps: ", .{});
if(self.exps == null)
{
std.debug.print("null\n", .{});
}
else
{
self.exps.?.dump(indent + 1);
}
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const UnopType = enum
{
Minus, LogicalNot, Length, BinaryNot,
};
pub const BinopType = enum
{
LogicalOr,
LocicalAnd,
Lt, Gt, LtEquals, GtEquals, NotEquals, Equals,
BinaryOr,
BinaryNot,
BinaryAnd,
Shl, Shr,
Concat,
Add, Sub,
Mul, Div, IntDiv, Mod,
Exp,
};
pub const ParlistNode = struct
{
names: std.ArrayList([]u8),
hasVarargs: bool,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const ParlistNode, indent: usize) void
{
std.debug.print("Parlist Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("names:\n", .{});
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("[\n", .{});
for(self.names.items) |name|
{
for (0..indent + 2) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("'{s}'\n", .{name});
}
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("]\n", .{});
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("has Varargs: {}\n", .{self.hasVarargs});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const NormalSuffixNode = struct
{
firstPart: SuffixexpFirstPart,
suffixes: std.ArrayList(SuffixexpSuffix),
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const NormalSuffixNode, indent: usize) void
{
std.debug.print("Normal Suffix Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("First Part: ", .{});
dumpSuffixExpFirstPart(self.firstPart, indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("Suffixes:\n", .{});
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("[\n", .{});
for(self.suffixes.items) |suffix|
{
for (0..(indent + 2)) |_|
{
std.debug.print("\t", .{});
}
dumpSuffixSuffix(suffix, indent + 2);
}
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("]\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const SuffixexpFirstPart = union(enum)
{
Name: []u8,
BracketedExpr: ExpNode,
};
fn dumpSuffixExpFirstPart(suffixexpFirstPart: SuffixexpFirstPart, indent: usize) void
{
switch(suffixexpFirstPart)
{
.Name => |*name| std.debug.print("Name: '{s}'\n", .{name.*}),
.BracketedExpr => |*node| dumpExpNode(node.*, indent),
}
}
pub const SuffixexpSuffix = union(enum)
{
Dot: []u8,
Indexed: ExpNode,
Args: ArgsNode,
ArgsFirstArg: ArgsFirstArgNode,
};
fn dumpSuffixSuffix(suffixexpSuffix: SuffixexpSuffix, indent: usize) void
{
switch(suffixexpSuffix)
{
.Dot => |*name| std.debug.print("Dot: '{s}'\n", .{name.*}),
.Indexed => |*node| dumpExpNode(node.*, indent),
.Args => |*node| dumpArgsNode(node.*, indent),
.ArgsFirstArg => |*node| node.dump(indent),
}
}
pub const ArgsFirstArgNode = struct
{
name: []u8,
rest: ArgsNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const ArgsFirstArgNode, indent: usize) void
{
std.debug.print("Args First Arg Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("name: '{s}'\n", .{self.name});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("rest: ", .{});
dumpArgsNode(self.rest, indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const FieldlistNode = struct
{
exps: std.ArrayList(FieldNode),
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const FieldlistNode, indent: usize) void
{
std.debug.print("Fieldlist Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("exps: ", .{});
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("[\n", .{});
for(self.exps.items) |exp|
{
for (0..(indent + 2)) |_|
{
std.debug.print("\t", .{});
}
dumpFieldNode(exp, indent + 2);
}
for (0..indent + 1) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("]\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub const FieldNode = union(enum)
{
IndexedAssignment: IndexedAssignmentNode,
Assignment: FieldAssignmentNode,
Exp: ExpNode,
};
pub const FieldAssignmentNode = struct
{
lhs: []u8,
rhs: ExpNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const FieldAssignmentNode, indent: usize) void
{
std.debug.print("Field Assignment Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("lhs: {s}\n", .{self.lhs});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("rhs: ", .{});
dumpExpNode(self.rhs, indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
fn dumpFieldNode(fieldNode: FieldNode, indent: usize) void
{
switch(fieldNode)
{
.IndexedAssignment => |*node| node.dump(indent),
.Assignment => |*node| node.dump(indent),
.Exp => |*node| dumpExpNode(node.*, indent),
}
}
pub const IndexedAssignmentNode = struct
{
index: ExpNode,
rhs: ExpNode,
startRegion: CodeRegion,
endRegion: CodeRegion,
fn dump(self: *const IndexedAssignmentNode, indent: usize) void
{
std.debug.print("Indexed Assignment Node:\n", .{});
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("{{\n", .{});
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("index: ", .{});
dumpExpNode(self.index, indent + 1);
for (0..(indent + 1)) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("rhs: ", .{});
dumpExpNode(self.rhs, indent + 1);
for (0..indent) |_|
{
std.debug.print("\t", .{});
}
std.debug.print("}}\n", .{});
}
};
pub fn parse(tokens: []Token, allocator: *std.heap.ArenaAllocator) !ChunkNode
{
var i: usize = 0;
const maybeParsedChunk = parseChunk(tokens, &i, allocator) catch |err|
{
//std.debug.print("{any}: data: {any}, type: {any}\n", .{tokens[i].region, tokens[i].tokenData, tokens[i].tokenType});
return err;
};
return maybeParsedChunk;
}
const ParserError = error
{
NotImplemented,
ReachedEOFParsing,
ReachedEOFExpectedNameForGoto,
ExpectedNameForGoto,
MissingEndForDoBlock,
MissingDoAfterWhileCondition,
MissingEndForWhileBody,
ExpectedUntilAfterRepeatBody,
ExpectedThenAfterIfCondition,
ReachedEOFAfterIfBody,
ExpectedThenAfterElseifCondition,
ReachedEOFAfterElseifs,
ExpectedEndClosingIf,
ExpectedNameAfterFor,
ExpectedCommaAfterForEqStartValue,
ReachedEOFAfterForEqEndValue,
ExpectedDoAfterForEqHead,
ExpectedAnotherNameInForInNamelist,
ReachedEOFAfterNameInForInNamelist,
ExpectedInAfterForInNamelist,
ExpectedDoAfterForInExplist,
ExpectedEndAfterForInBody,
ExpectedEndAfterForEqBody,
UnexpectedTokenAfterFirstNameInFor,
ReachedEOFInLocal,
ExpectedLocalFunctionName,
ExpectedNameAfterDoubleColonInLabelDeclaration,
ExpectedDoubleColonAfterNameInLabelDeclaration,
ExpectedFunctioncall,
ExpectedEqAfterAssignmentVarList,
ReachedEOFInSuffixExp,
ExpectedRoundClosedClosingBracketedPrimaryExp,
UnexpectedTokenAsFirstPartOfSuffixExp,
ExpectedNameInDottedSuffixExp,
ExpectedSquareClosedClosingIndexedSuffixExp,
ExpectedNameInArgsFirstArgSuffixExp,
ExpectedDotOrIndexedSuffixWhenConvertingSuffixExpToVar,
ReachedEOFExpectedPrimaryExpression,
ReachedEOFInArgs,
ReachedEOFInBracketedArgs,
ExpectedRoundClosedClosingBracketedArgs,
UnexpectedTokenInArgs,
NoPrecedenceForOperator,
NoBinopTypeForOperator,
ExpectednameInAttribName,
ExpectedAttributeInAttrib,
ExpectedGtInAttrib,
ExpectedFuncname,
ExpectedNameInDottedFuncname,
ExpectedNameOfFirstArgInFuncname,
ExpectedRoundOpenStartingFuncbody,
ReachedEOFInFuncbodyParlist,
ExpectedRoundClosedClosingFuncbodyParlist,
ExpectedEndClosingFuncbody,
ReachedEOFInParlist,
ExpectedNameStartingParlist,
ReachedEOFInParlistNameList,
UnexpectedTokenInParlistNameList,
ExpectedReturnStartingRetstat,
ExpectedCurlyOpenOpeningTableconstructor,
ExpectedCurlyClosedClosingTableconstructor,
ReachedEOFInField,
ExpectedSquareClosedClosingIndexedField,
ExpectedEqualsInIndexedFieldExpression,
OutOfMemory,
};
fn parseChunk(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ChunkNode
{
const block = try parseBlock(tokens, i, allocator);
return ChunkNode { .block = block, .startRegion = block.startRegion, .endRegion = block.endRegion };
}
fn parseBlock(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) ParserError!BlockNode
{
var ret = BlockNode { .stats = std.ArrayList(StatNode).init(allocator.*.allocator()), .retstat = null, .startRegion = tokens[i.*].region, .endRegion = tokens[i.*].region };
while(i.* < tokens.len and
tokens[i.*].tokenType != TokenType.Return and
tokens[i.*].tokenType != TokenType.End and
tokens[i.*].tokenType != TokenType.Elseif and
tokens[i.*].tokenType != TokenType.Else
)
{
try ret.stats.append(try parseStat(tokens, i, allocator));
}
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Return)
{
ret.retstat = try parseRetstat(tokens, i, allocator);
}
ret.endRegion = if(i.* - 1 < tokens.len) tokens[i.* - 1].region else tokens[tokens.len - 1].region;
return ret;
}
fn parseStat(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !StatNode
{
if(i.* >= tokens.len)
{
return error.ReachedEOFParsing;
}
switch(tokens[i.*].tokenType)
{
TokenType.Semicolon =>
{
i.* += 1;
return StatNode.Semicolon;
},
TokenType.Break =>
{
i.* += 1;
return StatNode.Break;
},
TokenType.Goto =>
{
i.* += 1;
if(i.* >= tokens.len)
{
return error.ReachedEOFExpectedNameForGoto;
}
if(tokens[i.*].tokenType == TokenType.Name)
{
const name = tokens[i.*].tokenData.string;
i.* += 1;
return StatNode { .Goto = name };
}
return error.ExpectedNameForGoto;
},
TokenType.Do =>
{
i.* += 1;
const body = try parseBlock(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
{
return error.MissingEndForDoBlock;
}
i.* += 1;
return StatNode { .Do = body };
},
TokenType.While =>
{
const startRegion = tokens[i.*].region;
i.* += 1;
const condition = try parseExp(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Do)
{
return error.MissingDoAfterWhileCondition;
}
const body = try parseBlock(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
{
return error.MissingEndForWhileBody;
}
const endRegion = tokens[i.*].region;
i.* += 1;
return StatNode { .While = WhileNode { .body = body, .condition = condition, .startRegion = startRegion, .endRegion = endRegion } };
},
TokenType.Repeat =>
{
const startRegion = tokens[i.*].region;
i.* += 1;
const body = try parseBlock(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Until)
{
return error.ExpectedUntilAfterRepeatBody;
}
const endRegion = tokens[i.*].region;
i.* += 1;
return StatNode { .Repeat = RepeatNode { .body = body, .condition = try parseExp(tokens, i, allocator), .startRegion = startRegion, .endRegion = endRegion } };
},
TokenType.If =>
{
const startRegion = tokens[i.*].region;
i.* += 1;
const condition = try parseExp(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Then)
{
return error.ExpectedThenAfterIfCondition;
}
i.* += 1;
const body = try parseBlock(tokens, i, allocator);
if(i.* >= tokens.len)
{
return error.ReachedEOFAfterIfBody;
}
var ifNode = IfNode { .body = body, .condition = condition, .elseifs = std.ArrayList(ElseifNode).init(allocator.*.allocator()), .else_ = null, .startRegion = startRegion, .endRegion = startRegion };
while(tokens[i.*].tokenType == TokenType.Elseif)
{
i.* += 1;
const elseifCondition = try parseExp(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Then)
{
return error.ExpectedThenAfterElseifCondition;
}
const endRegion = tokens[i.*].region;
i.* += 1;
try ifNode.elseifs.append(ElseifNode { .body = try parseBlock(tokens, i, allocator), .condition = elseifCondition, .startRegion = startRegion, .endRegion = endRegion });
}
if(i.* >= tokens.len)
{
return error.ReachedEOFAfterElseifs;
}
if(tokens[i.*].tokenType == TokenType.Else)
{
i.* += 1;
ifNode.else_ = try parseBlock(tokens, i, allocator);
}
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
{
return error.ExpectedEndClosingIf;
}
ifNode.endRegion = tokens[i.*].region;
i.* += 1;
return StatNode { .If = ifNode };
},
TokenType.For =>
{
const startRegion = tokens[i.*].region;
i.* += 1;
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
{
return error.ExpectedNameAfterFor;
}
const variable = tokens[i.*].tokenData.string;
i.* += 1;
switch(tokens[i.*].tokenType)
{
TokenType.Equals =>
{
i.* += 1;
const start = try parseExp(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Comma)
{
return error.ExpectedCommaAfterForEqStartValue;
}
i.* += 1;
const end = try parseExp(tokens, i, allocator);
if(i.* >= tokens.len)
{
return error.ReachedEOFAfterForEqEndValue;
}
var change: ?ExpNode = null;
if(tokens[i.*].tokenType == TokenType.Comma)
{
i.* += 1;
change = try parseExp(tokens, i, allocator);
}
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Do)
{
return error.ExpectedDoAfterForEqHead;
}
i.* += 1;
const body = try parseBlock(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
{
return error.ExpectedEndAfterForEqBody;
}
const endRegion = tokens[i.*].region;
i.* += 1;
return StatNode { .ForNumerical = ForNumericalNode { .variable = variable, .start = start, .end = end, .change = change, .body = body, .startRegion = startRegion, .endRegion = endRegion } };
},
TokenType.Comma =>
{
var names = std.ArrayList([]u8).init(allocator.*.allocator());
try names.append(variable);
while(tokens[i.*].tokenType == TokenType.Comma)
{
i.* += 1;
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
{
return error.ExpectedAnotherNameInForInNamelist;
}
try names.append(tokens[i.*].tokenData.string);
i.* += 1;
if(i.* >= tokens.len)
{
return error.ReachedEOFAfterNameInForInNamelist;
}
}
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.In)
{
return error.ExpectedInAfterForInNamelist;
}
i.* += 1;
const exps = try parseExplist(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Do)
{
return error.ExpectedDoAfterForInExplist;
}
i.* += 1;
const body = try parseBlock(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
{
return error.ExpectedEndAfterForInBody;
}
const endRegion = tokens[i.*].region;
i.* += 1;
return StatNode { .ForGeneric = ForGenericNode { .vars = names, .exps = exps, .body = body, .startRegion = startRegion, .endRegion = endRegion } };
},
TokenType.In =>
{
i.* += 1;
const exps = try parseExplist(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Do)
{
return error.ExpectedDoAfterForInExplist;
}
i.* += 1;
const body = try parseBlock(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
{
return error.ExpectedEndAfterForInBody;
}
const endRegion = tokens[i.*].region;
i.* += 1;
var names = try std.ArrayList([]u8).initCapacity(allocator.allocator(), 1);
try names.insert(0, variable);
return StatNode { .ForGeneric = ForGenericNode { .vars = names, .exps = exps, .body = body, .startRegion = startRegion, .endRegion = endRegion } };
},
else => return error.UnexpectedTokenAfterFirstNameInFor,
}
},
TokenType.Function =>
{
const startRegion = tokens[i.*].region;
i.* += 1;
const name = try parseFuncname(tokens, i, allocator);
const body = try parseFuncbody(tokens, i, allocator);
return StatNode { .Function = FunctionNode { .name = name, .body = body, .startRegion = startRegion, .endRegion = body.endRegion } };
},
TokenType.Local =>
{
const startRegion = tokens[i.*].region;
i.* += 1;
if(i.* >= tokens.len)
{
return error.ReachedEOFInLocal;
}
if(tokens[i.*].tokenType == TokenType.Function)
{
i.* += 1;
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
{
return error.ExpectedLocalFunctionName;
}
const name = tokens[i.*].tokenData.string;
const endRegion = tokens[i.*].region;
i.* += 1;
return StatNode { .LocalFunction = LocalFunctionNode { .name = name, .body = try parseFuncbody(tokens, i, allocator), .startRegion = startRegion, .endRegion = endRegion } };
}
else
{
var ret = LocalNode { .attnames = try parseAttnamelist(tokens, i, allocator), .values = null, .startRegion = startRegion, .endRegion = startRegion };
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Equals)
{
i.* += 1;
ret.values = try parseExplist(tokens, i, allocator);
ret.endRegion = ret.values.?.endRegion;
}
return StatNode { .Local = ret };
}
},
TokenType.ColonColon =>
{
i.* += 1;
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
{
return error.ExpectedNameAfterDoubleColonInLabelDeclaration;
}
const name = tokens[i.*].tokenData.string;
i.* += 1;
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.ColonColon)
{
return error.ExpectedDoubleColonAfterNameInLabelDeclaration;
}
i.* += 1;
return StatNode { .Label = name };
},
TokenType.Name, TokenType.RoundOpen =>
{
const startRegion = tokens[i.*].region;
const suffixExp = try parseSuffixExp(tokens, i, allocator);
if(i.* >= tokens.len)
{
switch(suffixExp)
{
.Normal => return error.ExpectedFunctioncall,
.Functioncall => |functioncall| return StatNode { .Functioncall = functioncall.* },
}
}
else
{
switch(tokens[i.*].tokenType)
{
TokenType.Equals =>
{
const endRegion = tokens[i.*].region;
i.* += 1;
var lhs = std.ArrayList(VarNode).init(allocator.allocator());
try lhs.append(try suffixExpToVar(suffixExp, startRegion, endRegion));
const rhs = try parseExplist(tokens, i, allocator);
return StatNode { .Assignment = AssignmentNode { .lhs = VarlistNode { .vars = lhs, .startRegion = endRegion, .endRegion = tokens[@min(i.*, tokens.len) - 1].region }, .rhs = rhs, .startRegion = startRegion, .endRegion = rhs.endRegion } };
},
TokenType.Comma =>
{
var varlistNode = VarlistNode { .vars = std.ArrayList(VarNode).init(allocator.allocator()), .startRegion = startRegion, .endRegion = startRegion };
try varlistNode.vars.append(try suffixExpToVar(suffixExp, startRegion, tokens[@min(i.*, tokens.len) - 1].region));
while(tokens[i.*].tokenType == TokenType.Comma)
{
i.* += 1;
try varlistNode.vars.append(try parseVar(tokens, i, allocator));
}
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Equals)
{
return error.ExpectedEqAfterAssignmentVarList;
}
varlistNode.endRegion = tokens[i.*].region;
i.* += 1;
const rhs = try parseExplist(tokens, i, allocator);
return StatNode { .Assignment = AssignmentNode { .lhs = varlistNode, .rhs = rhs, .startRegion = startRegion, .endRegion = rhs.endRegion } };
},
else =>
{
switch(suffixExp)
{
.Normal => return error.ExpectedFunctioncall,
.Functioncall => |functioncall| return StatNode { .Functioncall = functioncall.* },
}
}
}
}
},
else =>
{
std.debug.print("{}\n", .{tokens[i.*]});
return error.NotImplemented;
}
}
}
fn parseRetstat(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !RetstatNode
{
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Return)
{
return error.ExpectedReturnStartingRetstat;
}
const startRegion = tokens[i.*].region;
i.* += 1;
if(i.* >= tokens.len or
tokens[i.*].tokenType == TokenType.Semicolon or tokens[i.*].tokenType == TokenType.Else or tokens[i.*].tokenType == TokenType.Elseif or tokens[i.*].tokenType == TokenType.End)
{
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Semicolon)
{
i.* += 1;
}
return RetstatNode { .values = null, .startRegion = startRegion, .endRegion = tokens[@min(i.*, tokens.len) - 1].region };
}
const values = try parseExplist(tokens, i, allocator);
var endRegion = values.endRegion;
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Semicolon)
{
endRegion = tokens[i.*].region;
i.* += 1;
}
return RetstatNode { .values = values, .startRegion = startRegion, .endRegion = endRegion };
}
fn parseExp(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) ParserError!ExpNode
{
const lhs = try parseExpPrimary(tokens, i, allocator);
return parseExpPrecedence(tokens, i, allocator, lhs, 0);
}
fn parseExpPrimary(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ExpNode
{
if(i.* >= tokens.len)
{
return error.ReachedEOFExpectedPrimaryExpression;
}
const startRegion = tokens[i.*].region;
switch(tokens[i.*].tokenType)
{
TokenType.Nil =>
{
i.* += 1;
return ExpNode.Nil;
},
TokenType.True =>
{
i.* += 1;
return ExpNode.True;
},
TokenType.False =>
{
i.* += 1;
return ExpNode.False;
},
TokenType.Numeral =>
{
const numeral = tokens[i.*].tokenData.numeral;
i.* += 1;
return ExpNode { .Numeral = numeral };
},
TokenType.StringLiteral =>
{
const string = tokens[i.*].tokenData.string;
i.* += 1;
return ExpNode { .LiteralString = string };
},
TokenType.DotDotDot =>
{
i.* += 1;
return ExpNode.Varargs;
},
TokenType.Function =>
{
i.* += 1;
return ExpNode { .Functiondef = try parseFuncbody(tokens, i, allocator) };
},
TokenType.CurlyOpen => return ExpNode { .Tableconstructor = try parseTableconstructor(tokens, i, allocator) },
TokenType.Minus =>
{
i.* += 1;
const unop = try allocator.allocator().create(ExpNode);
unop.* = try parseExp(tokens, i, allocator);
const endRegion = tokens[@min(i.*, tokens.len) - 1].region;
return ExpNode { .Unop = UnopNode { .unopType = UnopType.Minus, .exp = unop, .startRegion = startRegion, .endRegion = endRegion } };
},
TokenType.Hash =>
{
i.* += 1;
const unop = try allocator.allocator().create(ExpNode);
unop.* = try parseExp(tokens, i, allocator);
const endRegion = tokens[@min(i.*, tokens.len) - 1].region;
return ExpNode { .Unop = UnopNode { .unopType = UnopType.Length, .exp = unop, .startRegion = startRegion, .endRegion = endRegion } };
},
TokenType.Not =>
{
i.* += 1;
const unop = try allocator.allocator().create(ExpNode);
unop.* = try parseExp(tokens, i, allocator);
const endRegion = tokens[@min(i.*, tokens.len) - 1].region;
return ExpNode { .Unop = UnopNode { .unopType = UnopType.LogicalNot, .exp = unop, .startRegion = startRegion, .endRegion = endRegion } };
},
TokenType.Tilde =>
{
i.* += 1;
const unop = try allocator.allocator().create(ExpNode);
unop.* = try parseExp(tokens, i, allocator);
const endRegion = tokens[@min(i.*, tokens.len) - 1].region;
return ExpNode { .Unop = UnopNode { .unopType = UnopType.BinaryNot, .exp = unop, .startRegion = startRegion, .endRegion = endRegion } };
},
else =>
{
const suffixexp = try allocator.allocator().create(SuffixexpNode);
suffixexp.* = try parseSuffixExp(tokens, i, allocator);
return ExpNode { .Suffixexp = suffixexp };
}
}
}
fn parseExpPrecedence(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator, lhs: ExpNode, minPrecedence: u8) !ExpNode
{
var currentLhs = lhs;
while(i.* < tokens.len and isBinop(tokens[i.*]))
{
const startRegion = tokens[i.*].region;
const precedence = try getPrecedence(tokens[i.*]);
if(precedence < minPrecedence)
{
break;
}
const op = try getBinopType(tokens[i.*]);
i.* += 1;
var rhs = try parseExpPrimary(tokens, i, allocator);
while(i.* < tokens.len and isBinop(tokens[i.*]) and
(try getPrecedence(tokens[i.*]) > precedence or
(try getPrecedence(tokens[i.*]) == precedence and isRightAssociative(tokens[i.*]))))
{
const associativityBoost: u8 = if(try getPrecedence(tokens[i.*]) == precedence) 0 else 1;
rhs = try parseExpPrecedence(tokens, i, allocator, rhs, precedence + associativityBoost);
}
const binop = try allocator.allocator().create(BinopNode);
binop.* = BinopNode { .lhs = currentLhs, .op = op, .rhs = rhs, .startRegion = startRegion, .endRegion = tokens[@min(i.*, tokens.len) - 1].region };
currentLhs = ExpNode { .Binop = binop };
}
return currentLhs;
}
fn isRightAssociative(token: Token) bool
{
return token.tokenType == TokenType.DotDot or token.tokenType == TokenType.Caret;
}
fn getBinopType(token: Token) !BinopType
{
return switch(token.tokenType)
{
TokenType.Or => BinopType.LogicalOr,
TokenType.And => BinopType.LocicalAnd,
TokenType.Lt => BinopType.Lt,
TokenType.Gt => BinopType.Gt,
TokenType.LtEquals => BinopType.LtEquals,
TokenType.GtEquals => BinopType.GtEquals,
TokenType.LtLt => BinopType.Shl,
TokenType.GtGt=> BinopType.Shr,
TokenType.TildeEquals => BinopType.NotEquals,
TokenType.EqualsEquals => BinopType.Equals,
TokenType.Pipe => BinopType.BinaryOr,
TokenType.Tilde => BinopType.BinaryNot,
TokenType.Ampersand => BinopType.BinaryAnd,
TokenType.DotDot => BinopType.Concat,
TokenType.Plus => BinopType.Add,
TokenType.Minus => BinopType.Sub,
TokenType.Star => BinopType.Mul,
TokenType.Slash => BinopType.Div,
TokenType.SlashSlash => BinopType.IntDiv,
TokenType.Percent => BinopType.Mod,
TokenType.Caret => BinopType.Exp,
else => error.NoBinopTypeForOperator,
};
}
fn getPrecedence(token: Token) !u8
{
return switch(token.tokenType)
{
TokenType.Or => 2,
TokenType.And => 4,
TokenType.Lt, TokenType.Gt, TokenType.LtEquals, TokenType.GtEquals, TokenType.TildeEquals, TokenType.EqualsEquals => 6,
TokenType.Pipe => 8,
TokenType.Tilde => 10,
TokenType.Ampersand => 12,
TokenType.LtLt, TokenType.GtGt => 14,
TokenType.DotDot => 16,
TokenType.Plus, TokenType.Minus => 18,
TokenType.Star, TokenType.Slash, TokenType.SlashSlash, TokenType.Percent => 20,
TokenType.Caret => 22,
else => error.NoPrecedenceForOperator,
};
}
fn isBinop(token: Token) bool
{
return switch(token.tokenType)
{
TokenType.Or, TokenType.And, TokenType.Lt, TokenType.Gt, TokenType.LtEquals, TokenType.GtEquals, TokenType.TildeEquals, TokenType.EqualsEquals,
TokenType.Pipe, TokenType.Tilde, TokenType.Ampersand, TokenType.LtLt, TokenType.GtGt, TokenType.DotDot, TokenType.Plus, TokenType.Minus,
TokenType.Star, TokenType.Slash, TokenType.SlashSlash, TokenType.Percent, TokenType.Caret => true,
else => false
};
}
fn parseExplist(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ExplistNode
{
const startRegion = tokens[@min(i.*, tokens.len) - 1].region;
var ret = ExplistNode { .exps = std.ArrayList(ExpNode).init(allocator.allocator()), .startRegion = startRegion, .endRegion = startRegion };
try ret.exps.append(try parseExp(tokens, i, allocator));
while(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Comma)
{
i.* += 1;
try ret.exps.append(try parseExp(tokens, i, allocator));
}
ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region;
return ret;
}
fn parseFuncname(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !FuncnameNode
{
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
{
return error.ExpectedFuncname;
}
const startRange = tokens[i.*].region;
var ret = FuncnameNode { .name = tokens[i.*].tokenData.string, .dottedNames = std.ArrayList([]u8).init(allocator.allocator()), .firstArg = null, .startRegion = startRange, .endRegion = startRange };
i.* += 1;
while(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Dot)
{
i.* += 1;
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
{
return error.ExpectedNameInDottedFuncname;
}
try ret.dottedNames.append(tokens[i.*].tokenData.string);
}
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Colon)
{
i.* += 1;
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
{
return error.ExpectedNameOfFirstArgInFuncname;
}
ret.firstArg = tokens[i.*].tokenData.string;
i.* += 1;
}
ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region;
return ret;
}
fn parseFuncbody(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !FuncbodyNode
{
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.RoundOpen)
{
return error.ExpectedRoundOpenStartingFuncbody;
}
const startRegion = tokens[i.*].region;
i.* += 1;
if(i.* >= tokens.len)
{
return error.ReachedEOFInFuncbodyParlist;
}
var pars: ?ParlistNode = null;
if(tokens[i.*].tokenType == TokenType.RoundClosed)
{
i.* += 1;
}
else
{
pars = try parseParlist(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.RoundClosed)
{
return error.ExpectedRoundClosedClosingFuncbodyParlist;
}
i.* += 1;
}
const ret = FuncbodyNode { .body = try parseBlock(tokens, i, allocator), .pars = pars, .startRegion = startRegion, .endRegion = tokens[i.*].region };
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
{
return error.ExpectedEndClosingFuncbody;
}
i.* += 1;
return ret;
}
fn parseParlist(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ParlistNode
{
if(i.* >= tokens.len)
{
return error.ReachedEOFInParlist;
}
const startRegion = tokens[i.*].region;
if(tokens[i.*].tokenType == TokenType.DotDotDot)
{
const endRegion = tokens[i.*].region;
i.* += 1;
return ParlistNode { .names = std.ArrayList([]u8).init(allocator.allocator()), .hasVarargs = true, .startRegion = startRegion, .endRegion = endRegion };
}
if(tokens[i.*].tokenType != TokenType.Name)
{
return error.ExpectedNameStartingParlist;
}
var ret = ParlistNode { .names = std.ArrayList([]u8).init(allocator.allocator()), .hasVarargs = false, .startRegion = startRegion, .endRegion = startRegion };
try ret.names.append(tokens[i.*].tokenData.string);
i.* += 1;
while(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Comma)
{
i.* += 1;
if(i.* >= tokens.len)
{
return error.ReachedEOFInParlistNameList;
}
switch(tokens[i.*].tokenType)
{
TokenType.Name =>
{
try ret.names.append(tokens[i.*].tokenData.string);
i.* += 1;
},
TokenType.DotDotDot =>
{
i.* += 1;
ret.hasVarargs = true;
break;
},
else => return error.UnexpectedTokenInParlistNameList,
}
}
ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region;
return ret;
}
fn parseAttnamelist(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !AttnamelistNode
{
// TODO: What happens if this is reaches EOF?
var ret = AttnamelistNode { .attnames = std.ArrayList(AttnameNode).init(allocator.allocator()), .startRegion = tokens[i.*].region, .endRegion = tokens[i.*].region };
try ret.attnames.append(try parseAttname(tokens, i));
while(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Comma)
{
i.* += 1;
try ret.attnames.append(try parseAttname(tokens, i));
}
ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region;
return ret;
}
fn parseSuffixExp(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !SuffixexpNode
{
// primaryexp { '.' 'Name' | '[' exp']' | ':' 'Name' args | args }
if(i.* >= tokens.len)
{
return error.ReachedEOFInSuffixExp;
}
const startRegion = tokens[i.*].region;
const firstPart = try switch(tokens[i.*].tokenType)
{
TokenType.Name =>
nameBlock: {
const name = tokens[i.*].tokenData.string;
i.* += 1;
break :nameBlock SuffixexpFirstPart { .Name = name };
},
TokenType.RoundOpen =>
roundOpenBlock: {
i.* += 1;
const ret = SuffixexpFirstPart { .BracketedExpr = try parseExp(tokens, i, allocator) };
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.RoundClosed)
{
return error.ExpectedRoundClosedClosingBracketedPrimaryExp;
}
i.* += 1;
break :roundOpenBlock ret;
},
else => error.UnexpectedTokenAsFirstPartOfSuffixExp,
};
var suffixes = std.ArrayList(SuffixexpSuffix).init(allocator.allocator());
while(i.* < tokens.len)
{
switch(tokens[i.*].tokenType)
{
TokenType.Dot =>
{
i.* += 1;
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
{
return error.ExpectedNameInDottedSuffixExp;
}
const name = tokens[i.*].tokenData.string;
i.* += 1;
try suffixes.append(SuffixexpSuffix { .Dot = name });
},
TokenType.SquareOpen =>
{
i.* += 1;
try suffixes.append(SuffixexpSuffix { .Indexed = try parseExp(tokens, i, allocator) });
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.SquareClosed)
{
return error.ExpectedSquareClosedClosingIndexedSuffixExp;
}
i.* += 1;
},
TokenType.Colon =>
{
const argsFirstArgStartRegion = tokens[i.*].region;
i.* += 1;
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
{
return error.ExpectedNameInArgsFirstArgSuffixExp;
}
const name = tokens[i.*].tokenData.string;
const argsFirstArgEndRegion = tokens[i.*].region;
i.* += 1;
try suffixes.append(SuffixexpSuffix { .ArgsFirstArg = ArgsFirstArgNode { .name = name, .rest = try parseArgs(tokens, i, allocator), .startRegion = argsFirstArgStartRegion, .endRegion = argsFirstArgEndRegion } });
},
TokenType.RoundOpen, TokenType.CurlyOpen, TokenType.StringLiteral =>
{
try suffixes.append(SuffixexpSuffix { .Args = try parseArgs(tokens, i, allocator) });
},
else => break,
}
}
const endRegion = tokens[@min(i.*, tokens.len) - 1].region;
const last = suffixes.getLastOrNull();
if(last != null)
{
switch(last.?)
{
SuffixexpSuffix.Args => |*args|
{
_ = suffixes.pop();
const functioncall = try allocator.allocator().create(FunctioncallNode);
functioncall.* = FunctioncallNode
{
.function = SuffixexpNode { .Normal = NormalSuffixNode { .firstPart = firstPart, .suffixes = suffixes, .startRegion = startRegion, .endRegion = endRegion } },
.args = args.*,
.objectArg = null,
.startRegion = startRegion,
.endRegion = endRegion,
};
return SuffixexpNode
{
.Functioncall = functioncall,
};
},
SuffixexpSuffix.ArgsFirstArg => |*node|
{
_ = suffixes.pop();
const functioncall = try allocator.allocator().create(FunctioncallNode);
functioncall.* = FunctioncallNode
{
.function = SuffixexpNode { .Normal = NormalSuffixNode { .firstPart = firstPart, .suffixes = suffixes, .startRegion = startRegion, .endRegion = endRegion } },
.args = node.rest,
.objectArg = node.name,
.startRegion = startRegion,
.endRegion = endRegion,
};
return SuffixexpNode
{
.Functioncall = functioncall,
};
},
else => {}
}
}
return SuffixexpNode { .Normal = NormalSuffixNode { .firstPart = firstPart, .suffixes = suffixes, .startRegion = startRegion, .endRegion = endRegion } };
}
fn parseVar(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !VarNode
{
const startRegion = tokens[i.*].region;
return suffixExpToVar(try parseSuffixExp(tokens, i, allocator), startRegion, tokens[@min(i.*, tokens.len) - 1].region);
}
fn parseAttname(tokens: []Token, i: *usize) !AttnameNode
{
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
{
return error.ExpectednameInAttribName;
}
const name = tokens[i.*].tokenData.string;
const startRegion = tokens[i.*].region;
i.* += 1;
var ret = AttnameNode { .name = name, .attribute = null, .startRegion = startRegion, .endRegion = startRegion };
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Lt)
{
ret.attribute = tokens[i.*].tokenData.string;
i.* += 1;
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
{
return error.ExpectedAttributeInAttrib;
}
ret.endRegion = tokens[i.*].region;
i.* += 1;
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Gt)
{
return error.ExpectedGtInAttrib;
}
}
return ret;
}
fn parseArgs(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ArgsNode
{
if(i.* >= tokens.len)
{
return error.ReachedEOFInArgs;
}
switch(tokens[i.*].tokenType)
{
TokenType.RoundOpen =>
{
i.* += 1;
if(i.* >= tokens.len)
{
return error.ReachedEOFInBracketedArgs;
}
if(tokens[i.*].tokenType == TokenType.RoundClosed)
{
i.* += 1;
return ArgsNode { .Bracketed = null };
}
const exps = try parseExplist(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.RoundClosed)
{
return error.ExpectedRoundClosedClosingBracketedArgs;
}
i.* += 1;
return ArgsNode { .Bracketed = exps };
},
TokenType.CurlyOpen => return ArgsNode { .Tableconstructor = try parseTableconstructor(tokens, i, allocator) },
TokenType.StringLiteral =>
{
const value = tokens[i.*].tokenData.string;
i.* += 1;
return ArgsNode { .Literal = value };
},
else => return error.UnexpectedTokenInArgs,
}
}
fn parseTableconstructor(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !TableconstructorNode
{
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.CurlyOpen)
{
return error.ExpectedCurlyOpenOpeningTableconstructor;
}
const startRegion = tokens[i.*].region;
i.* += 1;
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.CurlyClosed)
{
const endRegion = tokens[i.*].region;
i.* += 1;
return TableconstructorNode { .exps = null, .startRegion = startRegion, .endRegion = endRegion };
}
var ret = TableconstructorNode { .exps = try parseFieldlist(tokens, i, allocator), .startRegion = startRegion, .endRegion = startRegion };
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.CurlyClosed)
{
return error.ExpectedCurlyClosedClosingTableconstructor;
}
ret.endRegion = tokens[i.*].region;
i.* += 1;
return ret;
}
fn parseFieldlist(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !FieldlistNode
{
const startRegion = tokens[@min(i.*, tokens.len) - 1].region;
var ret = FieldlistNode { .exps = std.ArrayList(FieldNode).init(allocator.allocator()), .startRegion = startRegion, .endRegion = startRegion };
try ret.exps.append(try parseField(tokens, i, allocator));
while(i.* < tokens.len and isFieldsep(tokens[i.*]))
{
i.* += 1;
try ret.exps.append(try parseField(tokens, i, allocator));
}
if(i.* < tokens.len and isFieldsep(tokens[i.*]))
{
i.* += 1;
}
ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region;
return ret;
}
fn isFieldsep(token: Token) bool
{
return token.tokenType == TokenType.Comma or token.tokenType == TokenType.Semicolon;
}
fn parseField(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !FieldNode
{
if(i.* >= tokens.len)
{
return error.ReachedEOFInField;
}
const startRegion = tokens[i.*].region;
switch(tokens[i.*].tokenType)
{
TokenType.SquareOpen =>
{
i.* += 1;
const index = try parseExp(tokens, i, allocator);
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.SquareClosed)
{
return error.ExpectedSquareClosedClosingIndexedField;
}
i.* += 1;
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Equals)
{
return error.ExpectedEqualsInIndexedFieldExpression;
}
const endRegion = tokens[i.*].region;
i.* += 1;
return FieldNode { .IndexedAssignment = IndexedAssignmentNode { .index = index, .rhs = try parseExp(tokens, i, allocator), .startRegion = startRegion, .endRegion = endRegion } };
},
TokenType.Name =>
{
if(i.* + 1 < tokens.len and tokens[i.* + 1].tokenType == TokenType.Equals)
{
const name = tokens[i.*].tokenData.string;
i.* += 2;
return FieldNode { .Assignment = FieldAssignmentNode { .lhs = name, .rhs = try parseExp(tokens, i, allocator), .startRegion = startRegion, .endRegion = tokens[i.* - 1].region } };
}
return FieldNode { .Exp = try parseExp(tokens, i, allocator) };
},
else => return FieldNode { .Exp = try parseExp(tokens, i, allocator) },
}
}
fn suffixExpToVar(suffixexp: SuffixexpNode, startRegion: CodeRegion, endRegion: CodeRegion) !VarNode
{
var exp = suffixexp.Normal;
if(exp.suffixes.items.len == 0)
{
return VarNode { .Name = exp.firstPart.Name };
}
const last = exp.suffixes.pop();
return switch(last)
{
SuffixexpSuffix.Dot => |*name| VarNode { .Member = MemberVarNode { .name = name.*, .value = SuffixexpNode { .Normal = exp }, .startRegion = startRegion, .endRegion = endRegion } },
SuffixexpSuffix.Indexed => |*index| VarNode { .Indexed = IndexedVarNode { .index = index.*, .value = SuffixexpNode { .Normal = exp }, .startRegion = startRegion, .endRegion = endRegion } },
else => error.ExpectedDotOrIndexedSuffixWhenConvertingSuffixExpToVar,
};
}

1246
src/tokenizer.zig Normal file
View File

@ -0,0 +1,1246 @@
const types = @import("types.zig");
const std = @import("std");
const CodeRegion = @import("types.zig").CodeRegion;
const CodeLocation = @import("types.zig").CodeLocation;
pub const TokenType = enum
{
Name,
And, Break, Do, Else, Elseif, End,
False, For, Function, Goto, If, In,
Local, Nil, Not, Or, Repeat, Return,
Then, True, Until, While,
Plus, Minus, Star, Slash, Percent, Caret, Hash,
Ampersand, Tilde, Pipe, LtLt, GtGt, SlashSlash,
EqualsEquals, TildeEquals, LtEquals, GtEquals, Lt, Gt, Equals,
RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, ColonColon,
Semicolon, Colon, Comma, Dot, DotDot, DotDotDot,
Numeral,
StringLiteral,
};
const TokenData = union(enum)
{
string: []u8,
numeral: types.Numeral,
none,
};
pub const Token = struct
{
tokenType: TokenType,
tokenData: TokenData,
region: CodeRegion,
};
const TokenizerState = enum
{
Start,
Quote, SingleQuote, Name, Number, Zero,
A, B, D, E, F, G, I, L, N, O, R, T, U, W,
Plus, Minus, Star, Slash, Percent, Caret, Hash,
Ampersand, Tilde, Pipe, Lt, Gt, Equals, RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed,
Colon, Semicolon, Comma, Dot,
An, Br, Do, El, En, Fa, Fo, Fu, Go, If, In, Lo, Ni, No, Or, Re, Th, Tr, Un, Wh,
LtLt, GtGt, SlashSlash, EqualsEquals, TildeEquals, LtEquals, GtEquals, ColonColon, DotDot,
SmallCommentStart, QuoteBackslash, SingleQuoteBackslash, String, HexNumberX, ExpNumber,
And, Bre, Els, End, Fal, For, Fun, Got, Loc, Nil, Not, Rep, Ret, The, Tru, Unt, Whi,
DotDotDot, HexNumber, QuoteBackslashZ, SingleQuoteBackslashZ,
BigCommentLongBracketStart, SmallComment,
Brea, Else, Fals, Func, Goto, Loca, Repe, Retu, Then, True, Unti, Whil, HexExpNumber,
BigComment, BigCommentLongBracketEnd,
Break, Elsei, False, Funct, Local, Repea, Retur, Until, While,
Elseif, Functi, Repeat, Return,
Functio,
Function,
};
fn tokenizeUpdateIndexAndState(lastIndex: *?usize, index: ?usize, state: *TokenizerState, newState: TokenizerState, region: *CodeRegion) void
{
lastIndex.* = index;
state.* = newState;
if(index == null)
{
region.*.start = null;
region.*.length = 0;
}
else
{
if(region.*.start == null)
{
// TODO: There is no line/col info here and plumbing it to here would be pain.
region.*.start = CodeLocation { .col = 0, .line = 0 };
}
region.*.length += 1;
}
}
fn tokenizeTerminalBase(lastIndex: *?usize, index: ?usize, tokenType: *?TokenType, state: *TokenizerState, newTokenType: ?TokenType, newState: TokenizerState, region: *CodeRegion) void
{
tokenizeUpdateIndexAndState(lastIndex, index, state, newState, region);
tokenType.* = newTokenType;
}
fn tokenizeTerminalStr(lastIndex: *?usize, index: usize, tokenType: *?TokenType, state: *TokenizerState, newTokenType: ?TokenType, newState: TokenizerState, tokenStr: *std.ArrayList(u8), ch: u8, region: *CodeRegion) !void
{
tokenizeTerminalBase(lastIndex, index, tokenType, state, newTokenType, newState, region);
try tokenStr.append(ch);
}
fn tokenizeTerminalIntNum(lastIndex: *?usize, index: usize, tokenType: *?TokenType, state: *TokenizerState, newTokenType: TokenType, newState: TokenizerState, tokenNumeral: *?types.Numeral, ch: u8, region: *CodeRegion) !void
{
tokenizeTerminalBase(lastIndex, index, tokenType, state, newTokenType, newState, region);
if(!std.ascii.isDigit(ch))
{
return error.NoDigit;
}
const digitValue = @as(i64, ch - '0');
if(tokenNumeral.* == null)
{
tokenNumeral.* = types.Numeral { .Integer = digitValue };
}
else
{
switch(tokenNumeral.*.?)
{
.Integer => |*n| n.* = n.* * 10 + digitValue,
.Float => return error.ExpectedIntGotFloat
}
}
}
fn tokenizeTerminalNoToken(lastIndex: *?usize, index: usize, state: *TokenizerState, newState: TokenizerState, tokenStr: *std.ArrayList(u8), ch: u8, region: *CodeRegion) !void
{
tokenizeUpdateIndexAndState(lastIndex, index, state, newState, region);
try tokenStr.*.append(ch);
}
fn tokenizeBacktrack(lastIndex: *?usize, index: *usize, tokens: *std.ArrayList(Token), tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, state: *TokenizerState, allocator: std.mem.Allocator, region: *CodeRegion) !void
{
try tokenizeBacktrackCustomToken(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, tokenType.*.?, allocator, region);
}
fn tokenizeBacktrackCustomToken(lastIndex: *?usize, index: *usize, tokens: *std.ArrayList(Token), tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, state: *TokenizerState, newTokenType: TokenType, allocator: std.mem.Allocator, region: *CodeRegion) !void
{
if(lastIndex.* == null or tokenType.* == null)
{
return error.LexError;
}
if(newTokenType == TokenType.StringLiteral or newTokenType == TokenType.Name)
{
const content = try allocator.alloc(u8, tokenStr.*.items.len);
@memcpy(content, tokenStr.*.items);
try tokens.append(Token { .tokenType = newTokenType, .tokenData = TokenData { .string = content }, .region = region.* });
}
else
{
try tokens.append(Token { .tokenType = newTokenType, .region = region.*, .tokenData = if(tokenType.*.? == TokenType.Numeral) TokenData { .numeral = tokenNumeral.*.? }
else TokenData.none
});
}
tokenNumeral.* = null;
index.* = lastIndex.*.?;
tokenStr.*.clearAndFree();
// region is reset in tokenizeTerminalBase since null is passed as index
tokenizeTerminalBase(lastIndex, null, tokenType, state, null, TokenizerState.Start, region);
}
fn tokenizeAlphanumericNonstart(lastIndex: *?usize, index: *usize, tokens: *std.ArrayList(Token), tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, state: *TokenizerState, ch: u8, newTokenType: TokenType, allocator: std.mem.Allocator, region: *CodeRegion) !void
{
if(std.ascii.isAlphanumeric(ch) or ch == '_')
{
try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.Name, tokenStr, ch, region);
}
else
{
try tokenizeBacktrackCustomToken(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, newTokenType, allocator, region);
}
}
fn tokenizeChar(state: *TokenizerState, ch: u8, lastIndex: *?usize, index: *usize, tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, tokens: *std.ArrayList(Token), longBracketLevel: *u32, region: *CodeRegion, allocator: std.mem.Allocator) !void
{
switch(state.*)
{
TokenizerState.Start =>
{
switch(ch)
{
'-' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Minus, TokenizerState.Minus, region),
',' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Comma, TokenizerState.Comma, region),
'=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Equals, TokenizerState.Equals, region),
'(' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.RoundOpen, TokenizerState.RoundOpen, region),
')' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.RoundClosed, TokenizerState.RoundClosed, region),
'.' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Dot, TokenizerState.Dot, region),
':' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Colon, TokenizerState.Colon, region),
'{' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.CurlyOpen, TokenizerState.CurlyOpen, region),
'}' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.CurlyClosed, TokenizerState.CurlyClosed, region),
'[' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.SquareOpen, TokenizerState.SquareOpen, region),
']' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.SquareClosed, TokenizerState.SquareClosed, region),
'+' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Plus, TokenizerState.Plus, region),
'~' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Tilde, TokenizerState.Tilde, region),
'>' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Gt, TokenizerState.Gt, region),
'<' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Lt, TokenizerState.Lt, region),
'#' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Hash, TokenizerState.Hash, region),
'|' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Pipe, TokenizerState.Pipe, region),
'&' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Ampersand, TokenizerState.Ampersand, region),
'%' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Percent, TokenizerState.Percent, region),
'*' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Star, TokenizerState.Star, region),
'/' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Slash, TokenizerState.Slash, region),
';' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Semicolon, TokenizerState.Semicolon, region),
'^' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Caret, TokenizerState.Caret, region),
'a' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.A, tokenStr, ch, region),
'b' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.B, tokenStr, ch, region),
'd' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.D, tokenStr, ch, region),
'e' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.E, tokenStr, ch, region),
'f' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.F, tokenStr, ch, region),
'i' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.I, tokenStr, ch, region),
'g' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.G, tokenStr, ch, region),
'l' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.L, tokenStr, ch, region),
'n' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.N, tokenStr, ch, region),
'o' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.O, tokenStr, ch, region),
'r' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.R, tokenStr, ch, region),
't' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.T, tokenStr, ch, region),
'u' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.U, tokenStr, ch, region),
'w' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.W, tokenStr, ch, region),
'0' => try tokenizeTerminalIntNum(lastIndex, index.*, tokenType, state, TokenType.Numeral, TokenizerState.Zero, tokenNumeral, ch, region),
'"' =>
{
tokenType.* = null;
state.* = TokenizerState.Quote;
},
'\'' =>
{
tokenType.* = null;
state.* = TokenizerState.SingleQuote;
},
else =>
{
if(std.ascii.isWhitespace(ch))
{
}
else if(std.ascii.isAlphabetic(ch) or ch == '_')
{
try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.Name, tokenStr, ch, region);
}
else if(std.ascii.isDigit(ch))
{
try tokenizeTerminalIntNum(lastIndex, index.*, tokenType, state, TokenType.Numeral, TokenizerState.Number, tokenNumeral, ch, region);
}
else
{
std.debug.print("{}: {c}\n", .{state.*, ch});
return error.NotImplemented;
}
}
}
},
TokenizerState.Quote =>
{
switch(ch)
{
'\\' => state.* = TokenizerState.QuoteBackslash,
'"' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region),
else => try tokenStr.*.append(ch),
}
},
TokenizerState.QuoteBackslash =>
{
switch(ch)
{
'a' =>
{
try tokenStr.append('\u{0007}');
state.* = TokenizerState.Quote;
},
'b' =>
{
try tokenStr.append('\u{0008}');
state.* = TokenizerState.Quote;
},
't' =>
{
try tokenStr.append('\t');
state.* = TokenizerState.Quote;
},
'n' | '\n' =>
{
try tokenStr.append('\n');
state.* = TokenizerState.Quote;
},
'v' =>
{
try tokenStr.append('\u{000b}');
state.* = TokenizerState.Quote;
},
'f' =>
{
try tokenStr.append('\u{000c}');
state.* = TokenizerState.Quote;
},
'r' =>
{
try tokenStr.append('\r');
state.* = TokenizerState.Quote;
},
'\\' =>
{
try tokenStr.append('\\');
state.* = TokenizerState.Quote;
},
'"' =>
{
try tokenStr.append('\"');
state.* = TokenizerState.Quote;
},
'\'' =>
{
try tokenStr.append('\'');
state.* = TokenizerState.Quote;
},
'z' =>
{
state.* = TokenizerState.QuoteBackslashZ;
},
else => return error.UnknownEscapeSequence,
}
},
TokenizerState.QuoteBackslashZ =>
{
switch(ch)
{
'\\' => state.* = TokenizerState.QuoteBackslash,
'"' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region),
else =>
{
if(!std.ascii.isWhitespace(ch))
{
try tokenStr.append(ch);
state.* = TokenizerState.Quote;
}
else
{
// Noop, https://www.lua.org/manual/5.4/manual.html#3.1:
// "The escape sequence '\z' skips the following span of whitespace characters, including line breaks;"
}
}
}
},
TokenizerState.SingleQuote =>
{
switch(ch)
{
'\\' => state.* = TokenizerState.SingleQuoteBackslash,
'\'' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region),
else => try tokenStr.append(ch),
}
},
TokenizerState.SingleQuoteBackslash =>
{
switch(ch)
{
'a' =>
{
try tokenStr.append('\u{0007}');
state.* = TokenizerState.SingleQuote;
},
'b' =>
{
try tokenStr.append('\u{0008}');
state.* = TokenizerState.SingleQuote;
},
't' =>
{
try tokenStr.append('\t');
state.* = TokenizerState.SingleQuote;
},
'n' | '\n' =>
{
try tokenStr.append('\n');
state.* = TokenizerState.SingleQuote;
},
'v' =>
{
try tokenStr.append('\u{000b}');
state.* = TokenizerState.SingleQuote;
},
'f' =>
{
try tokenStr.append('\u{000c}');
state.* = TokenizerState.SingleQuote;
},
'r' =>
{
try tokenStr.append('\r');
state.* = TokenizerState.SingleQuote;
},
'\\' =>
{
try tokenStr.append('\\');
state.* = TokenizerState.SingleQuote;
},
'"' =>
{
try tokenStr.append('\"');
state.* = TokenizerState.SingleQuote;
},
'\'' =>
{
try tokenStr.append('\'');
state.* = TokenizerState.SingleQuote;
},
'z' =>
{
state.* = TokenizerState.SingleQuoteBackslashZ;
},
else => return error.UnknownEscapeSequence,
}
},
TokenizerState.SingleQuoteBackslashZ =>
{
switch(ch)
{
'\\' => state.* = TokenizerState.SingleQuoteBackslash,
'\'' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region),
else =>
{
if(!std.ascii.isWhitespace(ch))
{
try tokenStr.append(ch);
state.* = TokenizerState.SingleQuote;
}
else
{
// Noop, https://www.lua.org/manual/5.4/manual.html#3.1:
// "The escape sequence '\z' skips the following span of whitespace characters, including line breaks;"
}
}
}
},
TokenizerState.String => try tokenizeBacktrackCustomToken(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, TokenType.StringLiteral, allocator, region),
TokenizerState.Name => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
TokenizerState.Zero =>
{
switch(ch)
{
'x' =>
{
try tokenStr.*.append(ch);
tokenType.* = null;
state.* = TokenizerState.HexNumberX;
},
'.' => return error.NotImplemented,
else =>
{
if(std.ascii.isDigit(ch))
{
const digitValue = @as(i64, ch - '0');
lastIndex.* = index.*;
if(tokenNumeral.* == null)
{
tokenNumeral.* = types.Numeral { .Integer = digitValue };
tokenType.* = TokenType.Numeral;
}
else
{
tokenNumeral.*.?.Integer = tokenNumeral.*.?.Integer * 10 + digitValue;
}
}
else
{
try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region);
}
}
}
},
TokenizerState.HexNumberX =>
{
if(std.ascii.isHex(ch))
{
lastIndex.* = index.*;
tokenType.* = TokenType.Numeral;
if(std.ascii.isDigit(ch))
{
tokenNumeral.* = types.Numeral { .Integer = @as(i64, ch - '0') };
}
else
{
tokenNumeral.* = types.Numeral { .Integer = @as(i64, std.ascii.toLower(ch) - 'a') };
}
}
else
{
try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region);
}
},
TokenizerState.HexNumber =>
{
switch(ch)
{
'p' =>
{
try tokenStr.*.append(ch);
tokenType.* = null;
state.* = TokenizerState.HexExpNumber;
},
else =>
{
if(std.ascii.isHex(ch))
{
lastIndex.* = index.*;
tokenType.* = TokenType.Numeral;
if(std.ascii.isDigit(ch))
{
tokenNumeral.* = types.Numeral { .Integer = @as(i64, ch - '0') };
}
else
{
tokenNumeral.* = types.Numeral { .Integer = @as(i64, std.ascii.toLower(ch) - 'a') };
}
}
else
{
try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region);
}
}
}
},
TokenizerState.Number =>
{
switch(ch)
{
'e' =>
{
try tokenStr.*.append(ch);
tokenType.* = null;
state.* = TokenizerState.ExpNumber;
},
'.' => return error.NotImplemented,
else =>
{
if(std.ascii.isDigit(ch))
{
const digitValue = @as(i64, ch - '0');
lastIndex.* = index.*;
if(tokenNumeral.* == null)
{
tokenNumeral.* = types.Numeral { .Integer = digitValue };
tokenType.* = TokenType.Numeral;
}
else
{
tokenNumeral.*.?.Integer = tokenNumeral.*.?.Integer * 10 + digitValue;
}
}
else
{
try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region);
}
}
}
},
TokenizerState.Comma, TokenizerState.RoundOpen, TokenizerState.RoundClosed,
TokenizerState.CurlyOpen, TokenizerState.CurlyClosed, TokenizerState.Plus,
TokenizerState.TildeEquals, TokenizerState.EqualsEquals, TokenizerState.Hash,
TokenizerState.GtEquals, TokenizerState.LtEquals, TokenizerState.SquareOpen,
TokenizerState.SquareClosed, TokenizerState.Pipe, TokenizerState.Ampersand,
TokenizerState.Percent, TokenizerState.Star, TokenizerState.Semicolon,
TokenizerState.Caret, TokenizerState.DotDotDot, TokenizerState.GtGt,
TokenizerState.LtLt, TokenizerState.SlashSlash => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
TokenizerState.Tilde =>
{
switch(ch)
{
'=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.TildeEquals, TokenizerState.TildeEquals, region),
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
}
},
TokenizerState.Gt =>
{
switch (ch)
{
'>' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.GtGt, TokenizerState.GtGt, region),
'=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.GtEquals, TokenizerState.GtEquals, region),
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
}
},
TokenizerState.Lt =>
{
switch(ch)
{
'<' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.LtLt, TokenizerState.LtLt, region),
'=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.LtEquals, TokenizerState.LtEquals, region),
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
}
},
TokenizerState.Slash =>
{
switch(ch)
{
'/' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.SlashSlash, TokenizerState.SlashSlash, region),
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
}
},
TokenizerState.Dot =>
{
switch(ch)
{
'.' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.DotDot, TokenizerState.DotDot, region),
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
}
},
TokenizerState.DotDot =>
{
switch(ch)
{
'.' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.DotDotDot, TokenizerState.DotDotDot, region),
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
}
},
TokenizerState.Colon =>
{
switch(ch)
{
':' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.ColonColon, TokenizerState.ColonColon, region),
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
}
},
TokenizerState.Equals =>
{
switch(ch)
{
'=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.EqualsEquals, TokenizerState.EqualsEquals, region),
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
}
},
TokenizerState.Minus =>
{
switch(ch)
{
'-' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, null, TokenizerState.SmallCommentStart, region),
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
}
},
TokenizerState.SmallCommentStart =>
{
switch(ch)
{
'[' =>
{
tokenType.* = null;
state.* = TokenizerState.BigCommentLongBracketStart;
},
'\n' =>
{
state.* = TokenizerState.Start;
lastIndex.* = null;
},
else =>
{
state.* = TokenizerState.SmallComment;
},
}
},
TokenizerState.SmallComment =>
{
switch(ch)
{
'\n' =>
{
state.* = TokenizerState.Start;
lastIndex.* = null;
},
else => { }
}
},
TokenizerState.BigCommentLongBracketStart =>
{
switch(ch)
{
'=' =>
{
longBracketLevel.* += 1;
},
'[' =>
{
state.* = TokenizerState.BigComment;
},
else => return error.LongBracketMalformedStartBigComment,
}
},
TokenizerState.BigComment =>
{
switch(ch)
{
']' =>
{
state.* = TokenizerState.BigCommentLongBracketEnd;
},
else => { },
}
},
TokenizerState.BigCommentLongBracketEnd =>
{
switch(ch)
{
'=' =>
{
if(longBracketLevel.* == 0)
{
return error.LongBracketLevelTooBigEndBigComment;
}
longBracketLevel.* -= 1;
},
']' =>
{
if(longBracketLevel.* != 0)
{
return error.LongBracketLevelTooSmallEndBigComment;
}
state.* = TokenizerState.Start;
},
else => return error.LongBracketMalformedSmallEndBigComment,
}
},
TokenizerState.A =>
{
switch(ch)
{
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.An, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.An =>
{
switch(ch)
{
'd' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.And, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.And => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.And, allocator, region),
TokenizerState.W =>
{
switch(ch)
{
'h' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Wh, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Wh =>
{
switch(ch)
{
'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Whi, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Whi =>
{
switch(ch)
{
'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Whil, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Whil =>
{
switch(ch)
{
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.While, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.While => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.While, allocator, region),
TokenizerState.B =>
{
switch(ch)
{
'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Br, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Br =>
{
switch(ch)
{
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Bre, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Bre =>
{
switch(ch)
{
'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Brea, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Brea =>
{
switch(ch)
{
'k' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Break, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Break => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Break, allocator, region),
TokenizerState.G =>
{
switch(ch)
{
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Go, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Go =>
{
switch(ch)
{
't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Got, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Got =>
{
switch(ch)
{
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Goto, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Goto => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Goto, allocator, region),
TokenizerState.R =>
{
switch(ch)
{
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Re, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Re =>
{
switch(ch)
{
't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Ret, tokenStr, ch, region),
'p' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Rep, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Ret =>
{
switch(ch)
{
'u' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Retu, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Retu =>
{
switch(ch)
{
'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Retur, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Retur =>
{
switch(ch)
{
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Return, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Return => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Return, allocator, region),
TokenizerState.Rep =>
{
switch(ch)
{
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Repe, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Repe =>
{
switch(ch)
{
'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Repea, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Repea =>
{
switch(ch)
{
't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Repeat, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Repeat => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Repeat, allocator, region),
TokenizerState.N =>
{
switch(ch)
{
'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Ni, tokenStr, ch, region),
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.No, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.No =>
{
switch(ch)
{
't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Not, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Not => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Not, allocator, region),
TokenizerState.Ni =>
{
switch(ch)
{
'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Nil, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Nil => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Nil, allocator, region),
TokenizerState.T =>
{
switch(ch)
{
'h' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Th, tokenStr, ch, region),
'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Tr, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Th =>
{
switch(ch)
{
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.The, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.The =>
{
switch(ch)
{
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Then, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Then => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Then, allocator, region),
TokenizerState.Tr =>
{
switch(ch)
{
'u' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Tru, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Tru =>
{
switch(ch)
{
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.True, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.True => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.True, allocator, region),
TokenizerState.E =>
{
switch(ch)
{
'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.El, tokenStr, ch, region),
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.En, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.En =>
{
switch(ch)
{
'd' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.End, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.End => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.End, allocator, region),
TokenizerState.El =>
{
switch(ch)
{
's' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Els, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Els =>
{
switch(ch)
{
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Else, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Else =>
{
switch(ch)
{
'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Elsei, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Else, allocator, region),
}
},
TokenizerState.Elsei =>
{
switch(ch)
{
'f' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Elseif, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Elseif => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Elseif, allocator, region),
TokenizerState.O =>
{
switch(ch)
{
'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Or, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Or => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Or, allocator, region),
TokenizerState.D =>
{
switch(ch)
{
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Do, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Do => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Do, allocator, region),
TokenizerState.I =>
{
switch(ch)
{
'f' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.If, tokenStr, ch, region),
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.In, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.In => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.In, allocator, region),
TokenizerState.If => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.If, allocator, region),
TokenizerState.F =>
{
switch(ch)
{
'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fa, tokenStr, ch, region),
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fo, tokenStr, ch, region),
'u' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fu, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Fu =>
{
switch(ch)
{
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fun, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Fun =>
{
switch(ch)
{
'c' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Func, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Func =>
{
switch(ch)
{
't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Funct, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Funct =>
{
switch(ch)
{
'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Functi, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Functi =>
{
switch(ch)
{
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Functio, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Functio =>
{
switch(ch)
{
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Function, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Function => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Function, allocator, region),
TokenizerState.Fa =>
{
switch(ch)
{
'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fal, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Fal =>
{
switch(ch)
{
's' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fals, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Fals =>
{
switch(ch)
{
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.False, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.False => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.False, allocator, region),
TokenizerState.Fo =>
{
switch(ch)
{
'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.For, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.For => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.For, allocator, region),
TokenizerState.L =>
{
switch(ch)
{
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Lo, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Lo =>
{
switch(ch)
{
'c' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Loc, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Loc =>
{
switch(ch)
{
'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Loca, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Loca =>
{
switch(ch)
{
'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Local, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Local => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Local, allocator, region),
TokenizerState.U =>
{
switch(ch)
{
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Un, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Un =>
{
switch(ch)
{
't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Unt, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Unt =>
{
switch(ch)
{
'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Unti, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Unti =>
{
switch(ch)
{
'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Until, tokenStr, ch, region),
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
}
},
TokenizerState.Until => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Until, allocator, region),
else =>
{
std.debug.print("{}\n", . {state.*});
return error.NotImplemented;
}
}
}
pub fn tokenize(fileContent: []u8, allocator: std.mem.Allocator) ![]Token
{
var tokens = std.ArrayList(Token).init(allocator);
var state: TokenizerState = TokenizerState.Start;
var lastIndex: ?usize = null;
var index: usize = 0;
var tokenType: ?TokenType = null;
var tokenStr = std.ArrayList(u8).init(allocator);
defer tokenStr.deinit();
var tokenNumeral: ?types.Numeral = null;
var longBracketLevel: u32 = 0;
var region = CodeRegion { .start = null, .length = 0 };
while(index < fileContent.len)
{
const ch = fileContent[index];
try tokenizeChar(&state, ch, &lastIndex, &index, &tokenType, &tokenStr, &tokenNumeral, &tokens, &longBracketLevel, &region, allocator);
if(region.start != null and region.start.?.col == 0 and region.start.?.line == 0)
{
region.start = calculatePoint(fileContent, index);
}
index += 1;
}
if(longBracketLevel != 0)
{
return error.UnbalancedLongBracketLevel;
}
try tokenizeChar(&state, '\n', &lastIndex, &index, &tokenType, &tokenStr, &tokenNumeral, &tokens, &longBracketLevel, &region, allocator);
if(region.start != null and region.start.?.col == 0 and region.start.?.line == 0)
{
region.start = calculatePoint(fileContent, index);
}
return tokens.toOwnedSlice();
}
fn calculatePoint(fileContent: []u8, index: usize) CodeLocation
{
var ret = CodeLocation { .col = 1, .line = 1 };
for(0..index) |i|
{
ret.col += 1;
if(fileContent[i] == '\n')
{
ret.line += 1;
ret.col = 1;
}
}
return ret;
}

142
src/treewalker.zig Normal file
View File

@ -0,0 +1,142 @@
const std = @import("std");
const parser = @import("parser.zig");
const types = @import("types.zig");
pub fn interpret(root: parser.ChunkNode, allocator: std.mem.Allocator) !void
{
var _ENV = types.Table { .entries= std.ArrayList(types.TableEntry).init(allocator) };
try walkChunk(root, &_ENV, allocator);
}
fn walkChunk(node: parser.ChunkNode, environment: *types.Table, allocator: std.mem.Allocator) !void
{
try walkBlock(node.block, environment, allocator);
}
fn walkBlock(node: parser.BlockNode, environment: *types.Table, allocator: std.mem.Allocator) !void
{
for(node.stats.items) |stat|
{
try walkStat(stat, environment, allocator);
}
if(node.retstat != null)
{
try walkRetstat(node.retstat.?, environment, allocator);
}
}
fn walkStat(node: parser.StatNode, environment: *types.Table, allocator: std.mem.Allocator) !void
{
switch(node)
{
.Assignment => |assignmentNode|
{
return try walkAssignmentNode(assignmentNode, environment, allocator);
},
else =>
{
std.debug.print("{any}\n", .{node});
return error.NotImplemented;
}
}
}
fn walkRetstat(node: parser.RetstatNode, environment: *types.Table, allocator: std.mem.Allocator) !void
{
_ = node;
_ = environment;
_ = allocator;
return error.NotImplemented;
}
fn walkAssignmentNode(node: parser.AssignmentNode, environment: *types.Table, allocator: std.mem.Allocator) !void
{
const results = try walkExplist(node.rhs, environment, allocator);
var i: usize = 0;
_ = results;
_ = i;
for(node.lhs.vars.items) |variable|
{
switch(variable)
{
.Indexed => |indexedNode|
{
_ = indexedNode;
return error.NotImplemented;
},
else => return error.NotImplemented,
}
}
}
fn walkExplist(node: parser.ExplistNode, environment: *types.Table, allocator: std.mem.Allocator) ![]types.Value
{
var results = std.ArrayList(types.Value).init(allocator);
for(node.exps.items) |exp|
{
try results.append(try walkExp(exp, environment, allocator, false));
}
return results.toOwnedSlice();
}
fn walkExp(node: parser.ExpNode, environment: *types.Table, allocator: std.mem.Allocator, isVariadicFunction: bool) !types.Value
{
switch(node)
{
.Nil => return types.Value.Nil,
.False => return types.Value { .Bool = false },
.True => return types.Value { .Bool = true },
.Numeral => |numeral| return types.Value { .Numeral = numeral },
.LiteralString => |string| return types.Value { .String = string },
.Varargs =>
{
if(isVariadicFunction)
{
return error.NotImplemented;
}
else
{
return error.UseVarargsOutsideVariadicFunction;
}
},
.Suffixexp => |suffixExp|
{
return walkSuffixexp(suffixExp.*, environment, allocator);
},
else =>
{
std.debug.print("{}\n", .{node});
return error.NotImplemented;
}
}
return error.NotImplemented;
}
fn walkSuffixexp(node: parser.SuffixexpNode, environment: *types.Table, allocator: std.mem.Allocator) !types.Value
{
_ = allocator;
switch(node)
{
.Normal => |normal|
{
switch(normal.firstPart)
{
.Name => |name|
{
std.debug.print("name: {s}\n", .{name});
std.debug.print("val: {!}\n", .{environment.get(types.Value { .String = name })});
},
else =>
{
std.debug.print("{}\n", .{normal.firstPart});
}
}
return error.NotImplemented;
},
else =>
{
std.debug.print("{}\n", .{node});
return error.NotImplemented;
}
}
}

194
src/types.zig Normal file
View File

@ -0,0 +1,194 @@
const std = @import("std");
pub const NumeralTag = enum
{
Integer,
Float,
};
pub fn isFloatExactInt(value: f64) bool
{
return value == 0 or (std.math.isNormal(value) and @floor(value) == value);
}
pub const Numeral = union(NumeralTag)
{
Integer: i64,
Float: f64,
pub fn rawEqual(self: Numeral, other: Numeral) bool
{
if(@as(NumeralTag, self) == @as(NumeralTag, other))
{
switch(self)
{
.Float => |value| return value == other.Float,
.Integer => |value| return value == other.Integer,
}
}
// Other is the respective other type
switch(self)
{
.Float => |value|
{
return isFloatExactInt(value) and @as(u64, @intFromFloat(value)) == other.Integer;
},
.Integer => |value|
{
return isFloatExactInt(other.Float) and @as(u64, @intFromFloat(other.Float)) == value;
}
}
}
};
test "float int equality"
{
const a = Numeral { .Float = 12.0 };
const b = Numeral { .Integer = 12 };
try std.testing.expect(a.rawEqual(b));
try std.testing.expect(b.rawEqual(a));
try std.testing.expect(a.rawEqual(a));
try std.testing.expect(b.rawEqual(b));
const c = Numeral { .Float = (0.2 + 0.1) * 10.0 };
const d = Numeral { .Integer = 3 };
try std.testing.expect(c.rawEqual(d));
try std.testing.expect(d.rawEqual(c));
try std.testing.expect(c.rawEqual(c));
try std.testing.expect(d.rawEqual(d));
const e = Numeral { .Float = 3.2 };
try std.testing.expect(!a.rawEqual(e));
try std.testing.expect(!b.rawEqual(e));
try std.testing.expect(!c.rawEqual(e));
try std.testing.expect(!d.rawEqual(e));
try std.testing.expect(!e.rawEqual(a));
try std.testing.expect(!e.rawEqual(b));
try std.testing.expect(!e.rawEqual(c));
try std.testing.expect(!e.rawEqual(d));
}
pub const TableEntry = struct
{
key: Value,
value: Value,
};
pub const Table = struct
{
entries: std.ArrayList(TableEntry),
pub fn get(self: Table, key: Value) Value
{
if(@as(ValueTag, key) == ValueTag.Nil)
{
return Value.Nil;
}
for (self.entries.items) |entry|
{
if(entry.key.rawEqual(key))
{
return entry.value;
}
}
return Value.Nil;
}
};
pub const ValueTag = enum
{
Nil,
Bool,
Numeral,
String,
Table,
};
pub const Value = union(ValueTag)
{
Nil,
Bool: bool,
Numeral: Numeral,
String: []const u8,
Table: *Table,
pub fn rawEqual(self: Value, other: Value) bool
{
if(@as(ValueTag, self) != @as(ValueTag, other))
{
return false;
}
switch(self)
{
.Nil => return true,
.Bool => |value| return value == other.Bool,
.Numeral => |value| return value.rawEqual(other.Numeral),
.String => |value| return std.mem.eql(u8, value, other.String),
.Table => |value| return value == other.Table,
}
}
};
test "Value equalities"
{
const a = Value { .Bool = true };
const b = Value { .Numeral = Numeral { .Integer = 1 } };
// true != 1
try std.testing.expect(!a.rawEqual(b));
// 1 != true
try std.testing.expect(!b.rawEqual(a));
const c = Value { .Bool = false };
// true != false
try std.testing.expect(!a.rawEqual(c));
// false!= true
try std.testing.expect(!c.rawEqual(a));
const d = Value { .Bool = true };
// true == true
try std.testing.expect(a.rawEqual(d));
// true == true
try std.testing.expect(d.rawEqual(a));
const e = Value { .String = "foo" };
const f = Value { .String = "bar" };
// foo != bar
try std.testing.expect(!e.rawEqual(f));
// bar != foo
try std.testing.expect(!f.rawEqual(e));
const g = Value { .String = "foo" };
// foo != foo
try std.testing.expect(e.rawEqual(g));
// foo != foo
try std.testing.expect(g.rawEqual(e));
var table = Table { .entries = std.ArrayList(TableEntry).init(std.testing.allocator) };
const h = Value { .Table = &table };
var table2 = Table { .entries = std.ArrayList(TableEntry).init(std.testing.allocator) };
const i = Value { .Table = &table2 };
const j = Value { .Table = &table2 };
try std.testing.expect(h.rawEqual(h));
try std.testing.expect(i.rawEqual(i));
try std.testing.expect(!h.rawEqual(i));
try std.testing.expect(!i.rawEqual(h));
try std.testing.expect(i.rawEqual(j));
try std.testing.expect(!h.rawEqual(j));
}
test "Table get"
{
var a = Table { .entries = std.ArrayList(TableEntry).init(std.testing.allocator) };
defer a.entries.deinit();
try a.entries.append(TableEntry { .key = Value { .Bool = true }, .value = Value { .String = "foo" } });
try std.testing.expectEqualStrings(a.get(Value { .Bool = true }).String, "foo");
try std.testing.expectEqual(a.get(Value.Nil), Value.Nil);
try std.testing.expectEqual(a.get(Value { .Numeral = Numeral { .Integer = 12 } }), Value.Nil);
var c = a.get(Value { .Bool = true });
c.String = "bar";
try std.testing.expectEqual(a.get(Value { .Bool = true }).String, "foo");
}
pub const CodeRegion = struct
{
start: ?CodeLocation,
length: usize,
};
pub const CodeLocation = struct
{
line: usize,
col: usize,
};

6
test/gengc.lua
View File

@ -1,6 +0,0 @@
if T == nil then
(Message or print)('\n >>> testC not active: \z
skipping some generational tests <<<\n')
print 'OK'
return
end

1
test/stringDataNotSet.lua
View File

@ -1 +0,0 @@
local smt = getmetatable("")

5
test/tokenizerFloat.lua
View File

@ -1,5 +0,0 @@
0.000001
0.1
0.12345
1234566788.21212

22
test/tokenizerInt.lua
View File

@ -1,22 +0,0 @@
0xff11 123 , () ~ ~= > >> >= < << <= / // . .. ... : :: = == - --
-- asdf
--[[askf]]zzz
--[==========[asdfasdf]==========] zzzz
[[fsad]]
[=
]
[===[test]]=]==]===] zzzzz
a ab an anb ando and
w wo wh who whi whio whil whilo while whileo
b bo br bro bre breo brea breao break breako
g gz go goz got gotz goto gotoz
r ro re reo ret rep reto repo retu repe retuo repeo retur repea returo repeao return repeat returno repeato
n nz ni no niz noz nil not nilz notz
t to th tr tho the tro tru then true theno trueo
e eo el en elo eno els end elso endo else elso elsei elseif elseio elseifo
o oo or oro
d dz do doz
i io if in ifo ino
f fz fo fa fu foz faz fuz for fal fun forz falz funz fals func falsz funcz false funct falsez functz functi functiz functio functioz function functionz
l lz lo loz loc locz loca locaz local localz
u uo un uno unt unto unti untio until untilo

10
test/tokenizerString.lua
View File

@ -1,10 +0,0 @@
"test" "\z
abc" "123" "sdlfkgj<3" "asldkfj" zzz "" "" "" "" "" "fasd!" "afd" "" "as" zzzz