Compare commits
No commits in common. "zig" and "rust" have entirely different histories.
19
.gitignore
vendored
19
.gitignore
vendored
@ -1,4 +1,17 @@
|
||||
zig-out/
|
||||
zig-cache/
|
||||
.vscode/
|
||||
# ---> Rust
|
||||
# Generated by Cargo
|
||||
# will have compiled files and executables
|
||||
debug/
|
||||
target/
|
||||
.vscode/
|
||||
|
||||
# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
|
||||
# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
|
||||
Cargo.lock
|
||||
|
||||
# These are backup files generated by rustfmt
|
||||
**/*.rs.bk
|
||||
|
||||
# MSVC Windows builds of rustc generate these, which store debugging information
|
||||
*.pdb
|
||||
|
||||
|
||||
7
Cargo.lock
generated
Normal file
7
Cargo.lock
generated
Normal file
@ -0,0 +1,7 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "luaaaaah"
|
||||
version = "0.1.0"
|
||||
8
Cargo.toml
Normal file
8
Cargo.toml
Normal file
@ -0,0 +1,8 @@
|
||||
[package]
|
||||
name = "luaaaaah"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
9
LICENSE
Normal file
9
LICENSE
Normal file
@ -0,0 +1,9 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023 0x4261756D
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
83
build.zig
83
build.zig
@ -1,83 +0,0 @@
|
||||
const std = @import("std");
|
||||
|
||||
// Although this function looks imperative, note that its job is to
|
||||
// declaratively construct a build graph that will be executed by an external
|
||||
// runner.
|
||||
pub fn build(b: *std.Build) void {
|
||||
// Standard target options allows the person running `zig build` to choose
|
||||
// what target to build for. Here we do not override the defaults, which
|
||||
// means any target is allowed, and the default is native. Other options
|
||||
// for restricting supported target set are available.
|
||||
const target = b.standardTargetOptions(.{});
|
||||
|
||||
// Standard optimization options allow the person running `zig build` to select
|
||||
// between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall. Here we do not
|
||||
// set a preferred release mode, allowing the user to decide how to optimize.
|
||||
const optimize = b.standardOptimizeOption(.{});
|
||||
|
||||
const exe = b.addExecutable(.{
|
||||
.name = "luaaaaah_zig",
|
||||
// In this case the main source file is merely a path, however, in more
|
||||
// complicated build scripts, this could be a generated file.
|
||||
.root_source_file = .{ .path = "src/main.zig" },
|
||||
.target = target,
|
||||
.optimize = optimize,
|
||||
});
|
||||
|
||||
exe.addModule("types", b.addModule("types", .{
|
||||
.source_file = .{ .path = "src/types.zig" }
|
||||
}));
|
||||
exe.addModule("tokenizer", b.addModule("tokenizer", .{
|
||||
.source_file = .{ .path = "src/tokenizer.zig" },
|
||||
}));
|
||||
exe.addModule("parser", b.addModule("parser", .{
|
||||
.source_file = .{ .path = "src/parser.zig" },
|
||||
}));
|
||||
exe.addModule("treewalker", b.addModule("treewalker", .{
|
||||
.source_file = .{ .path = "src/treewalker.zig" },
|
||||
}));
|
||||
|
||||
// This declares intent for the executable to be installed into the
|
||||
// standard location when the user invokes the "install" step (the default
|
||||
// step when running `zig build`).
|
||||
b.installArtifact(exe);
|
||||
|
||||
// This *creates* a Run step in the build graph, to be executed when another
|
||||
// step is evaluated that depends on it. The next line below will establish
|
||||
// such a dependency.
|
||||
const run_cmd = b.addRunArtifact(exe);
|
||||
|
||||
// By making the run step depend on the install step, it will be run from the
|
||||
// installation directory rather than directly from within the cache directory.
|
||||
// This is not necessary, however, if the application depends on other installed
|
||||
// files, this ensures they will be present and in the expected location.
|
||||
run_cmd.step.dependOn(b.getInstallStep());
|
||||
|
||||
// This allows the user to pass arguments to the application in the build
|
||||
// command itself, like this: `zig build run -- arg1 arg2 etc`
|
||||
if (b.args) |args| {
|
||||
run_cmd.addArgs(args);
|
||||
}
|
||||
|
||||
// This creates a build step. It will be visible in the `zig build --help` menu,
|
||||
// and can be selected like this: `zig build run`
|
||||
// This will evaluate the `run` step rather than the default, which is "install".
|
||||
const run_step = b.step("run", "Run the app");
|
||||
run_step.dependOn(&run_cmd.step);
|
||||
|
||||
// Creates a step for unit testing. This only builds the test executable
|
||||
// but does not run it.
|
||||
const unit_tests = b.addTest(.{
|
||||
.root_source_file = .{ .path = "src/main.zig" },
|
||||
.target = target,
|
||||
.optimize = optimize,
|
||||
});
|
||||
|
||||
const run_unit_tests = b.addRunArtifact(unit_tests);
|
||||
|
||||
// Similar to creating the run step earlier, this exposes a `test` step to
|
||||
// the `zig build --help` menu, providing a way for the user to request
|
||||
// running the unit tests.
|
||||
const test_step = b.step("test", "Run unit tests");
|
||||
test_step.dependOn(&run_unit_tests.step);
|
||||
}
|
||||
480
src/grammar.rs
Normal file
480
src/grammar.rs
Normal file
@ -0,0 +1,480 @@
|
||||
use crate::tokenizer::Token;
|
||||
|
||||
pub enum Rule
|
||||
{
|
||||
Terminal(u8, Token),
|
||||
NonTerminal(u8, u8, u8)
|
||||
}
|
||||
|
||||
pub const NONTERMINAL_NAMES: [&str; 115] =
|
||||
[
|
||||
"stat__15",
|
||||
"funcbody__50",
|
||||
"fieldlist",
|
||||
"fieldlist__1",
|
||||
"namelist",
|
||||
"parlist",
|
||||
">_non",
|
||||
"chunk",
|
||||
",_non",
|
||||
"field__29",
|
||||
"stat__11",
|
||||
"stat__45",
|
||||
"stat__33",
|
||||
"funcname",
|
||||
"prefixexp__25",
|
||||
";_?",
|
||||
"stat__5",
|
||||
"stat__38",
|
||||
"else_non",
|
||||
"stat__16",
|
||||
"<_non",
|
||||
"explist",
|
||||
"stat_*",
|
||||
"{_non",
|
||||
"funcnamedotexpansion_*",
|
||||
"stat__6",
|
||||
"funcnamedotexpansion",
|
||||
"moreattribs_*",
|
||||
"field__28",
|
||||
"retstat_?",
|
||||
"stat__9",
|
||||
"stat__37",
|
||||
"field",
|
||||
"local_non",
|
||||
")_non",
|
||||
"morevars_*",
|
||||
"moreattribs__19",
|
||||
"morevars",
|
||||
"forthirdarg_?",
|
||||
"::_non",
|
||||
"stat",
|
||||
"morefields_*",
|
||||
"]_non",
|
||||
"[_non",
|
||||
"assign",
|
||||
"field__31",
|
||||
"stat__4",
|
||||
"do_non",
|
||||
"unop",
|
||||
"elseifblocks__18",
|
||||
"elseblock_?",
|
||||
"label__21",
|
||||
"funcbody__51",
|
||||
"in_non",
|
||||
"stat__34",
|
||||
"exp",
|
||||
"funcnamecolonexpansion_?",
|
||||
"(_non",
|
||||
"if_non",
|
||||
"stat__7",
|
||||
"attrib__20",
|
||||
"elseifblocks__17",
|
||||
"fieldsep_?",
|
||||
"while_non",
|
||||
"elseifblocks_*",
|
||||
"binop",
|
||||
"..._non",
|
||||
"then_non",
|
||||
"return_non",
|
||||
"._non",
|
||||
"stat__35",
|
||||
"prefixexp",
|
||||
"attnamelist",
|
||||
"functioncall__26",
|
||||
"elseifblocks",
|
||||
"function_non",
|
||||
"}_non",
|
||||
"var__22",
|
||||
"tableconstructor__53",
|
||||
"varlist",
|
||||
"S_0",
|
||||
"Name_non",
|
||||
"goto_non",
|
||||
"parlistvarargs_?",
|
||||
"elseif_non",
|
||||
"for_non",
|
||||
"stat__8",
|
||||
"args",
|
||||
"stat__41",
|
||||
"stat__14",
|
||||
"until_non",
|
||||
"morenames_*",
|
||||
"stat__39",
|
||||
"stat__32",
|
||||
"end_non",
|
||||
"attnamelist__46",
|
||||
"moreexps_*",
|
||||
"stat__10",
|
||||
"=_non",
|
||||
":_non",
|
||||
"stat__36",
|
||||
"functioncall__27",
|
||||
"funcname__48",
|
||||
"attrib",
|
||||
"repeat_non",
|
||||
"args__49",
|
||||
"var",
|
||||
"var__23",
|
||||
"stat__40",
|
||||
"exp__0",
|
||||
"stat__42",
|
||||
"morefields",
|
||||
"moreattribs",
|
||||
"funcbody",
|
||||
"retstat__47",
|
||||
];
|
||||
|
||||
pub const TERMINAL_RULES: [(u8, Token); 125] =
|
||||
[
|
||||
(57, Token::RoundOpen),
|
||||
(34, Token::RoundClosed),
|
||||
(8, Token::Comma),
|
||||
(66, Token::DotDotDot),
|
||||
(69, Token::Dot),
|
||||
(39, Token::ColonColon),
|
||||
(99, Token::Colon),
|
||||
(15, Token::Semicolon),
|
||||
(20, Token::Lt),
|
||||
(98, Token::Equals),
|
||||
(6, Token::Gt),
|
||||
(81, Token::Name(String::new())),
|
||||
(80, Token::Return),
|
||||
(80, Token::Semicolon),
|
||||
(80, Token::Break),
|
||||
(43, Token::SquareOpen),
|
||||
(42, Token::SquareClosed),
|
||||
(87, Token::StringLiteral(String::new())),
|
||||
(105, Token::RoundClosed),
|
||||
(72, Token::Name(String::new())),
|
||||
(65, Token::Plus),
|
||||
(65, Token::Minus),
|
||||
(65, Token::Star),
|
||||
(65, Token::Slash),
|
||||
(65, Token::SlashSlash),
|
||||
(65, Token::Caret),
|
||||
(65, Token::Percent),
|
||||
(65, Token::Ampersand),
|
||||
(65, Token::Pipe),
|
||||
(65, Token::GtGt),
|
||||
(65, Token::LtLt),
|
||||
(65, Token::DotDot),
|
||||
(65, Token::Lt),
|
||||
(65, Token::LtEquals),
|
||||
(65, Token::Gt),
|
||||
(65, Token::GtEquals),
|
||||
(65, Token::EqualsEquals),
|
||||
(65, Token::TildeEquals),
|
||||
(65, Token::And),
|
||||
(65, Token::Or),
|
||||
(7, Token::Return),
|
||||
(7, Token::Semicolon),
|
||||
(7, Token::Break),
|
||||
(47, Token::Do),
|
||||
(18, Token::Else),
|
||||
(50, Token::Else),
|
||||
(84, Token::Elseif),
|
||||
(49, Token::Then),
|
||||
(94, Token::End),
|
||||
(55, Token::Nil),
|
||||
(55, Token::False),
|
||||
(55, Token::True),
|
||||
(55, Token::Numeral(String::new())),
|
||||
(55, Token::StringLiteral(String::new())),
|
||||
(55, Token::DotDotDot),
|
||||
(55, Token::Name(String::new())),
|
||||
(21, Token::Nil),
|
||||
(21, Token::False),
|
||||
(21, Token::True),
|
||||
(21, Token::Numeral(String::new())),
|
||||
(21, Token::StringLiteral(String::new())),
|
||||
(21, Token::DotDotDot),
|
||||
(21, Token::Name(String::new())),
|
||||
(32, Token::Nil),
|
||||
(32, Token::False),
|
||||
(32, Token::True),
|
||||
(32, Token::Numeral(String::new())),
|
||||
(32, Token::StringLiteral(String::new())),
|
||||
(32, Token::DotDotDot),
|
||||
(32, Token::Name(String::new())),
|
||||
(2, Token::Nil),
|
||||
(2, Token::False),
|
||||
(2, Token::True),
|
||||
(2, Token::Numeral(String::new())),
|
||||
(2, Token::StringLiteral(String::new())),
|
||||
(2, Token::DotDotDot),
|
||||
(2, Token::Name(String::new())),
|
||||
(3, Token::Comma),
|
||||
(3, Token::Semicolon),
|
||||
(62, Token::Comma),
|
||||
(62, Token::Semicolon),
|
||||
(85, Token::For),
|
||||
(13, Token::Name(String::new())),
|
||||
(75, Token::Function),
|
||||
(82, Token::Goto),
|
||||
(58, Token::If),
|
||||
(53, Token::In),
|
||||
(33, Token::Local),
|
||||
(36, Token::Name(String::new())),
|
||||
(4, Token::Name(String::new())),
|
||||
(5, Token::DotDotDot),
|
||||
(5, Token::Name(String::new())),
|
||||
(71, Token::Name(String::new())),
|
||||
(104, Token::Repeat),
|
||||
(29, Token::Return),
|
||||
(114, Token::Semicolon),
|
||||
(114, Token::Nil),
|
||||
(114, Token::False),
|
||||
(114, Token::True),
|
||||
(114, Token::Numeral(String::new())),
|
||||
(114, Token::StringLiteral(String::new())),
|
||||
(114, Token::DotDotDot),
|
||||
(114, Token::Name(String::new())),
|
||||
(68, Token::Return),
|
||||
(40, Token::Semicolon),
|
||||
(40, Token::Break),
|
||||
(22, Token::Semicolon),
|
||||
(22, Token::Break),
|
||||
(54, Token::End),
|
||||
(70, Token::End),
|
||||
(100, Token::End),
|
||||
(11, Token::Name(String::new())),
|
||||
(25, Token::End),
|
||||
(78, Token::CurlyClosed),
|
||||
(67, Token::Then),
|
||||
(48, Token::Minus),
|
||||
(48, Token::Not),
|
||||
(48, Token::Hash),
|
||||
(48, Token::Tilde),
|
||||
(90, Token::Until),
|
||||
(106, Token::Name(String::new())),
|
||||
(79, Token::Name(String::new())),
|
||||
(63, Token::While),
|
||||
(23, Token::CurlyOpen),
|
||||
(76, Token::CurlyClosed),
|
||||
|
||||
];
|
||||
|
||||
pub const NONTERMINAL_RULES: [(u8, u8, u8); 219] =
|
||||
[
|
||||
(80, 22, 29),
|
||||
(80, 68, 114),
|
||||
(80, 40, 22),
|
||||
(80, 82, 81),
|
||||
(80, 79, 44),
|
||||
(80, 47, 25),
|
||||
(80, 63, 46),
|
||||
(80, 104, 59),
|
||||
(80, 85, 30),
|
||||
(80, 75, 89),
|
||||
(80, 33, 0),
|
||||
(80, 58, 93),
|
||||
(80, 85, 31),
|
||||
(80, 33, 11),
|
||||
(80, 71, 87),
|
||||
(80, 71, 73),
|
||||
(80, 39, 51),
|
||||
(87, 57, 105),
|
||||
(87, 23, 78),
|
||||
(105, 21, 34),
|
||||
(44, 98, 21),
|
||||
(72, 81, 95),
|
||||
(95, 103, 27),
|
||||
(95, 20, 60),
|
||||
(95, 112, 27),
|
||||
(95, 8, 36),
|
||||
(103, 20, 60),
|
||||
(60, 81, 6),
|
||||
(7, 22, 29),
|
||||
(7, 68, 114),
|
||||
(7, 40, 22),
|
||||
(7, 82, 81),
|
||||
(7, 79, 44),
|
||||
(7, 47, 25),
|
||||
(7, 63, 46),
|
||||
(7, 104, 59),
|
||||
(7, 85, 30),
|
||||
(7, 75, 89),
|
||||
(7, 33, 0),
|
||||
(7, 58, 93),
|
||||
(7, 85, 31),
|
||||
(7, 33, 11),
|
||||
(7, 71, 87),
|
||||
(7, 71, 73),
|
||||
(7, 39, 51),
|
||||
(50, 18, 7),
|
||||
(74, 84, 61),
|
||||
(64, 74, 64),
|
||||
(64, 84, 61),
|
||||
(61, 55, 49),
|
||||
(49, 67, 7),
|
||||
(55, 48, 55),
|
||||
(55, 55, 109),
|
||||
(55, 75, 113),
|
||||
(55, 23, 78),
|
||||
(55, 57, 14),
|
||||
(55, 71, 77),
|
||||
(55, 71, 26),
|
||||
(55, 71, 87),
|
||||
(55, 71, 73),
|
||||
(109, 65, 55),
|
||||
(21, 55, 96),
|
||||
(21, 48, 55),
|
||||
(21, 55, 109),
|
||||
(21, 75, 113),
|
||||
(21, 23, 78),
|
||||
(21, 57, 14),
|
||||
(21, 71, 77),
|
||||
(21, 71, 26),
|
||||
(21, 71, 87),
|
||||
(21, 71, 73),
|
||||
(32, 43, 28),
|
||||
(32, 81, 45),
|
||||
(32, 48, 55),
|
||||
(32, 55, 109),
|
||||
(32, 75, 113),
|
||||
(32, 23, 78),
|
||||
(32, 57, 14),
|
||||
(32, 71, 77),
|
||||
(32, 71, 26),
|
||||
(32, 71, 87),
|
||||
(32, 71, 73),
|
||||
(28, 55, 9),
|
||||
(9, 42, 45),
|
||||
(45, 98, 55),
|
||||
(2, 32, 3),
|
||||
(2, 43, 28),
|
||||
(2, 81, 45),
|
||||
(2, 48, 55),
|
||||
(2, 55, 109),
|
||||
(2, 75, 113),
|
||||
(2, 23, 78),
|
||||
(2, 57, 14),
|
||||
(2, 71, 77),
|
||||
(2, 71, 26),
|
||||
(2, 71, 87),
|
||||
(2, 71, 73),
|
||||
(3, 41, 62),
|
||||
(3, 111, 41),
|
||||
(3, 62, 32),
|
||||
(38, 8, 55),
|
||||
(113, 57, 1),
|
||||
(1, 5, 52),
|
||||
(1, 34, 25),
|
||||
(52, 34, 25),
|
||||
(13, 81, 102),
|
||||
(102, 24, 56),
|
||||
(102, 26, 24),
|
||||
(102, 69, 81),
|
||||
(102, 99, 81),
|
||||
(56, 99, 81),
|
||||
(26, 69, 81),
|
||||
(24, 26, 24),
|
||||
(24, 69, 81),
|
||||
(73, 99, 101),
|
||||
(101, 81, 87),
|
||||
(51, 81, 39),
|
||||
(112, 8, 36),
|
||||
(27, 112, 27),
|
||||
(27, 8, 36),
|
||||
(36, 81, 103),
|
||||
(96, 38, 96),
|
||||
(96, 8, 55),
|
||||
(111, 62, 32),
|
||||
(41, 111, 41),
|
||||
(41, 62, 32),
|
||||
(91, 37, 91),
|
||||
(91, 8, 81),
|
||||
(37, 8, 81),
|
||||
(35, 37, 35),
|
||||
(35, 8, 81),
|
||||
(4, 81, 91),
|
||||
(5, 4, 83),
|
||||
(5, 81, 91),
|
||||
(83, 8, 66),
|
||||
(71, 57, 14),
|
||||
(71, 71, 77),
|
||||
(71, 71, 26),
|
||||
(71, 71, 87),
|
||||
(71, 71, 73),
|
||||
(14, 55, 34),
|
||||
(29, 68, 114),
|
||||
(114, 21, 15),
|
||||
(114, 55, 96),
|
||||
(114, 48, 55),
|
||||
(114, 55, 109),
|
||||
(114, 75, 113),
|
||||
(114, 23, 78),
|
||||
(114, 57, 14),
|
||||
(114, 71, 77),
|
||||
(114, 71, 26),
|
||||
(114, 71, 87),
|
||||
(114, 71, 73),
|
||||
(40, 82, 81),
|
||||
(40, 79, 44),
|
||||
(40, 47, 25),
|
||||
(40, 63, 46),
|
||||
(40, 104, 59),
|
||||
(40, 85, 30),
|
||||
(40, 75, 89),
|
||||
(40, 33, 0),
|
||||
(40, 58, 93),
|
||||
(40, 85, 31),
|
||||
(40, 33, 11),
|
||||
(40, 71, 87),
|
||||
(40, 71, 73),
|
||||
(40, 39, 51),
|
||||
(22, 40, 22),
|
||||
(22, 82, 81),
|
||||
(22, 79, 44),
|
||||
(22, 47, 25),
|
||||
(22, 63, 46),
|
||||
(22, 104, 59),
|
||||
(22, 85, 30),
|
||||
(22, 75, 89),
|
||||
(22, 33, 0),
|
||||
(22, 58, 93),
|
||||
(22, 85, 31),
|
||||
(22, 33, 11),
|
||||
(22, 71, 87),
|
||||
(22, 71, 73),
|
||||
(22, 39, 51),
|
||||
(97, 53, 10),
|
||||
(10, 21, 16),
|
||||
(89, 13, 113),
|
||||
(0, 75, 19),
|
||||
(19, 81, 113),
|
||||
(93, 55, 12),
|
||||
(12, 67, 54),
|
||||
(54, 7, 70),
|
||||
(54, 64, 100),
|
||||
(54, 50, 94),
|
||||
(70, 64, 100),
|
||||
(70, 50, 94),
|
||||
(100, 50, 94),
|
||||
(31, 81, 17),
|
||||
(17, 98, 92),
|
||||
(92, 55, 108),
|
||||
(46, 55, 16),
|
||||
(108, 8, 88),
|
||||
(88, 55, 110),
|
||||
(110, 38, 16),
|
||||
(110, 47, 25),
|
||||
(11, 72, 44),
|
||||
(11, 81, 95),
|
||||
(16, 47, 25),
|
||||
(25, 7, 94),
|
||||
(59, 7, 86),
|
||||
(59, 90, 55),
|
||||
(86, 90, 55),
|
||||
(30, 4, 97),
|
||||
(78, 2, 76),
|
||||
(106, 71, 77),
|
||||
(106, 71, 26),
|
||||
(77, 43, 107),
|
||||
(107, 55, 42),
|
||||
(79, 106, 35),
|
||||
(79, 71, 77),
|
||||
(79, 71, 26),
|
||||
|
||||
];
|
||||
31
src/main.rs
Normal file
31
src/main.rs
Normal file
@ -0,0 +1,31 @@
|
||||
pub mod tokenizer;
|
||||
pub mod parser;
|
||||
pub mod grammar;
|
||||
|
||||
use std::{env, fs};
|
||||
|
||||
use crate::{tokenizer::{Token, tokenize}, parser::parse};
|
||||
|
||||
fn main()
|
||||
{
|
||||
let args: Vec<String> = env::args().collect();
|
||||
let file_content = fs::read_to_string(&args[1]).expect("Could not read source file");
|
||||
|
||||
match compile(&file_content)
|
||||
{
|
||||
Ok(()) =>
|
||||
{
|
||||
println!("Done compiling");
|
||||
}
|
||||
Err(msg) => println!("ERROR: {}", msg)
|
||||
}
|
||||
}
|
||||
|
||||
fn compile(file_content: &String) -> Result<(), &'static str>
|
||||
{
|
||||
let tokens: Vec<Token> = tokenize(&file_content)?;
|
||||
println!("{:?}", tokens);
|
||||
let node = parse(tokens)?;
|
||||
println!("{:?}", node);
|
||||
return Ok(());
|
||||
}
|
||||
40
src/main.zig
40
src/main.zig
@ -1,40 +0,0 @@
|
||||
const std = @import("std");
|
||||
const tokenize = @import("tokenizer.zig").tokenize;
|
||||
const parse = @import("parser.zig").parse;
|
||||
const treewalk = @import("treewalker.zig").interpret;
|
||||
|
||||
pub fn main() !void
|
||||
{
|
||||
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
|
||||
defer _ = gpa.deinit();
|
||||
const allocator = gpa.allocator();
|
||||
|
||||
const args = try std.process.argsAlloc(allocator);
|
||||
defer std.process.argsFree(allocator, args);
|
||||
const file = try std.fs.cwd().openFile(args[1], .{});
|
||||
defer file.close();
|
||||
const content = try file.readToEndAlloc(allocator, 13000);
|
||||
defer allocator.free(content);
|
||||
const tokens = try tokenize(content, allocator);
|
||||
defer
|
||||
{
|
||||
var i: usize = 0;
|
||||
while(i < tokens.len)
|
||||
{
|
||||
switch(tokens[i].tokenData)
|
||||
{
|
||||
.string => |*data|
|
||||
{
|
||||
allocator.free(data.*);
|
||||
},
|
||||
else => {}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
allocator.free(tokens);
|
||||
}
|
||||
var parserAllocator = std.heap.ArenaAllocator.init(std.heap.page_allocator);
|
||||
defer parserAllocator.deinit();
|
||||
const root = try parse(tokens, &parserAllocator);
|
||||
try treewalk(root, allocator);
|
||||
}
|
||||
1201
src/parser.rs
Normal file
1201
src/parser.rs
Normal file
@ -0,0 +1,1201 @@
|
||||
use crate::tokenizer::Token;
|
||||
use crate::grammar::{NONTERMINAL_NAMES, NONTERMINAL_RULES, TERMINAL_RULES};
|
||||
|
||||
pub fn parse(tokens: Vec<Token>) -> Result<ChunkNode, &'static str>
|
||||
{
|
||||
return own(tokens);
|
||||
}
|
||||
|
||||
fn own(tokens: Vec<Token>) -> Result<ChunkNode, &'static str>
|
||||
{
|
||||
return parse_chunk(&tokens, &mut 0);
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ChunkNode
|
||||
{
|
||||
block: BlockNode
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct BlockNode
|
||||
{
|
||||
stats: Vec<StatNode>,
|
||||
retstat: Option<RetstatNode>
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub enum StatNode
|
||||
{
|
||||
Semicolon,
|
||||
Assignment { lhs: VarlistNode, rhs: ExplistNode },
|
||||
Functioncall(FunctioncallNode),
|
||||
Label(String),
|
||||
Break,
|
||||
Goto(String),
|
||||
Do(BlockNode),
|
||||
While { condition: ExpNode, body: BlockNode },
|
||||
Repeat { condition: ExpNode, body: BlockNode },
|
||||
If { condition: ExpNode, body: BlockNode, elseifs: Vec<ElseifNode>, else_: Option<BlockNode> },
|
||||
ForEq { var: String, start: ExpNode, end: ExpNode, change: Option<ExpNode>, body: BlockNode },
|
||||
ForIn { vars: Vec<String>, exps: ExplistNode, body: BlockNode },
|
||||
Function { name: FuncnameNode, body: FuncbodyNode },
|
||||
LocalFunction { name: String, body: FuncbodyNode },
|
||||
Local { attnames: AttnamelistNode, values: Option<ExplistNode> }
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct RetstatNode
|
||||
{
|
||||
values: Option<ExplistNode>
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub enum ExpNode
|
||||
{
|
||||
Nil,
|
||||
False,
|
||||
True,
|
||||
Numeral(f64),
|
||||
LiteralString(String),
|
||||
Varargs,
|
||||
Functiondef(FuncbodyNode),
|
||||
Suffixexp(Box<SuffixexpNode>),
|
||||
Tableconstructor(TableconstructorNode),
|
||||
Unop(UnopType, Box<ExpNode>),
|
||||
Binop { lhs: Box<ExpNode>, op: BinopType, rhs: Box<ExpNode> }
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub enum UnopType
|
||||
{
|
||||
Minus, LogicalNot, Length, BinaryNot,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub enum BinopType
|
||||
{
|
||||
LogicalOr,
|
||||
LocicalAnd,
|
||||
Lt, Gt, LtEquals, GtEquals, NotEquals, Equals,
|
||||
BinaryOr,
|
||||
BinaryNot,
|
||||
BinaryAnd,
|
||||
Shl, Shr,
|
||||
Concat,
|
||||
Add, Sub,
|
||||
Mul, Div, IntDiv, Mod,
|
||||
Exp,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct ExplistNode
|
||||
{
|
||||
exps: Vec<ExpNode>
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct TableconstructorNode
|
||||
{
|
||||
exps: Option<FieldlistNode>
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct FieldlistNode
|
||||
{
|
||||
exps: Vec<FieldNode>
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub enum FieldNode
|
||||
{
|
||||
IndexedAssignment { index: ExpNode, rhs: ExpNode },
|
||||
Assignment { lhs: String, rhs: ExpNode },
|
||||
Exp(ExpNode),
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct VarlistNode
|
||||
{
|
||||
vars: Vec<VarNode>
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct FunctioncallNode
|
||||
{
|
||||
function: SuffixexpNode,
|
||||
object_arg: Option<String>,
|
||||
args: ArgsNode,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub enum ArgsNode
|
||||
{
|
||||
Bracketed(Option<ExplistNode>),
|
||||
Tableconstructor(TableconstructorNode),
|
||||
Literal(String),
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct ElseifNode
|
||||
{
|
||||
condition: ExpNode,
|
||||
body: BlockNode,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct FuncnameNode
|
||||
{
|
||||
name: String,
|
||||
dotted_names: Vec<String>,
|
||||
first_arg: Option<String>,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct ParlistNode
|
||||
{
|
||||
names: Vec<String>,
|
||||
has_varargs: bool,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct FuncbodyNode
|
||||
{
|
||||
pars: Option<ParlistNode>,
|
||||
body: BlockNode,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct AttnamelistNode
|
||||
{
|
||||
attnames: Vec<AttnameNode>
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct AttnameNode
|
||||
{
|
||||
name: String,
|
||||
attribute: Option<String>,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub enum VarNode
|
||||
{
|
||||
Name(String),
|
||||
Indexed { value: SuffixexpNode, index: ExpNode },
|
||||
Member { value: SuffixexpNode, name: String }
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct SuffixexpNode
|
||||
{
|
||||
first_part: SuffixexpFirstPart,
|
||||
suffixes: Vec<SuffixexpSuffix>,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub enum SuffixexpFirstPart // a:b:test() => a:b.test(b) => a.b.test(a, b)
|
||||
{
|
||||
Name(String),
|
||||
BracketedExpr(ExpNode),
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub enum SuffixexpSuffix
|
||||
{
|
||||
Dot(String),
|
||||
Indexed(ExpNode),
|
||||
Args(ArgsNode),
|
||||
ArgsFirstArg(String, ArgsNode),
|
||||
}
|
||||
fn parse_chunk(tokens: &Vec<Token>, i: &mut usize) -> Result<ChunkNode, &'static str>
|
||||
{
|
||||
return Ok(ChunkNode { block: parse_block(tokens, i)? });
|
||||
}
|
||||
fn parse_block(tokens: &Vec<Token>, i: &mut usize) -> Result<BlockNode, &'static str>
|
||||
{
|
||||
let mut stats: Vec<StatNode> = Vec::new();
|
||||
while *i < tokens.len() && tokens[*i] != Token::Return && tokens[*i] != Token::End && tokens[*i] != Token::Elseif &&
|
||||
tokens[*i] != Token::Else
|
||||
{
|
||||
stats.push(parse_stat(tokens, i)?);
|
||||
}
|
||||
let retstat =
|
||||
if *i < tokens.len() && tokens[*i] == Token::Return { Some(parse_retstat(tokens, i)?) }
|
||||
else { None };
|
||||
return Ok(BlockNode { stats, retstat });
|
||||
}
|
||||
fn parse_stat(tokens: &Vec<Token>, i: &mut usize) -> Result<StatNode, &'static str>
|
||||
{
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of file while parsing stat");
|
||||
}
|
||||
match tokens[*i]
|
||||
{
|
||||
Token::Semicolon =>
|
||||
{
|
||||
*i += 1;
|
||||
Ok(StatNode::Semicolon)
|
||||
}
|
||||
Token::Break =>
|
||||
{
|
||||
*i += 1;
|
||||
Ok(StatNode::Break)
|
||||
}
|
||||
Token::Goto =>
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of stream but expected name for goto");
|
||||
}
|
||||
return if let Token::Name(name) = &tokens[*i]
|
||||
{
|
||||
*i += 1;
|
||||
Ok(StatNode::Goto(name.clone()))
|
||||
}
|
||||
else
|
||||
{
|
||||
Err("Expecting name for goto")
|
||||
};
|
||||
}
|
||||
Token::Do =>
|
||||
{
|
||||
*i += 1;
|
||||
let body = parse_block(tokens, i)?;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::End
|
||||
{
|
||||
return Err("Missing 'end' for do block");
|
||||
}
|
||||
*i += 1;
|
||||
return Ok(StatNode::Do(body));
|
||||
}
|
||||
Token::While =>
|
||||
{
|
||||
*i += 1;
|
||||
let condition = parse_exp(tokens, i)?;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::Do
|
||||
{
|
||||
return Err("Expected 'do' after while condition")
|
||||
}
|
||||
*i += 1;
|
||||
let body = parse_block(tokens, i)?;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::End
|
||||
{
|
||||
return Err("Missing 'end' for do block");
|
||||
}
|
||||
*i += 1;
|
||||
return Ok(StatNode::While { condition, body });
|
||||
}
|
||||
Token::Repeat =>
|
||||
{
|
||||
*i += 1;
|
||||
let body = parse_block(tokens, i)?;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::Until
|
||||
{
|
||||
return Err("Expected 'until' after repeat body");
|
||||
}
|
||||
*i += 1;
|
||||
return Ok(StatNode::Repeat { condition: parse_exp(tokens, i)?, body });
|
||||
}
|
||||
Token::If =>
|
||||
{
|
||||
*i += 1;
|
||||
let condition = parse_exp(tokens, i)?;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::Then
|
||||
{
|
||||
return Err("Expected 'then' after if condition");
|
||||
}
|
||||
*i += 1;
|
||||
let body = parse_block(tokens, i)?;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing if");
|
||||
}
|
||||
let mut elseifs: Vec<ElseifNode> = Vec::new();
|
||||
while tokens[*i] == Token::Elseif
|
||||
{
|
||||
*i += 1;
|
||||
let elseif_condition = parse_exp(tokens, i)?;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::Then
|
||||
{
|
||||
return Err("Expected 'then' after elseif condition");
|
||||
}
|
||||
*i += 1;
|
||||
elseifs.push(ElseifNode { condition: elseif_condition, body: parse_block(tokens, i)? });
|
||||
}
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing if");
|
||||
}
|
||||
let else_ = if tokens[*i] == Token::Else
|
||||
{
|
||||
*i += 1;
|
||||
Some(parse_block(tokens, i)?)
|
||||
}
|
||||
else
|
||||
{
|
||||
None
|
||||
};
|
||||
if *i >= tokens.len() || tokens[*i] != Token::End
|
||||
{
|
||||
return Err("Expected 'end' to close if");
|
||||
}
|
||||
*i += 1;
|
||||
return Ok(StatNode::If { condition, body, elseifs, else_ });
|
||||
}
|
||||
Token::For =>
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing for");
|
||||
}
|
||||
if let Token::Name(name) = &tokens[*i]
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing for after first name");
|
||||
}
|
||||
match tokens[*i]
|
||||
{
|
||||
Token::Equals =>
|
||||
{
|
||||
*i += 1;
|
||||
let start = parse_exp(tokens, i)?;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::Comma
|
||||
{
|
||||
return Err("Expected ',' after 'for eq' start value");
|
||||
}
|
||||
*i += 1;
|
||||
let end = parse_exp(tokens, i)?;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens after end value in 'for eq'");
|
||||
}
|
||||
let change = if tokens[*i] == Token::Comma
|
||||
{
|
||||
*i += 1;
|
||||
Some(parse_exp(tokens, i)?)
|
||||
}
|
||||
else
|
||||
{
|
||||
None
|
||||
};
|
||||
if *i >= tokens.len() || tokens[*i] != Token::Do
|
||||
{
|
||||
return Err("Expected 'do' after 'for eq' head");
|
||||
}
|
||||
*i += 1;
|
||||
let body = parse_block(tokens, i)?;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::End
|
||||
{
|
||||
return Err("Expected 'end' to close 'for eq'");
|
||||
}
|
||||
return Ok(StatNode::ForEq { var: name.clone(), start, end, change, body });
|
||||
}
|
||||
Token::Comma =>
|
||||
{
|
||||
let mut names = Vec::from([name.clone()]);
|
||||
while tokens[*i] == Token::Comma
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing 'for in' namelist");
|
||||
}
|
||||
if let Token::Name(next_name) = &tokens[*i]
|
||||
{
|
||||
names.push(next_name.clone());
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("Expected another name in 'for in' namelist");
|
||||
}
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing 'for in' namelist after name");
|
||||
}
|
||||
}
|
||||
if tokens[*i] != Token::In
|
||||
{
|
||||
return Err("Expected 'in' after 'for in' namelist");
|
||||
}
|
||||
*i += 1;
|
||||
let exps = parse_explist(tokens, i)?;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::Do
|
||||
{
|
||||
return Err("Expected 'do' after 'for in' explist");
|
||||
}
|
||||
*i += 1;
|
||||
let body = parse_block(tokens, i)?;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::End
|
||||
{
|
||||
return Err("Expected 'end' after 'for in' body");
|
||||
}
|
||||
*i += 1;
|
||||
return Ok(StatNode::ForIn { vars: names, exps, body });
|
||||
}
|
||||
_ => Err("Unexpected token after first name in for")
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("Expected name after 'for'");
|
||||
}
|
||||
}
|
||||
Token::Function =>
|
||||
{
|
||||
*i += 1;
|
||||
let funcname = parse_funcname(tokens, i)?;
|
||||
return Ok(StatNode::Function { name: funcname, body: parse_funcbody(tokens, i)? });
|
||||
}
|
||||
Token::Local =>
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing local");
|
||||
}
|
||||
if tokens[*i] == Token::Function
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing local function");
|
||||
}
|
||||
if let Token::Name(name) = &tokens[*i]
|
||||
{
|
||||
*i += 1;
|
||||
return Ok(StatNode::LocalFunction { name: name.clone(), body: parse_funcbody(tokens, i)? });
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("Expected local function name");
|
||||
}
|
||||
}
|
||||
let attnames = parse_attnamelist(tokens, i)?;
|
||||
let initials = if *i < tokens.len() && tokens[*i] == Token::Equals
|
||||
{
|
||||
*i += 1;
|
||||
Some(parse_explist(tokens, i)?)
|
||||
}
|
||||
else
|
||||
{
|
||||
None
|
||||
};
|
||||
return Ok(StatNode::Local { attnames, values: initials });
|
||||
}
|
||||
Token::ColonColon =>
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing label");
|
||||
}
|
||||
if let Token::Name(name) = &tokens[*i]
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::ColonColon
|
||||
{
|
||||
return Err("Expected '::' after name in label declaration");
|
||||
}
|
||||
*i += 1;
|
||||
return Ok(StatNode::Label(name.clone()));
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("Expected a name after '::' in label declaration")
|
||||
}
|
||||
}
|
||||
Token::Name(_) | Token::RoundOpen =>
|
||||
{
|
||||
// assignment or functioncall
|
||||
let suffix_expression = parse_suffixexp(tokens, i)?;
|
||||
match tokens[*i]
|
||||
{
|
||||
Token::Equals =>
|
||||
{
|
||||
*i += 1;
|
||||
return Ok(StatNode::Assignment { lhs: VarlistNode { vars: Vec::from([suffix_expression_to_var(suffix_expression)?]) }, rhs: parse_explist(tokens, i)? });
|
||||
}
|
||||
Token::Comma =>
|
||||
{
|
||||
let mut vars = Vec::from([suffix_expression_to_var(suffix_expression)?]);
|
||||
while tokens[*i] == Token::Comma
|
||||
{
|
||||
*i += 1;
|
||||
vars.push(parse_var(tokens, i)?);
|
||||
}
|
||||
if *i >= tokens.len() || tokens[*i] != Token::Equals
|
||||
{
|
||||
return Err("Expected '=' for assignment");
|
||||
}
|
||||
*i += 1;
|
||||
return Ok(StatNode::Assignment { lhs: VarlistNode { vars }, rhs: parse_explist(tokens, i)? });
|
||||
}
|
||||
_ =>
|
||||
{
|
||||
if suffix_expression.suffixes.is_empty()
|
||||
{
|
||||
println!("{:?} {} {:?}", tokens[*i], i, suffix_expression);
|
||||
return Err("Expected function call but suffix is empty");
|
||||
}
|
||||
if let Some(SuffixexpSuffix::Args(_)) = suffix_expression.suffixes.last()
|
||||
{
|
||||
return Ok(StatNode::Functioncall(suffix_expression_to_functioncall(suffix_expression)?));
|
||||
}
|
||||
if let Some(SuffixexpSuffix::ArgsFirstArg(_, _)) = suffix_expression.suffixes.last()
|
||||
{
|
||||
return Ok(StatNode::Functioncall(suffix_expression_to_functioncall(suffix_expression)?));
|
||||
}
|
||||
else
|
||||
{
|
||||
println!("{:?} {} {:?}", tokens[*i], i, suffix_expression.suffixes.last());
|
||||
return Err("Expected function call");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ =>
|
||||
{
|
||||
println!("{:?} {:?} {:?}", tokens[*i - 2], tokens[*i - 1], tokens[*i]);
|
||||
Err("Unexpected token while parsing stat")
|
||||
}
|
||||
}
|
||||
}
|
||||
fn suffix_expression_to_functioncall(suffixexp: SuffixexpNode) -> Result<FunctioncallNode, &'static str>
|
||||
{
|
||||
let mut new_suffixexp = suffixexp;
|
||||
let last = new_suffixexp.suffixes.pop();
|
||||
if let Some(SuffixexpSuffix::Args(args)) = last
|
||||
{
|
||||
return Ok(FunctioncallNode { function: new_suffixexp, object_arg: None, args });
|
||||
}
|
||||
if let Some(SuffixexpSuffix::ArgsFirstArg(first_arg, args)) = last
|
||||
{
|
||||
return Ok(FunctioncallNode { function: new_suffixexp, object_arg: Some(first_arg.clone()), args });
|
||||
}
|
||||
return Err("Cannot convert suffixexp to functioncall");
|
||||
}
|
||||
fn suffix_expression_to_var(suffixexp: SuffixexpNode) -> Result<VarNode, &'static str>
|
||||
{
|
||||
if suffixexp.suffixes.is_empty()
|
||||
{
|
||||
return if let SuffixexpFirstPart::Name(name) = suffixexp.first_part
|
||||
{
|
||||
Ok(VarNode::Name(name.clone()))
|
||||
}
|
||||
else
|
||||
{
|
||||
Err("Can only convert suffix exp without suffix to var if its first part is a name")
|
||||
};
|
||||
}
|
||||
let mut new_suffixexp = suffixexp;
|
||||
let last = new_suffixexp.suffixes.pop();
|
||||
if let Some(SuffixexpSuffix::Dot(name)) = last
|
||||
{
|
||||
return Ok(VarNode::Member { value: new_suffixexp, name: name.clone() });
|
||||
}
|
||||
if let Some(SuffixexpSuffix::Indexed(index)) = last
|
||||
{
|
||||
return Ok(VarNode::Indexed { value: new_suffixexp, index: index });
|
||||
}
|
||||
return Err("Cannot convert suffixexp to var");
|
||||
}
|
||||
fn parse_var(tokens: &Vec<Token>, i: &mut usize) -> Result<VarNode, &'static str>
|
||||
{
|
||||
todo!()
|
||||
}
|
||||
fn parse_args(tokens: &Vec<Token>, i: &mut usize) -> Result<ArgsNode, &'static str>
|
||||
{
|
||||
if *i > tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing args");
|
||||
}
|
||||
match &tokens[*i]
|
||||
{
|
||||
Token::RoundOpen =>
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while paring bracketed args");
|
||||
}
|
||||
if tokens[*i] == Token::RoundClosed
|
||||
{
|
||||
*i += 1;
|
||||
return Ok(ArgsNode::Bracketed(None));
|
||||
}
|
||||
let exps = parse_explist(tokens, i)?;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::RoundClosed
|
||||
{
|
||||
println!("|{:?}|{}|{:?}|", tokens[*i], i, exps);
|
||||
return Err("Expected ')' to close bracketed args");
|
||||
}
|
||||
*i += 1;
|
||||
return Ok(ArgsNode::Bracketed(Some(exps)));
|
||||
}
|
||||
Token::CurlyOpen =>
|
||||
{
|
||||
return Ok(ArgsNode::Tableconstructor(parse_tableconstructor(tokens, i)?));
|
||||
}
|
||||
Token::StringLiteral(name) =>
|
||||
{
|
||||
*i += 1;
|
||||
return Ok(ArgsNode::Literal(name.clone()));
|
||||
}
|
||||
_ => return Err("Unexpected token while parsing args")
|
||||
}
|
||||
}
|
||||
fn parse_suffixexp(tokens: &Vec<Token>, i: &mut usize) -> Result<SuffixexpNode, &'static str>
|
||||
{
|
||||
// primaryexp { '.' 'Name' | '[' exp']' | ':' 'Name' args | args }
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing suffixexp");
|
||||
}
|
||||
let first_part = match &tokens[*i]
|
||||
{
|
||||
Token::Name(name) =>
|
||||
{
|
||||
*i += 1;
|
||||
SuffixexpFirstPart::Name(name.clone())
|
||||
},
|
||||
Token::RoundOpen =>
|
||||
{
|
||||
*i += 1;
|
||||
let ret = SuffixexpFirstPart::BracketedExpr(parse_exp(tokens, i)?);
|
||||
if *i >= tokens.len() || tokens[*i] != Token::RoundClosed
|
||||
{
|
||||
return Err("Expected ')' to close bracketed primary expression");
|
||||
}
|
||||
*i += 1;
|
||||
ret
|
||||
}
|
||||
_ => return Err("Unexpected token as first part of suffixexp")
|
||||
};
|
||||
let mut suffixes = Vec::new();
|
||||
while *i < tokens.len()
|
||||
{
|
||||
match tokens[*i]
|
||||
{
|
||||
Token::Dot =>
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens but expected name for dotted suffix expression");
|
||||
}
|
||||
if let Token::Name(name) = &tokens[*i]
|
||||
{
|
||||
*i += 1;
|
||||
suffixes.push(SuffixexpSuffix::Dot(name.clone()));
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("Expected name for dotted suffix expression");
|
||||
}
|
||||
}
|
||||
Token::SquareOpen =>
|
||||
{
|
||||
*i += 1;
|
||||
suffixes.push(SuffixexpSuffix::Indexed(parse_exp(tokens, i)?));
|
||||
if *i >= tokens.len() || tokens[*i] != Token::SquareClosed
|
||||
{
|
||||
return Err("Expected ']' to close indexed suffix expression");
|
||||
}
|
||||
*i += 1;
|
||||
}
|
||||
Token::Colon =>
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens but expected name for dotted suffix expression");
|
||||
}
|
||||
if let Token::Name(name) = &tokens[*i]
|
||||
{
|
||||
*i += 1;
|
||||
suffixes.push(SuffixexpSuffix::ArgsFirstArg(name.clone(), parse_args(tokens, i)?));
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("Expected name for dotted suffix expression");
|
||||
}
|
||||
}
|
||||
Token::RoundOpen | Token::CurlyOpen | Token::StringLiteral(_) =>
|
||||
{
|
||||
suffixes.push(SuffixexpSuffix::Args(parse_args(tokens, i)?));
|
||||
}
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
return Ok(SuffixexpNode { first_part, suffixes });
|
||||
}
|
||||
fn parse_retstat(tokens: &Vec<Token>, i: &mut usize) -> Result<RetstatNode, &'static str>
|
||||
{
|
||||
if *i >= tokens.len() || tokens[*i] != Token::Return
|
||||
{
|
||||
return Err("Expected 'return' to start retstat");
|
||||
}
|
||||
*i += 1;
|
||||
if *i >= tokens.len() || tokens[*i] == Token::Semicolon || tokens[*i] == Token::Else || tokens[*i] == Token::Elseif ||
|
||||
tokens[*i] == Token::End
|
||||
{
|
||||
if *i < tokens.len() && tokens[*i] == Token::Semicolon
|
||||
{
|
||||
*i += 1;
|
||||
}
|
||||
return Ok(RetstatNode { values: None });
|
||||
}
|
||||
let values = parse_explist(tokens, i)?;
|
||||
if *i < tokens.len() && tokens[*i] == Token::Semicolon
|
||||
{
|
||||
*i += 1;
|
||||
}
|
||||
return Ok(RetstatNode { values: Some(values) });
|
||||
}
|
||||
fn parse_exp(tokens: &Vec<Token>, i: &mut usize) -> Result<ExpNode, &'static str>
|
||||
{
|
||||
let lhs = parse_exp_primary(tokens, i)?;
|
||||
return parse_exp_precedence(tokens, i, lhs, 0);
|
||||
}
|
||||
fn get_precedence(token: &Token) -> Result<u8, &'static str>
|
||||
{
|
||||
match token
|
||||
{
|
||||
Token::Or => Ok(2),
|
||||
Token::And => Ok(4),
|
||||
Token::Lt | Token::Gt | Token::LtEquals | Token::GtEquals | Token::TildeEquals | Token::EqualsEquals => Ok(6),
|
||||
Token::Pipe => Ok(8),
|
||||
Token::Tilde => Ok(10),
|
||||
Token::Ampersand => Ok(12),
|
||||
Token::LtLt | Token::GtGt => Ok(14),
|
||||
Token::DotDot => Ok(16),
|
||||
Token::Plus | Token::Minus => Ok(18),
|
||||
Token::Star | Token::Slash | Token::SlashSlash | Token::Percent => Ok(20),
|
||||
Token::Caret => Ok(22),
|
||||
_ => Err("Tried to get precedence for unknown operator"),
|
||||
}
|
||||
}
|
||||
fn get_binop(token: &Token) -> Result<BinopType, &'static str>
|
||||
{
|
||||
match token
|
||||
{
|
||||
Token::Or => Ok(BinopType::LogicalOr),
|
||||
Token::And => Ok(BinopType::LocicalAnd),
|
||||
Token::Lt => Ok(BinopType::Lt),
|
||||
Token::Gt => Ok(BinopType::Lt),
|
||||
Token::LtEquals => Ok(BinopType::LtEquals),
|
||||
Token::GtEquals => Ok(BinopType::GtEquals),
|
||||
Token::TildeEquals => Ok(BinopType::NotEquals),
|
||||
Token::EqualsEquals => Ok(BinopType::Equals),
|
||||
Token::Pipe => Ok(BinopType::BinaryOr),
|
||||
Token::Tilde => Ok(BinopType::BinaryNot),
|
||||
Token::Ampersand => Ok(BinopType::BinaryAnd),
|
||||
Token::DotDot => Ok(BinopType::Concat),
|
||||
Token::Plus => Ok(BinopType::Add),
|
||||
Token::Minus => Ok(BinopType::Sub),
|
||||
Token::Star => Ok(BinopType::Mul),
|
||||
Token::Slash => Ok(BinopType::Div),
|
||||
Token::SlashSlash => Ok(BinopType::IntDiv),
|
||||
Token::Percent => Ok(BinopType::Mod),
|
||||
Token::Caret => Ok(BinopType::Exp),
|
||||
_ =>
|
||||
{
|
||||
println!("{:?}", token);
|
||||
Err("Tried to get binop type for unknown operator")
|
||||
}
|
||||
}
|
||||
}
|
||||
fn is_binop(token: &Token) -> bool
|
||||
{
|
||||
match token
|
||||
{
|
||||
Token::Or | Token::And | Token::Lt | Token::Gt | Token::LtEquals | Token::GtEquals | Token::TildeEquals | Token::EqualsEquals |
|
||||
Token::Pipe | Token::Tilde | Token::Ampersand | Token::LtLt | Token::GtGt | Token::DotDot | Token::Plus | Token::Minus |
|
||||
Token::Star | Token::Slash | Token::SlashSlash | Token::Percent | Token::Caret =>
|
||||
{
|
||||
true
|
||||
}
|
||||
_ => false
|
||||
}
|
||||
}
|
||||
fn is_right_associative(token: &Token) -> bool
|
||||
{
|
||||
return token == &Token::DotDot || token == &Token::Caret;
|
||||
}
|
||||
fn parse_exp_precedence(tokens: &Vec<Token>, i: &mut usize, lhs: ExpNode, min_precedence: u8) -> Result<ExpNode, &'static str>
|
||||
{
|
||||
let mut lhs = lhs;
|
||||
while *i < tokens.len() && is_binop(&tokens[*i])
|
||||
{
|
||||
let precedence = get_precedence(&tokens[*i])?;
|
||||
if precedence < min_precedence
|
||||
{
|
||||
break;
|
||||
}
|
||||
let op = get_binop(&tokens[*i])?;
|
||||
*i += 1;
|
||||
let mut rhs = parse_exp_primary(tokens, i)?;
|
||||
while *i < tokens.len() && is_binop(&tokens[*i]) && (get_precedence(&tokens[*i])? > precedence ||
|
||||
(get_precedence(&tokens[*i])? == precedence && is_right_associative(&tokens[*i])))
|
||||
{
|
||||
rhs = parse_exp_precedence(tokens, i, rhs, precedence + if precedence == get_precedence(&tokens[*i])? {0} else {1})?;
|
||||
}
|
||||
lhs = ExpNode::Binop { lhs: Box::new(lhs), op, rhs: Box::new(rhs) };
|
||||
}
|
||||
return Ok(lhs);
|
||||
}
|
||||
fn parse_exp_primary(tokens: &Vec<Token>, i: &mut usize) -> Result<ExpNode, &'static str>
|
||||
{
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens but expected primary expression");
|
||||
}
|
||||
match &tokens[*i]
|
||||
{
|
||||
Token::Nil =>
|
||||
{
|
||||
*i += 1;
|
||||
Ok(ExpNode::Nil)
|
||||
},
|
||||
Token::True =>
|
||||
{
|
||||
*i += 1;
|
||||
Ok(ExpNode::True)
|
||||
},
|
||||
Token::False =>
|
||||
{
|
||||
*i += 1;
|
||||
Ok(ExpNode::False)
|
||||
},
|
||||
Token::Numeral(number_str) =>
|
||||
{
|
||||
*i += 1;
|
||||
Ok(ExpNode::Numeral(number_str.parse::<f64>().map_err(|_| "Could not parse number")?))
|
||||
},
|
||||
Token::StringLiteral(string) =>
|
||||
{
|
||||
*i += 1;
|
||||
Ok(ExpNode::LiteralString(string.clone()))
|
||||
},
|
||||
Token::DotDotDot =>
|
||||
{
|
||||
*i += 1;
|
||||
Ok(ExpNode::Varargs)
|
||||
},
|
||||
Token::Function =>
|
||||
{
|
||||
*i += 1;
|
||||
Ok(ExpNode::Functiondef(parse_funcbody(tokens, i)?))
|
||||
}
|
||||
Token::CurlyOpen => Ok(ExpNode::Tableconstructor(parse_tableconstructor(tokens, i)?)),
|
||||
Token::Minus =>
|
||||
{
|
||||
Ok(ExpNode::Unop(UnopType::Minus, Box::new(parse_exp(tokens, i)?)))
|
||||
}
|
||||
Token::Hash =>
|
||||
{
|
||||
Ok(ExpNode::Unop(UnopType::Length, Box::new(parse_exp(tokens, i)?)))
|
||||
}
|
||||
Token::Not =>
|
||||
{
|
||||
Ok(ExpNode::Unop(UnopType::LogicalNot, Box::new(parse_exp(tokens, i)?)))
|
||||
}
|
||||
Token::Tilde =>
|
||||
{
|
||||
Ok(ExpNode::Unop(UnopType::BinaryNot, Box::new(parse_exp(tokens, i)?)))
|
||||
}
|
||||
_ => Ok(ExpNode::Suffixexp(Box::new(parse_suffixexp(tokens, i)?))),
|
||||
}
|
||||
}
|
||||
fn parse_tableconstructor(tokens: &Vec<Token>, i: &mut usize) -> Result<TableconstructorNode, &'static str>
|
||||
{
|
||||
todo!()
|
||||
}
|
||||
fn parse_explist(tokens: &Vec<Token>, i: &mut usize) -> Result<ExplistNode, &'static str>
|
||||
{
|
||||
let mut exps: Vec<ExpNode> = Vec::from([parse_exp(tokens, i)?]);
|
||||
while *i < tokens.len() && tokens[*i] == Token::Comma
|
||||
{
|
||||
*i += 1;
|
||||
exps.push(parse_exp(tokens, i)?);
|
||||
}
|
||||
return Ok(ExplistNode { exps });
|
||||
}
|
||||
fn parse_funcname(tokens: &Vec<Token>, i: &mut usize) -> Result<FuncnameNode, &'static str>
|
||||
{
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing funcname");
|
||||
}
|
||||
if let Token::Name(name) = &tokens[*i]
|
||||
{
|
||||
*i += 1;
|
||||
let mut dotted_names = Vec::new();
|
||||
while *i < tokens.len() && tokens[*i] == Token::Dot
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing dotted part of funcname");
|
||||
}
|
||||
if let Token::Name(dotted_name) = &tokens[*i]
|
||||
{
|
||||
*i += 1;
|
||||
dotted_names.push(dotted_name.clone());
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("Expected name in dotted funcname");
|
||||
}
|
||||
}
|
||||
let first_arg = if *i < tokens.len() && tokens[*i] == Token::Colon
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing funcname first arg");
|
||||
}
|
||||
if let Token::Name(arg_name) = &tokens[*i]
|
||||
{
|
||||
*i += 1;
|
||||
Some(arg_name.clone())
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("Expected name of first arg in funcname");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
None
|
||||
};
|
||||
return Ok(FuncnameNode { name: name.clone(), dotted_names, first_arg });
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("Expected func name");
|
||||
}
|
||||
}
|
||||
fn parse_funcbody(tokens: &Vec<Token>, i: &mut usize) -> Result<FuncbodyNode, &'static str>
|
||||
{
|
||||
if *i >= tokens.len() || tokens[*i] != Token::RoundOpen
|
||||
{
|
||||
return Err("Expected '(' to start funcbody");
|
||||
}
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing funcbody parlist");
|
||||
}
|
||||
let pars = if tokens[*i] == Token::RoundClosed
|
||||
{
|
||||
*i += 1;
|
||||
None
|
||||
}
|
||||
else
|
||||
{
|
||||
let ret = Some(parse_parlist(tokens, i)?);
|
||||
if *i >= tokens.len() || tokens[*i] != Token::RoundClosed
|
||||
{
|
||||
return Err("Expected ')' to close funcbody parlist");
|
||||
}
|
||||
*i += 1;
|
||||
ret
|
||||
};
|
||||
let block = parse_block(tokens, i)?;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::End
|
||||
{
|
||||
println!("{:?}", &tokens[(*i - 10)..(*i + 10)]);
|
||||
return Err("Expected 'end' to close funcbody");
|
||||
}
|
||||
*i += 1;
|
||||
return Ok(FuncbodyNode { pars, body: block });
|
||||
}
|
||||
fn parse_parlist(tokens: &Vec<Token>, i: &mut usize) -> Result<ParlistNode, &'static str>
|
||||
{
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing parlist");
|
||||
}
|
||||
if tokens[*i] == Token::DotDotDot
|
||||
{
|
||||
*i += 1;
|
||||
return Ok(ParlistNode { names: Vec::new(), has_varargs: true });
|
||||
}
|
||||
let first_name = if let Token::Name(name) = &tokens[*i]
|
||||
{
|
||||
*i += 1;
|
||||
name.clone()
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("Expected name to start parlist");
|
||||
};
|
||||
let mut names = Vec::from([first_name]);
|
||||
let mut has_varargs = false;
|
||||
while *i < tokens.len() && tokens[*i] == Token::Comma
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens while parsing parlist name list");
|
||||
}
|
||||
match &tokens[*i]
|
||||
{
|
||||
Token::Name(name) =>
|
||||
{
|
||||
*i += 1;
|
||||
names.push(name.clone());
|
||||
}
|
||||
Token::DotDotDot =>
|
||||
{
|
||||
*i += 1;
|
||||
has_varargs = true;
|
||||
break;
|
||||
}
|
||||
_ => return Err("Unexpected token while parsing parlist name list"),
|
||||
}
|
||||
}
|
||||
return Ok(ParlistNode { names, has_varargs });
|
||||
}
|
||||
fn parse_attnamelist(tokens: &Vec<Token>, i: &mut usize) -> Result<AttnamelistNode, &'static str>
|
||||
{
|
||||
let mut attnames: Vec<AttnameNode> = Vec::from([parse_attname(tokens, i)?]);
|
||||
while *i < tokens.len() && tokens[*i] == Token::Comma
|
||||
{
|
||||
*i += 1;
|
||||
attnames.push(parse_attname(tokens, i)?);
|
||||
}
|
||||
return Ok(AttnamelistNode { attnames });
|
||||
}
|
||||
fn parse_attname(tokens: &Vec<Token>, i: &mut usize) -> Result<AttnameNode, &'static str>
|
||||
{
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens but expected name for attrib name");
|
||||
}
|
||||
if let Token::Name(name) = &tokens[*i]
|
||||
{
|
||||
*i += 1;
|
||||
let attribute = if *i < tokens.len() && tokens[*i] == Token::Lt
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len()
|
||||
{
|
||||
return Err("Reached end of tokens but expected attribute");
|
||||
}
|
||||
if let Token::Name(attrib) = &tokens[*i]
|
||||
{
|
||||
*i += 1;
|
||||
if *i >= tokens.len() || tokens[*i] != Token::Gt
|
||||
{
|
||||
return Err("Exptected '>' to close attribute name");
|
||||
}
|
||||
Some(attrib.clone())
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("Expected attribute in attrib name");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
None
|
||||
};
|
||||
return Ok(AttnameNode { name: name.clone(), attribute });
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("Expected name for attrib name");
|
||||
}
|
||||
}
|
||||
//===============================================================================================================================================
|
||||
//===============================================================================================================================================
|
||||
//===============================================================================================================================================
|
||||
//===============================================================================================================================================
|
||||
//===============================================================================================================================================
|
||||
//===============================================================================================================================================
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct Node
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct AmbiguousNode
|
||||
{
|
||||
|
||||
}
|
||||
pub fn cyk(tokens: Vec<Token>) -> Result<ChunkNode, &'static str>
|
||||
{
|
||||
let r = NONTERMINAL_NAMES.len();
|
||||
let n = tokens.len();
|
||||
macro_rules! index {
|
||||
($x:expr, $y:expr, $z:expr) => {
|
||||
($x + $y * n + ($z as usize) * n * n)
|
||||
};
|
||||
}
|
||||
let mut p = vec![false; n * n * r];
|
||||
//let mut back: Vec<Vec<(usize, u8, u8)>> = vec![Vec::new(); n * n * r];
|
||||
println!("{n}, {r}, {}", p.len());
|
||||
for s in 0..n
|
||||
{
|
||||
for (index, token) in TERMINAL_RULES
|
||||
{
|
||||
if let Token::Name(_) = tokens[s]
|
||||
{
|
||||
if let Token::Name(_) = token
|
||||
{
|
||||
p[index!(0, s, index)] = true
|
||||
}
|
||||
}
|
||||
else if let Token::StringLiteral(_) = tokens[s]
|
||||
{
|
||||
if let Token::StringLiteral(_) = token
|
||||
{
|
||||
p[index!(0, s, index)] = true
|
||||
}
|
||||
}
|
||||
else if let Token::Numeral(_) = tokens[s]
|
||||
{
|
||||
if let Token::Numeral(_) = token
|
||||
{
|
||||
p[index!(0, s, index)] = true
|
||||
}
|
||||
}
|
||||
else if token == tokens[s]
|
||||
{
|
||||
p[index!(0, s, index)] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("Done initializing");
|
||||
|
||||
for l in 2..=n
|
||||
{
|
||||
for s in 1..=(n - l + 1)
|
||||
{
|
||||
for _p in 1..=(l-1)
|
||||
{
|
||||
for &(a, b, c) in &NONTERMINAL_RULES
|
||||
{
|
||||
if p[index!(_p - 1, s - 1, b)] && p[index!(l - _p - 1, s + _p - 1, c)]
|
||||
{
|
||||
let index = index!(l - 1, s - 1, a);
|
||||
p[index] = true;
|
||||
/* if !back[index].contains(&(_p, b, c))
|
||||
{
|
||||
back[index].push((_p, b, c));
|
||||
}*/
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("{l}");
|
||||
}
|
||||
let start_index = NONTERMINAL_NAMES.iter().position(|x| x == &"S_0").expect("no start index found");
|
||||
if p[index!(n - 1, 0, start_index)]
|
||||
{
|
||||
println!("Is part of the language");
|
||||
todo!()
|
||||
//return Ok(disambiguate(traverse_back(back, tokens, n, 1, start_index)));
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("Input is not part of the language")
|
||||
}
|
||||
}
|
||||
|
||||
fn traverse_back(back: Vec<Vec<(usize, u8, u8)>>, tokens: Vec<Token>, l: usize, s: usize, a: usize) -> AmbiguousNode
|
||||
{
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn disambiguate(root: AmbiguousNode) -> Node
|
||||
{
|
||||
todo!()
|
||||
}
|
||||
2572
src/parser.zig
2572
src/parser.zig
@ -1,2572 +0,0 @@
|
||||
const Token = @import("tokenizer.zig").Token;
|
||||
const TokenType = @import("tokenizer.zig").TokenType;
|
||||
const std = @import("std");
|
||||
const types = @import("types.zig");
|
||||
const CodeRegion = @import("types.zig").CodeRegion;
|
||||
|
||||
pub const ChunkNode = struct
|
||||
{
|
||||
block: BlockNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
pub fn dump(self: *const ChunkNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("ChunkNode ({} - {}):\n", .{self.startRegion, self.endRegion});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("block: ", .{});
|
||||
self.block.dump(indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
|
||||
pub const BlockNode = struct
|
||||
{
|
||||
stats: std.ArrayList(StatNode),
|
||||
retstat: ?RetstatNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const BlockNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("BlockNode:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("stats:\n", .{});
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("[\n", .{});
|
||||
for(self.stats.items) |stat|
|
||||
{
|
||||
for (0..indent + 2) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
dumpStatNode(stat, indent + 2);
|
||||
}
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("]\n", .{});
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("retstat: ", .{});
|
||||
if(self.retstat == null)
|
||||
{
|
||||
std.debug.print("null\n", .{});
|
||||
}
|
||||
else
|
||||
{
|
||||
self.retstat.?.dump(indent + 1);
|
||||
}
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const RetstatNode = struct
|
||||
{
|
||||
values: ?ExplistNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const RetstatNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Retstat Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("values: ", .{});
|
||||
if(self.values == null)
|
||||
{
|
||||
std.debug.print("null\n", .{});
|
||||
}
|
||||
else
|
||||
{
|
||||
self.values.?.dump(indent + 1);
|
||||
}
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
|
||||
pub const StatNode = union(enum)
|
||||
{
|
||||
Semicolon,
|
||||
Assignment: AssignmentNode,
|
||||
Functioncall: FunctioncallNode,
|
||||
Label: []u8,
|
||||
Break,
|
||||
Goto: []u8,
|
||||
Do: BlockNode,
|
||||
While: WhileNode,
|
||||
Repeat: RepeatNode,
|
||||
If: IfNode,
|
||||
ForNumerical: ForNumericalNode,
|
||||
ForGeneric: ForGenericNode,
|
||||
Function: FunctionNode,
|
||||
LocalFunction: LocalFunctionNode,
|
||||
Local: LocalNode,
|
||||
};
|
||||
fn dumpStatNode(stat: StatNode, indent: usize) void
|
||||
{
|
||||
switch(stat)
|
||||
{
|
||||
.Semicolon => std.debug.print("Semicolon\n", .{}),
|
||||
.Assignment => |*node| node.dump(indent),
|
||||
.Functioncall => |*node| node.dump(indent),
|
||||
.Label => |*label| std.debug.print("Label: '{s}'\n", .{label.*}),
|
||||
.Break => std.debug.print("Break\n", .{}),
|
||||
.Goto => |*label| std.debug.print("Goto: '{s}'\n", .{label.*}),
|
||||
.Do => |*node| node.dump(indent),
|
||||
.While => |*node| node.dump(indent),
|
||||
.Repeat => |*node| node.dump(indent),
|
||||
.If => |*node| node.dump(indent),
|
||||
.ForNumerical => |*node| node.dump(indent),
|
||||
.ForGeneric => |*node| node.dump(indent),
|
||||
.Function => |*node| node.dump(indent),
|
||||
.LocalFunction => |*node| node.dump(indent),
|
||||
.Local => |*node| node.dump(indent),
|
||||
}
|
||||
}
|
||||
|
||||
pub const AssignmentNode = struct
|
||||
{
|
||||
lhs: VarlistNode, rhs: ExplistNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const AssignmentNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Assignment Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("lhs: ", .{});
|
||||
self.lhs.dump(indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("rhs: ", .{});
|
||||
self.rhs.dump(indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const WhileNode = struct
|
||||
{
|
||||
condition: ExpNode,
|
||||
body: BlockNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const WhileNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("While Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("condition: ", .{});
|
||||
dumpExpNode(self.condition, indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("body: ", .{});
|
||||
self.body.dump(indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const RepeatNode = struct
|
||||
{
|
||||
condition: ExpNode,
|
||||
body: BlockNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const RepeatNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Repeat Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("condition: ", .{});
|
||||
dumpExpNode(self.condition, indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("body: ", .{});
|
||||
self.body.dump(indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const IfNode = struct
|
||||
{
|
||||
condition: ExpNode,
|
||||
body: BlockNode,
|
||||
elseifs: std.ArrayList(ElseifNode),
|
||||
else_: ?BlockNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const IfNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("If Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("condition: ", .{});
|
||||
dumpExpNode(self.condition, indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("body: ", .{});
|
||||
self.body.dump(indent + 1);
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("elseifs:\n", .{});
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("[\n", .{});
|
||||
for(self.elseifs.items) |elseif|
|
||||
{
|
||||
for (0..indent + 2) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
elseif.dump(indent + 2);
|
||||
}
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("]\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("else: ", .{});
|
||||
if(self.else_ == null)
|
||||
{
|
||||
std.debug.print("null\n", .{});
|
||||
}
|
||||
else
|
||||
{
|
||||
self.else_.?.dump(indent + 1);
|
||||
}
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const ForNumericalNode = struct
|
||||
{
|
||||
variable: []u8,
|
||||
start: ExpNode,
|
||||
end: ExpNode,
|
||||
change: ?ExpNode,
|
||||
body: BlockNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const ForNumericalNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("For Numerical Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("variable: '{s}'\n", .{self.variable});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("start: ", .{});
|
||||
dumpExpNode(self.start, indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("end: ", .{});
|
||||
dumpExpNode(self.end, indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("change: ", .{});
|
||||
if(self.change == null)
|
||||
{
|
||||
std.debug.print("null\n", .{});
|
||||
}
|
||||
else
|
||||
{
|
||||
dumpExpNode(self.start, indent + 1);
|
||||
}
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("body: ", .{});
|
||||
self.body.dump(indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const ForGenericNode = struct
|
||||
{
|
||||
vars: std.ArrayList([]u8),
|
||||
exps: ExplistNode,
|
||||
body: BlockNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const ForGenericNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("For Generic Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("vars:\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("[\n", .{});
|
||||
for(self.vars.items) |v|
|
||||
{
|
||||
for (0..(indent + 2)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("'{s}'\n", .{v});
|
||||
}
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("]\n", .{});
|
||||
std.debug.print("exps: ", .{});
|
||||
self.exps.dump(indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("body: ", .{});
|
||||
self.body.dump(indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const FunctionNode = struct
|
||||
{
|
||||
name: FuncnameNode,
|
||||
body: FuncbodyNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const FunctionNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Function Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("name: ", .{});
|
||||
self.name.dump(indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("body: ", .{});
|
||||
self.body.dump(indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const LocalFunctionNode = struct
|
||||
{
|
||||
name: []u8,
|
||||
body: FuncbodyNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const LocalFunctionNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Local Function Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("name: '{s}'\n", .{self.name});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("body: ", .{});
|
||||
self.body.dump(indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
|
||||
};
|
||||
pub const LocalNode = struct
|
||||
{
|
||||
attnames: AttnamelistNode,
|
||||
values: ?ExplistNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const LocalNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Local Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("attnames: ", .{});
|
||||
self.attnames.dump(indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("values: ", .{});
|
||||
if(self.values == null)
|
||||
{
|
||||
std.debug.print("null\n", .{});
|
||||
}
|
||||
else
|
||||
{
|
||||
self.values.?.dump(indent + 1);
|
||||
}
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const FunctioncallNode = struct
|
||||
{
|
||||
function: SuffixexpNode,
|
||||
objectArg: ?[]u8,
|
||||
args: ArgsNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const FunctioncallNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Functioncall Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("function: ", .{});
|
||||
dumpSuffixExpNode(self.function, indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("object arg: ", .{});
|
||||
if(self.objectArg == null)
|
||||
{
|
||||
std.debug.print("null\n", .{});
|
||||
}
|
||||
else
|
||||
{
|
||||
std.debug.print("'{s}'\n", .{self.objectArg.?});
|
||||
}
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("args: ", .{});
|
||||
dumpArgsNode(self.args, indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const VarlistNode = struct
|
||||
{
|
||||
vars: std.ArrayList(VarNode),
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const VarlistNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Varlist Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("vars:\n", .{});
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("[\n", .{});
|
||||
for(self.vars.items) |item|
|
||||
{
|
||||
for (0..indent + 2) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
dumpVarNode(item, indent + 2);
|
||||
}
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("]\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const ExplistNode = struct
|
||||
{
|
||||
exps: std.ArrayList(ExpNode),
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const ExplistNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Explist Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("exps:\n", .{});
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("[\n", .{});
|
||||
for(self.exps.items) |item|
|
||||
{
|
||||
for (0..indent + 2) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
dumpExpNode(item, indent + 2);
|
||||
}
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("]\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const ExpNode = union(enum)
|
||||
{
|
||||
Nil,
|
||||
False,
|
||||
True,
|
||||
Numeral: types.Numeral,
|
||||
LiteralString: []u8,
|
||||
Varargs,
|
||||
Functiondef: FuncbodyNode,
|
||||
Suffixexp: *SuffixexpNode,
|
||||
Tableconstructor: TableconstructorNode,
|
||||
Unop: UnopNode,
|
||||
Binop: *BinopNode,
|
||||
};
|
||||
fn dumpExpNode(expNode: ExpNode, indent: usize) void
|
||||
{
|
||||
switch(expNode)
|
||||
{
|
||||
.Nil => std.debug.print("Nil\n", .{}),
|
||||
.False => std.debug.print("False\n", .{}),
|
||||
.True => std.debug.print("True\n", .{}),
|
||||
.Numeral => |*numeral| std.debug.print("{}\n", .{numeral.*}),
|
||||
.LiteralString => |*string| std.debug.print("LiteralString: '{s}'\n", .{string.*}),
|
||||
.Varargs => std.debug.print("Varargs", .{}),
|
||||
.Functiondef => |*node| node.dump(indent),
|
||||
.Suffixexp => |*node| dumpSuffixExpNode(node.*.*, indent),
|
||||
.Tableconstructor => |*node| node.dump(indent),
|
||||
.Unop => |*node| node.dump(indent),
|
||||
.Binop => |*node| node.*.dump(indent),
|
||||
}
|
||||
}
|
||||
pub const UnopNode = struct
|
||||
{
|
||||
unopType: UnopType,
|
||||
exp: *ExpNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const UnopNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Unop Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("unop type: {}\n", .{self.unopType});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("exp: ", .{});
|
||||
dumpExpNode(self.exp.*, indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const BinopNode = struct
|
||||
{
|
||||
lhs: ExpNode,
|
||||
op: BinopType,
|
||||
rhs: ExpNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const BinopNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Binop Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("lhs: ", .{});
|
||||
dumpExpNode(self.lhs, indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("op: {}\n", .{self.op});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("rhs: ", .{});
|
||||
dumpExpNode(self.rhs, indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const ElseifNode = struct
|
||||
{
|
||||
condition: ExpNode,
|
||||
body: BlockNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const ElseifNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Elseif Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("condition: ", .{});
|
||||
dumpExpNode(self.condition, indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("body: ", .{});
|
||||
self.body.dump(indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const FuncnameNode = struct
|
||||
{
|
||||
name: []u8,
|
||||
dottedNames: std.ArrayList([]u8),
|
||||
firstArg: ?[]u8,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const FuncnameNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Funcname Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("name: '{s}'\n", .{self.name});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("dottedNames:\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("[\n", .{});
|
||||
for(self.dottedNames.items) |dottedName|
|
||||
{
|
||||
for (0..(indent + 2)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("'{s}'\n", .{dottedName});
|
||||
}
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("]\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("firstArg: ", .{});
|
||||
if(self.firstArg == null)
|
||||
{
|
||||
std.debug.print("null\n", .{});
|
||||
}
|
||||
else
|
||||
{
|
||||
std.debug.print("'{s}'\n", .{self.firstArg.?});
|
||||
}
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
|
||||
};
|
||||
pub const FuncbodyNode = struct
|
||||
{
|
||||
pars: ?ParlistNode,
|
||||
body: BlockNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const FuncbodyNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Funcbody Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("pars: ", .{});
|
||||
if(self.pars == null)
|
||||
{
|
||||
std.debug.print("null\n", .{});
|
||||
}
|
||||
else
|
||||
{
|
||||
self.pars.?.dump(indent + 1);
|
||||
}
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
self.body.dump(indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const AttnamelistNode = struct
|
||||
{
|
||||
attnames: std.ArrayList(AttnameNode),
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const AttnamelistNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Attnamelist Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("attNames:\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("[\n", .{});
|
||||
for(self.attnames.items) |attNames|
|
||||
{
|
||||
for (0..(indent + 2)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
attNames.dump(indent + 2);
|
||||
}
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("]\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const AttnameNode = struct
|
||||
{
|
||||
name: []u8,
|
||||
attribute: ?[]u8,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const AttnameNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Funcname Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("name: '{s}'\n", .{self.name});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("attribute: ", .{});
|
||||
if(self.attribute == null)
|
||||
{
|
||||
std.debug.print("null\n", .{});
|
||||
}
|
||||
else
|
||||
{
|
||||
std.debug.print("'{s}'\n", .{self.attribute.?});
|
||||
}
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const SuffixexpNode = union(enum)
|
||||
{
|
||||
Normal: NormalSuffixNode,
|
||||
Functioncall: *FunctioncallNode,
|
||||
};
|
||||
fn dumpSuffixExpNode(suffixexpNode: SuffixexpNode, indent: usize) void
|
||||
{
|
||||
switch(suffixexpNode)
|
||||
{
|
||||
.Normal => |*node| node.dump(indent),
|
||||
.Functioncall => |*node| node.*.dump(indent),
|
||||
}
|
||||
}
|
||||
pub const ArgsNode = union(enum)
|
||||
{
|
||||
Bracketed: ?ExplistNode,
|
||||
Tableconstructor: TableconstructorNode,
|
||||
Literal: []u8,
|
||||
};
|
||||
fn dumpArgsNode(argsNode: ArgsNode, indent: usize) void
|
||||
{
|
||||
switch(argsNode)
|
||||
{
|
||||
.Bracketed => |*name|
|
||||
{
|
||||
if(name.* == null)
|
||||
{
|
||||
std.debug.print("null\n", .{});
|
||||
}
|
||||
else
|
||||
{
|
||||
name.*.?.dump(indent);
|
||||
}
|
||||
},
|
||||
.Tableconstructor => |*node| node.dump(indent),
|
||||
.Literal => |*string| std.debug.print("Literal: '{s}'\n", .{string}),
|
||||
}
|
||||
}
|
||||
pub const VarNode = union(enum)
|
||||
{
|
||||
Name: []u8,
|
||||
Indexed: IndexedVarNode,
|
||||
Member: MemberVarNode,
|
||||
};
|
||||
fn dumpVarNode(varNode: VarNode, indent: usize) void
|
||||
{
|
||||
switch(varNode)
|
||||
{
|
||||
.Name => |*name| std.debug.print("Name: '{s}'\n", .{name.*}),
|
||||
.Indexed => |*node| node.dump(indent),
|
||||
.Member => |*node| node.dump(indent),
|
||||
}
|
||||
}
|
||||
pub const IndexedVarNode = struct
|
||||
{
|
||||
value: SuffixexpNode,
|
||||
index: ExpNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const IndexedVarNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Indexed Var Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("value: ", .{});
|
||||
dumpSuffixExpNode(self.value, indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("index: ", .{});
|
||||
dumpExpNode(self.index, indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const MemberVarNode = struct
|
||||
{
|
||||
value: SuffixexpNode,
|
||||
name: []u8,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const MemberVarNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Member Var Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("value: ", .{});
|
||||
dumpSuffixExpNode(self.value, indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("name: '{s}'\n", .{self.name});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
|
||||
};
|
||||
pub const TableconstructorNode = struct
|
||||
{
|
||||
exps: ?FieldlistNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const TableconstructorNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Tableconstructor Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("exps: ", .{});
|
||||
if(self.exps == null)
|
||||
{
|
||||
std.debug.print("null\n", .{});
|
||||
}
|
||||
else
|
||||
{
|
||||
self.exps.?.dump(indent + 1);
|
||||
}
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const UnopType = enum
|
||||
{
|
||||
Minus, LogicalNot, Length, BinaryNot,
|
||||
};
|
||||
pub const BinopType = enum
|
||||
{
|
||||
LogicalOr,
|
||||
LocicalAnd,
|
||||
Lt, Gt, LtEquals, GtEquals, NotEquals, Equals,
|
||||
BinaryOr,
|
||||
BinaryNot,
|
||||
BinaryAnd,
|
||||
Shl, Shr,
|
||||
Concat,
|
||||
Add, Sub,
|
||||
Mul, Div, IntDiv, Mod,
|
||||
Exp,
|
||||
};
|
||||
pub const ParlistNode = struct
|
||||
{
|
||||
names: std.ArrayList([]u8),
|
||||
hasVarargs: bool,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const ParlistNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Parlist Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("names:\n", .{});
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("[\n", .{});
|
||||
for(self.names.items) |name|
|
||||
{
|
||||
for (0..indent + 2) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("'{s}'\n", .{name});
|
||||
}
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("]\n", .{});
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("has Varargs: {}\n", .{self.hasVarargs});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const NormalSuffixNode = struct
|
||||
{
|
||||
firstPart: SuffixexpFirstPart,
|
||||
suffixes: std.ArrayList(SuffixexpSuffix),
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const NormalSuffixNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Normal Suffix Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("First Part: ", .{});
|
||||
dumpSuffixExpFirstPart(self.firstPart, indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("Suffixes:\n", .{});
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("[\n", .{});
|
||||
for(self.suffixes.items) |suffix|
|
||||
{
|
||||
for (0..(indent + 2)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
dumpSuffixSuffix(suffix, indent + 2);
|
||||
}
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("]\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const SuffixexpFirstPart = union(enum)
|
||||
{
|
||||
Name: []u8,
|
||||
BracketedExpr: ExpNode,
|
||||
};
|
||||
fn dumpSuffixExpFirstPart(suffixexpFirstPart: SuffixexpFirstPart, indent: usize) void
|
||||
{
|
||||
switch(suffixexpFirstPart)
|
||||
{
|
||||
.Name => |*name| std.debug.print("Name: '{s}'\n", .{name.*}),
|
||||
.BracketedExpr => |*node| dumpExpNode(node.*, indent),
|
||||
}
|
||||
}
|
||||
pub const SuffixexpSuffix = union(enum)
|
||||
{
|
||||
Dot: []u8,
|
||||
Indexed: ExpNode,
|
||||
Args: ArgsNode,
|
||||
ArgsFirstArg: ArgsFirstArgNode,
|
||||
};
|
||||
fn dumpSuffixSuffix(suffixexpSuffix: SuffixexpSuffix, indent: usize) void
|
||||
{
|
||||
switch(suffixexpSuffix)
|
||||
{
|
||||
.Dot => |*name| std.debug.print("Dot: '{s}'\n", .{name.*}),
|
||||
.Indexed => |*node| dumpExpNode(node.*, indent),
|
||||
.Args => |*node| dumpArgsNode(node.*, indent),
|
||||
.ArgsFirstArg => |*node| node.dump(indent),
|
||||
}
|
||||
}
|
||||
|
||||
pub const ArgsFirstArgNode = struct
|
||||
{
|
||||
name: []u8,
|
||||
rest: ArgsNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const ArgsFirstArgNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Args First Arg Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("name: '{s}'\n", .{self.name});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("rest: ", .{});
|
||||
dumpArgsNode(self.rest, indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
|
||||
};
|
||||
pub const FieldlistNode = struct
|
||||
{
|
||||
exps: std.ArrayList(FieldNode),
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const FieldlistNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Fieldlist Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("exps: ", .{});
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("[\n", .{});
|
||||
for(self.exps.items) |exp|
|
||||
{
|
||||
for (0..(indent + 2)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
dumpFieldNode(exp, indent + 2);
|
||||
}
|
||||
for (0..indent + 1) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("]\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
pub const FieldNode = union(enum)
|
||||
{
|
||||
IndexedAssignment: IndexedAssignmentNode,
|
||||
Assignment: FieldAssignmentNode,
|
||||
Exp: ExpNode,
|
||||
};
|
||||
pub const FieldAssignmentNode = struct
|
||||
{
|
||||
lhs: []u8,
|
||||
rhs: ExpNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const FieldAssignmentNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Field Assignment Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("lhs: {s}\n", .{self.lhs});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("rhs: ", .{});
|
||||
dumpExpNode(self.rhs, indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
fn dumpFieldNode(fieldNode: FieldNode, indent: usize) void
|
||||
{
|
||||
switch(fieldNode)
|
||||
{
|
||||
.IndexedAssignment => |*node| node.dump(indent),
|
||||
.Assignment => |*node| node.dump(indent),
|
||||
.Exp => |*node| dumpExpNode(node.*, indent),
|
||||
}
|
||||
}
|
||||
pub const IndexedAssignmentNode = struct
|
||||
{
|
||||
index: ExpNode,
|
||||
rhs: ExpNode,
|
||||
startRegion: CodeRegion,
|
||||
endRegion: CodeRegion,
|
||||
|
||||
fn dump(self: *const IndexedAssignmentNode, indent: usize) void
|
||||
{
|
||||
std.debug.print("Indexed Assignment Node:\n", .{});
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("{{\n", .{});
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("index: ", .{});
|
||||
dumpExpNode(self.index, indent + 1);
|
||||
for (0..(indent + 1)) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("rhs: ", .{});
|
||||
dumpExpNode(self.rhs, indent + 1);
|
||||
for (0..indent) |_|
|
||||
{
|
||||
std.debug.print("\t", .{});
|
||||
}
|
||||
std.debug.print("}}\n", .{});
|
||||
}
|
||||
};
|
||||
|
||||
pub fn parse(tokens: []Token, allocator: *std.heap.ArenaAllocator) !ChunkNode
|
||||
{
|
||||
var i: usize = 0;
|
||||
const maybeParsedChunk = parseChunk(tokens, &i, allocator) catch |err|
|
||||
{
|
||||
//std.debug.print("{any}: data: {any}, type: {any}\n", .{tokens[i].region, tokens[i].tokenData, tokens[i].tokenType});
|
||||
return err;
|
||||
};
|
||||
return maybeParsedChunk;
|
||||
}
|
||||
|
||||
const ParserError = error
|
||||
{
|
||||
NotImplemented,
|
||||
ReachedEOFParsing,
|
||||
ReachedEOFExpectedNameForGoto,
|
||||
ExpectedNameForGoto,
|
||||
MissingEndForDoBlock,
|
||||
MissingDoAfterWhileCondition,
|
||||
MissingEndForWhileBody,
|
||||
ExpectedUntilAfterRepeatBody,
|
||||
ExpectedThenAfterIfCondition,
|
||||
ReachedEOFAfterIfBody,
|
||||
ExpectedThenAfterElseifCondition,
|
||||
ReachedEOFAfterElseifs,
|
||||
ExpectedEndClosingIf,
|
||||
ExpectedNameAfterFor,
|
||||
ExpectedCommaAfterForEqStartValue,
|
||||
ReachedEOFAfterForEqEndValue,
|
||||
ExpectedDoAfterForEqHead,
|
||||
ExpectedAnotherNameInForInNamelist,
|
||||
ReachedEOFAfterNameInForInNamelist,
|
||||
ExpectedInAfterForInNamelist,
|
||||
ExpectedDoAfterForInExplist,
|
||||
ExpectedEndAfterForInBody,
|
||||
ExpectedEndAfterForEqBody,
|
||||
UnexpectedTokenAfterFirstNameInFor,
|
||||
ReachedEOFInLocal,
|
||||
ExpectedLocalFunctionName,
|
||||
ExpectedNameAfterDoubleColonInLabelDeclaration,
|
||||
ExpectedDoubleColonAfterNameInLabelDeclaration,
|
||||
ExpectedFunctioncall,
|
||||
ExpectedEqAfterAssignmentVarList,
|
||||
ReachedEOFInSuffixExp,
|
||||
ExpectedRoundClosedClosingBracketedPrimaryExp,
|
||||
UnexpectedTokenAsFirstPartOfSuffixExp,
|
||||
ExpectedNameInDottedSuffixExp,
|
||||
ExpectedSquareClosedClosingIndexedSuffixExp,
|
||||
ExpectedNameInArgsFirstArgSuffixExp,
|
||||
ExpectedDotOrIndexedSuffixWhenConvertingSuffixExpToVar,
|
||||
ReachedEOFExpectedPrimaryExpression,
|
||||
ReachedEOFInArgs,
|
||||
ReachedEOFInBracketedArgs,
|
||||
ExpectedRoundClosedClosingBracketedArgs,
|
||||
UnexpectedTokenInArgs,
|
||||
NoPrecedenceForOperator,
|
||||
NoBinopTypeForOperator,
|
||||
ExpectednameInAttribName,
|
||||
ExpectedAttributeInAttrib,
|
||||
ExpectedGtInAttrib,
|
||||
ExpectedFuncname,
|
||||
ExpectedNameInDottedFuncname,
|
||||
ExpectedNameOfFirstArgInFuncname,
|
||||
ExpectedRoundOpenStartingFuncbody,
|
||||
ReachedEOFInFuncbodyParlist,
|
||||
ExpectedRoundClosedClosingFuncbodyParlist,
|
||||
ExpectedEndClosingFuncbody,
|
||||
ReachedEOFInParlist,
|
||||
ExpectedNameStartingParlist,
|
||||
ReachedEOFInParlistNameList,
|
||||
UnexpectedTokenInParlistNameList,
|
||||
ExpectedReturnStartingRetstat,
|
||||
ExpectedCurlyOpenOpeningTableconstructor,
|
||||
ExpectedCurlyClosedClosingTableconstructor,
|
||||
ReachedEOFInField,
|
||||
ExpectedSquareClosedClosingIndexedField,
|
||||
ExpectedEqualsInIndexedFieldExpression,
|
||||
OutOfMemory,
|
||||
};
|
||||
|
||||
fn parseChunk(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ChunkNode
|
||||
{
|
||||
const block = try parseBlock(tokens, i, allocator);
|
||||
return ChunkNode { .block = block, .startRegion = block.startRegion, .endRegion = block.endRegion };
|
||||
}
|
||||
fn parseBlock(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) ParserError!BlockNode
|
||||
{
|
||||
var ret = BlockNode { .stats = std.ArrayList(StatNode).init(allocator.*.allocator()), .retstat = null, .startRegion = tokens[i.*].region, .endRegion = tokens[i.*].region };
|
||||
while(i.* < tokens.len and
|
||||
tokens[i.*].tokenType != TokenType.Return and
|
||||
tokens[i.*].tokenType != TokenType.End and
|
||||
tokens[i.*].tokenType != TokenType.Elseif and
|
||||
tokens[i.*].tokenType != TokenType.Else
|
||||
)
|
||||
{
|
||||
try ret.stats.append(try parseStat(tokens, i, allocator));
|
||||
}
|
||||
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Return)
|
||||
{
|
||||
ret.retstat = try parseRetstat(tokens, i, allocator);
|
||||
}
|
||||
ret.endRegion = if(i.* - 1 < tokens.len) tokens[i.* - 1].region else tokens[tokens.len - 1].region;
|
||||
return ret;
|
||||
}
|
||||
fn parseStat(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !StatNode
|
||||
{
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFParsing;
|
||||
}
|
||||
switch(tokens[i.*].tokenType)
|
||||
{
|
||||
TokenType.Semicolon =>
|
||||
{
|
||||
i.* += 1;
|
||||
return StatNode.Semicolon;
|
||||
},
|
||||
TokenType.Break =>
|
||||
{
|
||||
i.* += 1;
|
||||
return StatNode.Break;
|
||||
},
|
||||
TokenType.Goto =>
|
||||
{
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFExpectedNameForGoto;
|
||||
}
|
||||
if(tokens[i.*].tokenType == TokenType.Name)
|
||||
{
|
||||
const name = tokens[i.*].tokenData.string;
|
||||
i.* += 1;
|
||||
return StatNode { .Goto = name };
|
||||
}
|
||||
return error.ExpectedNameForGoto;
|
||||
},
|
||||
TokenType.Do =>
|
||||
{
|
||||
i.* += 1;
|
||||
const body = try parseBlock(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
|
||||
{
|
||||
return error.MissingEndForDoBlock;
|
||||
}
|
||||
i.* += 1;
|
||||
return StatNode { .Do = body };
|
||||
},
|
||||
TokenType.While =>
|
||||
{
|
||||
const startRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
const condition = try parseExp(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Do)
|
||||
{
|
||||
return error.MissingDoAfterWhileCondition;
|
||||
}
|
||||
const body = try parseBlock(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
|
||||
{
|
||||
return error.MissingEndForWhileBody;
|
||||
}
|
||||
const endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
return StatNode { .While = WhileNode { .body = body, .condition = condition, .startRegion = startRegion, .endRegion = endRegion } };
|
||||
},
|
||||
TokenType.Repeat =>
|
||||
{
|
||||
const startRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
const body = try parseBlock(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Until)
|
||||
{
|
||||
return error.ExpectedUntilAfterRepeatBody;
|
||||
}
|
||||
const endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
return StatNode { .Repeat = RepeatNode { .body = body, .condition = try parseExp(tokens, i, allocator), .startRegion = startRegion, .endRegion = endRegion } };
|
||||
},
|
||||
TokenType.If =>
|
||||
{
|
||||
const startRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
const condition = try parseExp(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Then)
|
||||
{
|
||||
return error.ExpectedThenAfterIfCondition;
|
||||
}
|
||||
i.* += 1;
|
||||
const body = try parseBlock(tokens, i, allocator);
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFAfterIfBody;
|
||||
}
|
||||
var ifNode = IfNode { .body = body, .condition = condition, .elseifs = std.ArrayList(ElseifNode).init(allocator.*.allocator()), .else_ = null, .startRegion = startRegion, .endRegion = startRegion };
|
||||
while(tokens[i.*].tokenType == TokenType.Elseif)
|
||||
{
|
||||
i.* += 1;
|
||||
const elseifCondition = try parseExp(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Then)
|
||||
{
|
||||
return error.ExpectedThenAfterElseifCondition;
|
||||
}
|
||||
const endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
try ifNode.elseifs.append(ElseifNode { .body = try parseBlock(tokens, i, allocator), .condition = elseifCondition, .startRegion = startRegion, .endRegion = endRegion });
|
||||
}
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFAfterElseifs;
|
||||
}
|
||||
if(tokens[i.*].tokenType == TokenType.Else)
|
||||
{
|
||||
i.* += 1;
|
||||
ifNode.else_ = try parseBlock(tokens, i, allocator);
|
||||
}
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
|
||||
{
|
||||
return error.ExpectedEndClosingIf;
|
||||
}
|
||||
ifNode.endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
return StatNode { .If = ifNode };
|
||||
},
|
||||
TokenType.For =>
|
||||
{
|
||||
const startRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
|
||||
{
|
||||
return error.ExpectedNameAfterFor;
|
||||
}
|
||||
const variable = tokens[i.*].tokenData.string;
|
||||
i.* += 1;
|
||||
switch(tokens[i.*].tokenType)
|
||||
{
|
||||
TokenType.Equals =>
|
||||
{
|
||||
i.* += 1;
|
||||
const start = try parseExp(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Comma)
|
||||
{
|
||||
return error.ExpectedCommaAfterForEqStartValue;
|
||||
}
|
||||
i.* += 1;
|
||||
const end = try parseExp(tokens, i, allocator);
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFAfterForEqEndValue;
|
||||
}
|
||||
var change: ?ExpNode = null;
|
||||
if(tokens[i.*].tokenType == TokenType.Comma)
|
||||
{
|
||||
i.* += 1;
|
||||
change = try parseExp(tokens, i, allocator);
|
||||
}
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Do)
|
||||
{
|
||||
return error.ExpectedDoAfterForEqHead;
|
||||
}
|
||||
i.* += 1;
|
||||
const body = try parseBlock(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
|
||||
{
|
||||
return error.ExpectedEndAfterForEqBody;
|
||||
}
|
||||
const endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
return StatNode { .ForNumerical = ForNumericalNode { .variable = variable, .start = start, .end = end, .change = change, .body = body, .startRegion = startRegion, .endRegion = endRegion } };
|
||||
},
|
||||
TokenType.Comma =>
|
||||
{
|
||||
var names = std.ArrayList([]u8).init(allocator.*.allocator());
|
||||
try names.append(variable);
|
||||
while(tokens[i.*].tokenType == TokenType.Comma)
|
||||
{
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
|
||||
{
|
||||
return error.ExpectedAnotherNameInForInNamelist;
|
||||
}
|
||||
try names.append(tokens[i.*].tokenData.string);
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFAfterNameInForInNamelist;
|
||||
}
|
||||
}
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.In)
|
||||
{
|
||||
return error.ExpectedInAfterForInNamelist;
|
||||
}
|
||||
i.* += 1;
|
||||
const exps = try parseExplist(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Do)
|
||||
{
|
||||
return error.ExpectedDoAfterForInExplist;
|
||||
}
|
||||
i.* += 1;
|
||||
const body = try parseBlock(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
|
||||
{
|
||||
return error.ExpectedEndAfterForInBody;
|
||||
}
|
||||
const endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
return StatNode { .ForGeneric = ForGenericNode { .vars = names, .exps = exps, .body = body, .startRegion = startRegion, .endRegion = endRegion } };
|
||||
},
|
||||
TokenType.In =>
|
||||
{
|
||||
i.* += 1;
|
||||
const exps = try parseExplist(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Do)
|
||||
{
|
||||
return error.ExpectedDoAfterForInExplist;
|
||||
}
|
||||
i.* += 1;
|
||||
const body = try parseBlock(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
|
||||
{
|
||||
return error.ExpectedEndAfterForInBody;
|
||||
}
|
||||
const endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
var names = try std.ArrayList([]u8).initCapacity(allocator.allocator(), 1);
|
||||
try names.insert(0, variable);
|
||||
return StatNode { .ForGeneric = ForGenericNode { .vars = names, .exps = exps, .body = body, .startRegion = startRegion, .endRegion = endRegion } };
|
||||
},
|
||||
else => return error.UnexpectedTokenAfterFirstNameInFor,
|
||||
}
|
||||
},
|
||||
TokenType.Function =>
|
||||
{
|
||||
const startRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
const name = try parseFuncname(tokens, i, allocator);
|
||||
const body = try parseFuncbody(tokens, i, allocator);
|
||||
return StatNode { .Function = FunctionNode { .name = name, .body = body, .startRegion = startRegion, .endRegion = body.endRegion } };
|
||||
},
|
||||
TokenType.Local =>
|
||||
{
|
||||
const startRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFInLocal;
|
||||
}
|
||||
if(tokens[i.*].tokenType == TokenType.Function)
|
||||
{
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
|
||||
{
|
||||
return error.ExpectedLocalFunctionName;
|
||||
}
|
||||
const name = tokens[i.*].tokenData.string;
|
||||
const endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
return StatNode { .LocalFunction = LocalFunctionNode { .name = name, .body = try parseFuncbody(tokens, i, allocator), .startRegion = startRegion, .endRegion = endRegion } };
|
||||
}
|
||||
else
|
||||
{
|
||||
var ret = LocalNode { .attnames = try parseAttnamelist(tokens, i, allocator), .values = null, .startRegion = startRegion, .endRegion = startRegion };
|
||||
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Equals)
|
||||
{
|
||||
i.* += 1;
|
||||
ret.values = try parseExplist(tokens, i, allocator);
|
||||
ret.endRegion = ret.values.?.endRegion;
|
||||
}
|
||||
return StatNode { .Local = ret };
|
||||
}
|
||||
},
|
||||
TokenType.ColonColon =>
|
||||
{
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
|
||||
{
|
||||
return error.ExpectedNameAfterDoubleColonInLabelDeclaration;
|
||||
}
|
||||
const name = tokens[i.*].tokenData.string;
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.ColonColon)
|
||||
{
|
||||
return error.ExpectedDoubleColonAfterNameInLabelDeclaration;
|
||||
}
|
||||
i.* += 1;
|
||||
return StatNode { .Label = name };
|
||||
},
|
||||
TokenType.Name, TokenType.RoundOpen =>
|
||||
{
|
||||
const startRegion = tokens[i.*].region;
|
||||
const suffixExp = try parseSuffixExp(tokens, i, allocator);
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
switch(suffixExp)
|
||||
{
|
||||
.Normal => return error.ExpectedFunctioncall,
|
||||
.Functioncall => |functioncall| return StatNode { .Functioncall = functioncall.* },
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
switch(tokens[i.*].tokenType)
|
||||
{
|
||||
TokenType.Equals =>
|
||||
{
|
||||
const endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
var lhs = std.ArrayList(VarNode).init(allocator.allocator());
|
||||
try lhs.append(try suffixExpToVar(suffixExp, startRegion, endRegion));
|
||||
const rhs = try parseExplist(tokens, i, allocator);
|
||||
return StatNode { .Assignment = AssignmentNode { .lhs = VarlistNode { .vars = lhs, .startRegion = endRegion, .endRegion = tokens[@min(i.*, tokens.len) - 1].region }, .rhs = rhs, .startRegion = startRegion, .endRegion = rhs.endRegion } };
|
||||
},
|
||||
TokenType.Comma =>
|
||||
{
|
||||
var varlistNode = VarlistNode { .vars = std.ArrayList(VarNode).init(allocator.allocator()), .startRegion = startRegion, .endRegion = startRegion };
|
||||
try varlistNode.vars.append(try suffixExpToVar(suffixExp, startRegion, tokens[@min(i.*, tokens.len) - 1].region));
|
||||
while(tokens[i.*].tokenType == TokenType.Comma)
|
||||
{
|
||||
i.* += 1;
|
||||
try varlistNode.vars.append(try parseVar(tokens, i, allocator));
|
||||
}
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Equals)
|
||||
{
|
||||
return error.ExpectedEqAfterAssignmentVarList;
|
||||
}
|
||||
varlistNode.endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
const rhs = try parseExplist(tokens, i, allocator);
|
||||
return StatNode { .Assignment = AssignmentNode { .lhs = varlistNode, .rhs = rhs, .startRegion = startRegion, .endRegion = rhs.endRegion } };
|
||||
},
|
||||
else =>
|
||||
{
|
||||
switch(suffixExp)
|
||||
{
|
||||
.Normal => return error.ExpectedFunctioncall,
|
||||
.Functioncall => |functioncall| return StatNode { .Functioncall = functioncall.* },
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
else =>
|
||||
{
|
||||
std.debug.print("{}\n", .{tokens[i.*]});
|
||||
return error.NotImplemented;
|
||||
}
|
||||
}
|
||||
}
|
||||
fn parseRetstat(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !RetstatNode
|
||||
{
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Return)
|
||||
{
|
||||
return error.ExpectedReturnStartingRetstat;
|
||||
}
|
||||
const startRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len or
|
||||
tokens[i.*].tokenType == TokenType.Semicolon or tokens[i.*].tokenType == TokenType.Else or tokens[i.*].tokenType == TokenType.Elseif or tokens[i.*].tokenType == TokenType.End)
|
||||
{
|
||||
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Semicolon)
|
||||
{
|
||||
i.* += 1;
|
||||
}
|
||||
return RetstatNode { .values = null, .startRegion = startRegion, .endRegion = tokens[@min(i.*, tokens.len) - 1].region };
|
||||
}
|
||||
const values = try parseExplist(tokens, i, allocator);
|
||||
var endRegion = values.endRegion;
|
||||
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Semicolon)
|
||||
{
|
||||
endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
}
|
||||
return RetstatNode { .values = values, .startRegion = startRegion, .endRegion = endRegion };
|
||||
}
|
||||
fn parseExp(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) ParserError!ExpNode
|
||||
{
|
||||
const lhs = try parseExpPrimary(tokens, i, allocator);
|
||||
return parseExpPrecedence(tokens, i, allocator, lhs, 0);
|
||||
}
|
||||
fn parseExpPrimary(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ExpNode
|
||||
{
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFExpectedPrimaryExpression;
|
||||
}
|
||||
const startRegion = tokens[i.*].region;
|
||||
switch(tokens[i.*].tokenType)
|
||||
{
|
||||
TokenType.Nil =>
|
||||
{
|
||||
i.* += 1;
|
||||
return ExpNode.Nil;
|
||||
},
|
||||
TokenType.True =>
|
||||
{
|
||||
i.* += 1;
|
||||
return ExpNode.True;
|
||||
},
|
||||
TokenType.False =>
|
||||
{
|
||||
i.* += 1;
|
||||
return ExpNode.False;
|
||||
},
|
||||
TokenType.Numeral =>
|
||||
{
|
||||
const numeral = tokens[i.*].tokenData.numeral;
|
||||
i.* += 1;
|
||||
return ExpNode { .Numeral = numeral };
|
||||
},
|
||||
TokenType.StringLiteral =>
|
||||
{
|
||||
const string = tokens[i.*].tokenData.string;
|
||||
i.* += 1;
|
||||
return ExpNode { .LiteralString = string };
|
||||
},
|
||||
TokenType.DotDotDot =>
|
||||
{
|
||||
i.* += 1;
|
||||
return ExpNode.Varargs;
|
||||
},
|
||||
TokenType.Function =>
|
||||
{
|
||||
i.* += 1;
|
||||
return ExpNode { .Functiondef = try parseFuncbody(tokens, i, allocator) };
|
||||
},
|
||||
TokenType.CurlyOpen => return ExpNode { .Tableconstructor = try parseTableconstructor(tokens, i, allocator) },
|
||||
TokenType.Minus =>
|
||||
{
|
||||
i.* += 1;
|
||||
const unop = try allocator.allocator().create(ExpNode);
|
||||
unop.* = try parseExp(tokens, i, allocator);
|
||||
const endRegion = tokens[@min(i.*, tokens.len) - 1].region;
|
||||
return ExpNode { .Unop = UnopNode { .unopType = UnopType.Minus, .exp = unop, .startRegion = startRegion, .endRegion = endRegion } };
|
||||
},
|
||||
TokenType.Hash =>
|
||||
{
|
||||
i.* += 1;
|
||||
const unop = try allocator.allocator().create(ExpNode);
|
||||
unop.* = try parseExp(tokens, i, allocator);
|
||||
const endRegion = tokens[@min(i.*, tokens.len) - 1].region;
|
||||
return ExpNode { .Unop = UnopNode { .unopType = UnopType.Length, .exp = unop, .startRegion = startRegion, .endRegion = endRegion } };
|
||||
},
|
||||
TokenType.Not =>
|
||||
{
|
||||
i.* += 1;
|
||||
const unop = try allocator.allocator().create(ExpNode);
|
||||
unop.* = try parseExp(tokens, i, allocator);
|
||||
const endRegion = tokens[@min(i.*, tokens.len) - 1].region;
|
||||
return ExpNode { .Unop = UnopNode { .unopType = UnopType.LogicalNot, .exp = unop, .startRegion = startRegion, .endRegion = endRegion } };
|
||||
},
|
||||
TokenType.Tilde =>
|
||||
{
|
||||
i.* += 1;
|
||||
const unop = try allocator.allocator().create(ExpNode);
|
||||
unop.* = try parseExp(tokens, i, allocator);
|
||||
const endRegion = tokens[@min(i.*, tokens.len) - 1].region;
|
||||
return ExpNode { .Unop = UnopNode { .unopType = UnopType.BinaryNot, .exp = unop, .startRegion = startRegion, .endRegion = endRegion } };
|
||||
},
|
||||
else =>
|
||||
{
|
||||
const suffixexp = try allocator.allocator().create(SuffixexpNode);
|
||||
suffixexp.* = try parseSuffixExp(tokens, i, allocator);
|
||||
return ExpNode { .Suffixexp = suffixexp };
|
||||
}
|
||||
}
|
||||
}
|
||||
fn parseExpPrecedence(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator, lhs: ExpNode, minPrecedence: u8) !ExpNode
|
||||
{
|
||||
var currentLhs = lhs;
|
||||
while(i.* < tokens.len and isBinop(tokens[i.*]))
|
||||
{
|
||||
const startRegion = tokens[i.*].region;
|
||||
const precedence = try getPrecedence(tokens[i.*]);
|
||||
if(precedence < minPrecedence)
|
||||
{
|
||||
break;
|
||||
}
|
||||
const op = try getBinopType(tokens[i.*]);
|
||||
i.* += 1;
|
||||
var rhs = try parseExpPrimary(tokens, i, allocator);
|
||||
while(i.* < tokens.len and isBinop(tokens[i.*]) and
|
||||
(try getPrecedence(tokens[i.*]) > precedence or
|
||||
(try getPrecedence(tokens[i.*]) == precedence and isRightAssociative(tokens[i.*]))))
|
||||
{
|
||||
const associativityBoost: u8 = if(try getPrecedence(tokens[i.*]) == precedence) 0 else 1;
|
||||
rhs = try parseExpPrecedence(tokens, i, allocator, rhs, precedence + associativityBoost);
|
||||
}
|
||||
const binop = try allocator.allocator().create(BinopNode);
|
||||
binop.* = BinopNode { .lhs = currentLhs, .op = op, .rhs = rhs, .startRegion = startRegion, .endRegion = tokens[@min(i.*, tokens.len) - 1].region };
|
||||
currentLhs = ExpNode { .Binop = binop };
|
||||
}
|
||||
return currentLhs;
|
||||
}
|
||||
fn isRightAssociative(token: Token) bool
|
||||
{
|
||||
return token.tokenType == TokenType.DotDot or token.tokenType == TokenType.Caret;
|
||||
}
|
||||
fn getBinopType(token: Token) !BinopType
|
||||
{
|
||||
return switch(token.tokenType)
|
||||
{
|
||||
TokenType.Or => BinopType.LogicalOr,
|
||||
TokenType.And => BinopType.LocicalAnd,
|
||||
TokenType.Lt => BinopType.Lt,
|
||||
TokenType.Gt => BinopType.Gt,
|
||||
TokenType.LtEquals => BinopType.LtEquals,
|
||||
TokenType.GtEquals => BinopType.GtEquals,
|
||||
TokenType.LtLt => BinopType.Shl,
|
||||
TokenType.GtGt=> BinopType.Shr,
|
||||
TokenType.TildeEquals => BinopType.NotEquals,
|
||||
TokenType.EqualsEquals => BinopType.Equals,
|
||||
TokenType.Pipe => BinopType.BinaryOr,
|
||||
TokenType.Tilde => BinopType.BinaryNot,
|
||||
TokenType.Ampersand => BinopType.BinaryAnd,
|
||||
TokenType.DotDot => BinopType.Concat,
|
||||
TokenType.Plus => BinopType.Add,
|
||||
TokenType.Minus => BinopType.Sub,
|
||||
TokenType.Star => BinopType.Mul,
|
||||
TokenType.Slash => BinopType.Div,
|
||||
TokenType.SlashSlash => BinopType.IntDiv,
|
||||
TokenType.Percent => BinopType.Mod,
|
||||
TokenType.Caret => BinopType.Exp,
|
||||
else => error.NoBinopTypeForOperator,
|
||||
};
|
||||
|
||||
}
|
||||
fn getPrecedence(token: Token) !u8
|
||||
{
|
||||
return switch(token.tokenType)
|
||||
{
|
||||
TokenType.Or => 2,
|
||||
TokenType.And => 4,
|
||||
TokenType.Lt, TokenType.Gt, TokenType.LtEquals, TokenType.GtEquals, TokenType.TildeEquals, TokenType.EqualsEquals => 6,
|
||||
TokenType.Pipe => 8,
|
||||
TokenType.Tilde => 10,
|
||||
TokenType.Ampersand => 12,
|
||||
TokenType.LtLt, TokenType.GtGt => 14,
|
||||
TokenType.DotDot => 16,
|
||||
TokenType.Plus, TokenType.Minus => 18,
|
||||
TokenType.Star, TokenType.Slash, TokenType.SlashSlash, TokenType.Percent => 20,
|
||||
TokenType.Caret => 22,
|
||||
else => error.NoPrecedenceForOperator,
|
||||
};
|
||||
}
|
||||
fn isBinop(token: Token) bool
|
||||
{
|
||||
return switch(token.tokenType)
|
||||
{
|
||||
TokenType.Or, TokenType.And, TokenType.Lt, TokenType.Gt, TokenType.LtEquals, TokenType.GtEquals, TokenType.TildeEquals, TokenType.EqualsEquals,
|
||||
TokenType.Pipe, TokenType.Tilde, TokenType.Ampersand, TokenType.LtLt, TokenType.GtGt, TokenType.DotDot, TokenType.Plus, TokenType.Minus,
|
||||
TokenType.Star, TokenType.Slash, TokenType.SlashSlash, TokenType.Percent, TokenType.Caret => true,
|
||||
else => false
|
||||
};
|
||||
}
|
||||
fn parseExplist(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ExplistNode
|
||||
{
|
||||
const startRegion = tokens[@min(i.*, tokens.len) - 1].region;
|
||||
var ret = ExplistNode { .exps = std.ArrayList(ExpNode).init(allocator.allocator()), .startRegion = startRegion, .endRegion = startRegion };
|
||||
try ret.exps.append(try parseExp(tokens, i, allocator));
|
||||
while(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Comma)
|
||||
{
|
||||
i.* += 1;
|
||||
try ret.exps.append(try parseExp(tokens, i, allocator));
|
||||
}
|
||||
ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region;
|
||||
return ret;
|
||||
}
|
||||
fn parseFuncname(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !FuncnameNode
|
||||
{
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
|
||||
{
|
||||
return error.ExpectedFuncname;
|
||||
}
|
||||
const startRange = tokens[i.*].region;
|
||||
var ret = FuncnameNode { .name = tokens[i.*].tokenData.string, .dottedNames = std.ArrayList([]u8).init(allocator.allocator()), .firstArg = null, .startRegion = startRange, .endRegion = startRange };
|
||||
i.* += 1;
|
||||
while(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Dot)
|
||||
{
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
|
||||
{
|
||||
return error.ExpectedNameInDottedFuncname;
|
||||
}
|
||||
try ret.dottedNames.append(tokens[i.*].tokenData.string);
|
||||
}
|
||||
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Colon)
|
||||
{
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
|
||||
{
|
||||
return error.ExpectedNameOfFirstArgInFuncname;
|
||||
}
|
||||
ret.firstArg = tokens[i.*].tokenData.string;
|
||||
i.* += 1;
|
||||
}
|
||||
ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region;
|
||||
return ret;
|
||||
}
|
||||
fn parseFuncbody(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !FuncbodyNode
|
||||
{
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.RoundOpen)
|
||||
{
|
||||
return error.ExpectedRoundOpenStartingFuncbody;
|
||||
}
|
||||
const startRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFInFuncbodyParlist;
|
||||
}
|
||||
var pars: ?ParlistNode = null;
|
||||
if(tokens[i.*].tokenType == TokenType.RoundClosed)
|
||||
{
|
||||
i.* += 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
pars = try parseParlist(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.RoundClosed)
|
||||
{
|
||||
return error.ExpectedRoundClosedClosingFuncbodyParlist;
|
||||
}
|
||||
i.* += 1;
|
||||
}
|
||||
const ret = FuncbodyNode { .body = try parseBlock(tokens, i, allocator), .pars = pars, .startRegion = startRegion, .endRegion = tokens[i.*].region };
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.End)
|
||||
{
|
||||
return error.ExpectedEndClosingFuncbody;
|
||||
}
|
||||
i.* += 1;
|
||||
return ret;
|
||||
}
|
||||
fn parseParlist(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ParlistNode
|
||||
{
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFInParlist;
|
||||
}
|
||||
const startRegion = tokens[i.*].region;
|
||||
if(tokens[i.*].tokenType == TokenType.DotDotDot)
|
||||
{
|
||||
const endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
return ParlistNode { .names = std.ArrayList([]u8).init(allocator.allocator()), .hasVarargs = true, .startRegion = startRegion, .endRegion = endRegion };
|
||||
}
|
||||
if(tokens[i.*].tokenType != TokenType.Name)
|
||||
{
|
||||
return error.ExpectedNameStartingParlist;
|
||||
}
|
||||
var ret = ParlistNode { .names = std.ArrayList([]u8).init(allocator.allocator()), .hasVarargs = false, .startRegion = startRegion, .endRegion = startRegion };
|
||||
try ret.names.append(tokens[i.*].tokenData.string);
|
||||
i.* += 1;
|
||||
while(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Comma)
|
||||
{
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFInParlistNameList;
|
||||
}
|
||||
switch(tokens[i.*].tokenType)
|
||||
{
|
||||
TokenType.Name =>
|
||||
{
|
||||
try ret.names.append(tokens[i.*].tokenData.string);
|
||||
i.* += 1;
|
||||
},
|
||||
TokenType.DotDotDot =>
|
||||
{
|
||||
i.* += 1;
|
||||
ret.hasVarargs = true;
|
||||
break;
|
||||
},
|
||||
else => return error.UnexpectedTokenInParlistNameList,
|
||||
}
|
||||
}
|
||||
ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region;
|
||||
return ret;
|
||||
}
|
||||
fn parseAttnamelist(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !AttnamelistNode
|
||||
{
|
||||
// TODO: What happens if this is reaches EOF?
|
||||
var ret = AttnamelistNode { .attnames = std.ArrayList(AttnameNode).init(allocator.allocator()), .startRegion = tokens[i.*].region, .endRegion = tokens[i.*].region };
|
||||
try ret.attnames.append(try parseAttname(tokens, i));
|
||||
while(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Comma)
|
||||
{
|
||||
i.* += 1;
|
||||
try ret.attnames.append(try parseAttname(tokens, i));
|
||||
}
|
||||
ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region;
|
||||
return ret;
|
||||
}
|
||||
fn parseSuffixExp(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !SuffixexpNode
|
||||
{
|
||||
// primaryexp { '.' 'Name' | '[' exp']' | ':' 'Name' args | args }
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFInSuffixExp;
|
||||
}
|
||||
const startRegion = tokens[i.*].region;
|
||||
const firstPart = try switch(tokens[i.*].tokenType)
|
||||
{
|
||||
TokenType.Name =>
|
||||
nameBlock: {
|
||||
const name = tokens[i.*].tokenData.string;
|
||||
i.* += 1;
|
||||
break :nameBlock SuffixexpFirstPart { .Name = name };
|
||||
},
|
||||
TokenType.RoundOpen =>
|
||||
roundOpenBlock: {
|
||||
i.* += 1;
|
||||
const ret = SuffixexpFirstPart { .BracketedExpr = try parseExp(tokens, i, allocator) };
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.RoundClosed)
|
||||
{
|
||||
return error.ExpectedRoundClosedClosingBracketedPrimaryExp;
|
||||
}
|
||||
i.* += 1;
|
||||
break :roundOpenBlock ret;
|
||||
},
|
||||
else => error.UnexpectedTokenAsFirstPartOfSuffixExp,
|
||||
};
|
||||
var suffixes = std.ArrayList(SuffixexpSuffix).init(allocator.allocator());
|
||||
while(i.* < tokens.len)
|
||||
{
|
||||
switch(tokens[i.*].tokenType)
|
||||
{
|
||||
TokenType.Dot =>
|
||||
{
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
|
||||
{
|
||||
return error.ExpectedNameInDottedSuffixExp;
|
||||
}
|
||||
const name = tokens[i.*].tokenData.string;
|
||||
i.* += 1;
|
||||
try suffixes.append(SuffixexpSuffix { .Dot = name });
|
||||
},
|
||||
TokenType.SquareOpen =>
|
||||
{
|
||||
i.* += 1;
|
||||
try suffixes.append(SuffixexpSuffix { .Indexed = try parseExp(tokens, i, allocator) });
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.SquareClosed)
|
||||
{
|
||||
return error.ExpectedSquareClosedClosingIndexedSuffixExp;
|
||||
}
|
||||
i.* += 1;
|
||||
},
|
||||
TokenType.Colon =>
|
||||
{
|
||||
const argsFirstArgStartRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
|
||||
{
|
||||
return error.ExpectedNameInArgsFirstArgSuffixExp;
|
||||
}
|
||||
const name = tokens[i.*].tokenData.string;
|
||||
const argsFirstArgEndRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
try suffixes.append(SuffixexpSuffix { .ArgsFirstArg = ArgsFirstArgNode { .name = name, .rest = try parseArgs(tokens, i, allocator), .startRegion = argsFirstArgStartRegion, .endRegion = argsFirstArgEndRegion } });
|
||||
},
|
||||
TokenType.RoundOpen, TokenType.CurlyOpen, TokenType.StringLiteral =>
|
||||
{
|
||||
try suffixes.append(SuffixexpSuffix { .Args = try parseArgs(tokens, i, allocator) });
|
||||
},
|
||||
else => break,
|
||||
}
|
||||
}
|
||||
const endRegion = tokens[@min(i.*, tokens.len) - 1].region;
|
||||
const last = suffixes.getLastOrNull();
|
||||
if(last != null)
|
||||
{
|
||||
switch(last.?)
|
||||
{
|
||||
SuffixexpSuffix.Args => |*args|
|
||||
{
|
||||
_ = suffixes.pop();
|
||||
const functioncall = try allocator.allocator().create(FunctioncallNode);
|
||||
functioncall.* = FunctioncallNode
|
||||
{
|
||||
.function = SuffixexpNode { .Normal = NormalSuffixNode { .firstPart = firstPart, .suffixes = suffixes, .startRegion = startRegion, .endRegion = endRegion } },
|
||||
.args = args.*,
|
||||
.objectArg = null,
|
||||
.startRegion = startRegion,
|
||||
.endRegion = endRegion,
|
||||
};
|
||||
return SuffixexpNode
|
||||
{
|
||||
.Functioncall = functioncall,
|
||||
};
|
||||
},
|
||||
SuffixexpSuffix.ArgsFirstArg => |*node|
|
||||
{
|
||||
_ = suffixes.pop();
|
||||
const functioncall = try allocator.allocator().create(FunctioncallNode);
|
||||
functioncall.* = FunctioncallNode
|
||||
{
|
||||
.function = SuffixexpNode { .Normal = NormalSuffixNode { .firstPart = firstPart, .suffixes = suffixes, .startRegion = startRegion, .endRegion = endRegion } },
|
||||
.args = node.rest,
|
||||
.objectArg = node.name,
|
||||
.startRegion = startRegion,
|
||||
.endRegion = endRegion,
|
||||
};
|
||||
return SuffixexpNode
|
||||
{
|
||||
.Functioncall = functioncall,
|
||||
};
|
||||
},
|
||||
else => {}
|
||||
}
|
||||
}
|
||||
return SuffixexpNode { .Normal = NormalSuffixNode { .firstPart = firstPart, .suffixes = suffixes, .startRegion = startRegion, .endRegion = endRegion } };
|
||||
}
|
||||
fn parseVar(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !VarNode
|
||||
{
|
||||
const startRegion = tokens[i.*].region;
|
||||
return suffixExpToVar(try parseSuffixExp(tokens, i, allocator), startRegion, tokens[@min(i.*, tokens.len) - 1].region);
|
||||
}
|
||||
fn parseAttname(tokens: []Token, i: *usize) !AttnameNode
|
||||
{
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
|
||||
{
|
||||
return error.ExpectednameInAttribName;
|
||||
}
|
||||
const name = tokens[i.*].tokenData.string;
|
||||
const startRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
var ret = AttnameNode { .name = name, .attribute = null, .startRegion = startRegion, .endRegion = startRegion };
|
||||
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.Lt)
|
||||
{
|
||||
ret.attribute = tokens[i.*].tokenData.string;
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Name)
|
||||
{
|
||||
return error.ExpectedAttributeInAttrib;
|
||||
}
|
||||
ret.endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Gt)
|
||||
{
|
||||
return error.ExpectedGtInAttrib;
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
fn parseArgs(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !ArgsNode
|
||||
{
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFInArgs;
|
||||
}
|
||||
switch(tokens[i.*].tokenType)
|
||||
{
|
||||
TokenType.RoundOpen =>
|
||||
{
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFInBracketedArgs;
|
||||
}
|
||||
if(tokens[i.*].tokenType == TokenType.RoundClosed)
|
||||
{
|
||||
i.* += 1;
|
||||
return ArgsNode { .Bracketed = null };
|
||||
}
|
||||
const exps = try parseExplist(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.RoundClosed)
|
||||
{
|
||||
return error.ExpectedRoundClosedClosingBracketedArgs;
|
||||
}
|
||||
i.* += 1;
|
||||
return ArgsNode { .Bracketed = exps };
|
||||
},
|
||||
TokenType.CurlyOpen => return ArgsNode { .Tableconstructor = try parseTableconstructor(tokens, i, allocator) },
|
||||
TokenType.StringLiteral =>
|
||||
{
|
||||
const value = tokens[i.*].tokenData.string;
|
||||
i.* += 1;
|
||||
return ArgsNode { .Literal = value };
|
||||
},
|
||||
else => return error.UnexpectedTokenInArgs,
|
||||
}
|
||||
}
|
||||
fn parseTableconstructor(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !TableconstructorNode
|
||||
{
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.CurlyOpen)
|
||||
{
|
||||
return error.ExpectedCurlyOpenOpeningTableconstructor;
|
||||
}
|
||||
const startRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
if(i.* < tokens.len and tokens[i.*].tokenType == TokenType.CurlyClosed)
|
||||
{
|
||||
const endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
return TableconstructorNode { .exps = null, .startRegion = startRegion, .endRegion = endRegion };
|
||||
}
|
||||
var ret = TableconstructorNode { .exps = try parseFieldlist(tokens, i, allocator), .startRegion = startRegion, .endRegion = startRegion };
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.CurlyClosed)
|
||||
{
|
||||
return error.ExpectedCurlyClosedClosingTableconstructor;
|
||||
}
|
||||
ret.endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
return ret;
|
||||
}
|
||||
fn parseFieldlist(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !FieldlistNode
|
||||
{
|
||||
const startRegion = tokens[@min(i.*, tokens.len) - 1].region;
|
||||
var ret = FieldlistNode { .exps = std.ArrayList(FieldNode).init(allocator.allocator()), .startRegion = startRegion, .endRegion = startRegion };
|
||||
try ret.exps.append(try parseField(tokens, i, allocator));
|
||||
while(i.* < tokens.len and isFieldsep(tokens[i.*]))
|
||||
{
|
||||
i.* += 1;
|
||||
try ret.exps.append(try parseField(tokens, i, allocator));
|
||||
}
|
||||
if(i.* < tokens.len and isFieldsep(tokens[i.*]))
|
||||
{
|
||||
i.* += 1;
|
||||
}
|
||||
ret.endRegion = tokens[@min(i.*, tokens.len) - 1].region;
|
||||
return ret;
|
||||
}
|
||||
fn isFieldsep(token: Token) bool
|
||||
{
|
||||
return token.tokenType == TokenType.Comma or token.tokenType == TokenType.Semicolon;
|
||||
}
|
||||
fn parseField(tokens: []Token, i: *usize, allocator: *std.heap.ArenaAllocator) !FieldNode
|
||||
{
|
||||
if(i.* >= tokens.len)
|
||||
{
|
||||
return error.ReachedEOFInField;
|
||||
}
|
||||
const startRegion = tokens[i.*].region;
|
||||
switch(tokens[i.*].tokenType)
|
||||
{
|
||||
TokenType.SquareOpen =>
|
||||
{
|
||||
i.* += 1;
|
||||
const index = try parseExp(tokens, i, allocator);
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.SquareClosed)
|
||||
{
|
||||
return error.ExpectedSquareClosedClosingIndexedField;
|
||||
}
|
||||
i.* += 1;
|
||||
if(i.* >= tokens.len or tokens[i.*].tokenType != TokenType.Equals)
|
||||
{
|
||||
return error.ExpectedEqualsInIndexedFieldExpression;
|
||||
}
|
||||
const endRegion = tokens[i.*].region;
|
||||
i.* += 1;
|
||||
return FieldNode { .IndexedAssignment = IndexedAssignmentNode { .index = index, .rhs = try parseExp(tokens, i, allocator), .startRegion = startRegion, .endRegion = endRegion } };
|
||||
},
|
||||
TokenType.Name =>
|
||||
{
|
||||
if(i.* + 1 < tokens.len and tokens[i.* + 1].tokenType == TokenType.Equals)
|
||||
{
|
||||
const name = tokens[i.*].tokenData.string;
|
||||
i.* += 2;
|
||||
return FieldNode { .Assignment = FieldAssignmentNode { .lhs = name, .rhs = try parseExp(tokens, i, allocator), .startRegion = startRegion, .endRegion = tokens[i.* - 1].region } };
|
||||
}
|
||||
return FieldNode { .Exp = try parseExp(tokens, i, allocator) };
|
||||
},
|
||||
else => return FieldNode { .Exp = try parseExp(tokens, i, allocator) },
|
||||
}
|
||||
}
|
||||
fn suffixExpToVar(suffixexp: SuffixexpNode, startRegion: CodeRegion, endRegion: CodeRegion) !VarNode
|
||||
{
|
||||
var exp = suffixexp.Normal;
|
||||
if(exp.suffixes.items.len == 0)
|
||||
{
|
||||
return VarNode { .Name = exp.firstPart.Name };
|
||||
}
|
||||
const last = exp.suffixes.pop();
|
||||
return switch(last)
|
||||
{
|
||||
SuffixexpSuffix.Dot => |*name| VarNode { .Member = MemberVarNode { .name = name.*, .value = SuffixexpNode { .Normal = exp }, .startRegion = startRegion, .endRegion = endRegion } },
|
||||
SuffixexpSuffix.Indexed => |*index| VarNode { .Indexed = IndexedVarNode { .index = index.*, .value = SuffixexpNode { .Normal = exp }, .startRegion = startRegion, .endRegion = endRegion } },
|
||||
else => error.ExpectedDotOrIndexedSuffixWhenConvertingSuffixExpToVar,
|
||||
};
|
||||
}
|
||||
1210
src/tokenizer.rs
Normal file
1210
src/tokenizer.rs
Normal file
@ -0,0 +1,1210 @@
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub enum Token
|
||||
{
|
||||
Name(String),
|
||||
And, Break, Do, Else, Elseif, End,
|
||||
False, For, Function, Goto, If, In,
|
||||
Local, Nil, Not, Or, Repeat, Return,
|
||||
Then, True, Until, While,
|
||||
Plus, Minus, Star, Slash, Percent, Caret, Hash,
|
||||
Ampersand, Tilde, Pipe, LtLt, GtGt, SlashSlash,
|
||||
EqualsEquals, TildeEquals, LtEquals, GtEquals, Lt, Gt, Equals,
|
||||
RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, ColonColon,
|
||||
Semicolon, Colon, Comma, Dot, DotDot, DotDotDot,
|
||||
Numeral(String),
|
||||
StringLiteral(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub enum TokenizerState
|
||||
{
|
||||
Start,
|
||||
Quote, SingleQuote, Name, Number, Zero,
|
||||
A, B, D, E, F, G, I, L, N, O, R, T, U, W,
|
||||
Plus, Minus, Star, Slash, Percent, Caret, Hash,
|
||||
Ampersand, Tilde, Pipe, Lt, Gt, Equals, RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed,
|
||||
Colon, Semicolon, Comma, Dot,
|
||||
|
||||
An, Br, Do, El, En, Fa, Fo, Fu, Go, If, In, Lo, Ni, No, Or, Re, Th, Tr, Un, Wh,
|
||||
LtLt, GtGt, SlashSlash, EqualsEquals, TildeEquals, LtEquals, GtEquals, ColonColon, DotDot,
|
||||
SmallCommentStart, QuoteBackslash, SingleQuoteBackslash, String, HexNumberX, ExpNumber,
|
||||
|
||||
And, Bre, Els, End, Fal, For, Fun, Got, Loc, Nil, Not, Rep, Ret, The, Tru, Unt, Whi,
|
||||
DotDotDot, HexNumber, QuoteBackslashZ, SingleQuoteBackslashZ,
|
||||
BigCommentLongBracketStart, SmallComment,
|
||||
|
||||
Brea, Else, Fals, Func, Goto, Loca, Repe, Retu, Then, True, Unti, Whil, HexExpNumber,
|
||||
BigComment, BigCommentLongBracketEnd,
|
||||
|
||||
Break, Elsei, False, Funct, Local, Repea, Retur, Until, While,
|
||||
|
||||
Elseif, Functi, Repeat, Return,
|
||||
|
||||
Functio,
|
||||
|
||||
Function,
|
||||
}
|
||||
|
||||
fn tokenize_update_index_and_state(last_index: &mut i32, index: usize, state: &mut TokenizerState, new_state: TokenizerState)
|
||||
{
|
||||
*last_index = index as i32;
|
||||
*state = new_state;
|
||||
}
|
||||
fn tokenize_terminal_no_str(last_index: &mut i32, index: usize, token: &mut Option<Token>, state: &mut TokenizerState, new_token: Option<Token>, new_state: TokenizerState)
|
||||
{
|
||||
tokenize_update_index_and_state(last_index, index, state, new_state);
|
||||
*token = new_token;
|
||||
}
|
||||
fn tokenize_terminal_no_token(last_index: &mut i32, index: usize, state: &mut TokenizerState, new_state: TokenizerState, token_str: &mut String, ch: char)
|
||||
{
|
||||
tokenize_update_index_and_state(last_index, index, state, new_state);
|
||||
token_str.push(ch);
|
||||
}
|
||||
fn tokenize_terminal(last_index: &mut i32, index: usize, token: &mut Option<Token>, state: &mut TokenizerState, new_token: Option<Token>, new_state: TokenizerState, token_str: &mut String, ch: char)
|
||||
{
|
||||
tokenize_terminal_no_str(last_index, index, token, state, new_token, new_state);
|
||||
token_str.push(ch);
|
||||
}
|
||||
fn tokenize_backtrack(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState) -> Result<(), &'static str>
|
||||
{
|
||||
return tokenize_backtrack_custom_token(last_index, index, tokens, token, token_str, state, token.clone().unwrap());
|
||||
}
|
||||
fn tokenize_backtrack_name(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState) -> Result<(), &'static str>
|
||||
{
|
||||
if *last_index == -1 || token.is_none()
|
||||
{
|
||||
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
||||
return Err("Lexerr");
|
||||
}
|
||||
*index = *last_index as usize;
|
||||
*last_index = -1;
|
||||
tokens.push(Token::Name(token_str.clone()));
|
||||
*token = None;
|
||||
token_str.clear();
|
||||
*state = TokenizerState::Start;
|
||||
return Ok(());
|
||||
}
|
||||
fn tokenize_backtrack_custom_token(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState, new_token: Token) -> Result<(), &'static str>
|
||||
{
|
||||
if *last_index == -1 || token.is_none()
|
||||
{
|
||||
println!("{}|{}|{:?} | {:?}", last_index, index, token, tokens);
|
||||
return Err("Lexerr");
|
||||
}
|
||||
*index = *last_index as usize;
|
||||
*last_index = -1;
|
||||
tokens.push(new_token);
|
||||
*token = None;
|
||||
token_str.clear();
|
||||
*state = TokenizerState::Start;
|
||||
return Ok(());
|
||||
}
|
||||
fn tokenize_alphanumeric_nonstart(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState, ch: char) -> Result<(), &'static str>
|
||||
{
|
||||
if ch.is_ascii_alphanumeric() || ch == '_'
|
||||
{
|
||||
tokenize_update_index_and_state(last_index, *index, state, TokenizerState::Name);
|
||||
token_str.push(ch);
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenize_backtrack_name(last_index, index, tokens, token, token_str, state)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
fn tokenize_alphanumeric_nonstart_custom(last_index: &mut i32, index: &mut usize, tokens: &mut Vec<Token>, token: &mut Option<Token>, token_str: &mut String, state: &mut TokenizerState, ch: char, new_token: Token) -> Result<(), &'static str>
|
||||
{
|
||||
if ch.is_ascii_alphanumeric() || ch == '_'
|
||||
{
|
||||
tokenize_update_index_and_state(last_index, *index, state, TokenizerState::Name);
|
||||
token_str.push(ch);
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenize_backtrack_custom_token(last_index, index, tokens, token, token_str, state, new_token)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
fn tokenize_char(state: &mut TokenizerState, ch: char, last_index: &mut i32, index: &mut usize, token: &mut Option<Token>, token_str: &mut String, tokens: &mut Vec<Token>, long_bracket_level: &mut u32) -> Result<(), &'static str>
|
||||
{
|
||||
match state
|
||||
{
|
||||
TokenizerState::Start =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'-' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Minus), TokenizerState::Minus),
|
||||
'a' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("a".to_string())), TokenizerState::A, token_str, ch),
|
||||
'b' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("b".to_string())), TokenizerState::B, token_str, ch),
|
||||
'd' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("d".to_string())), TokenizerState::D, token_str, ch),
|
||||
'e' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("e".to_string())), TokenizerState::E, token_str, ch),
|
||||
'f' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("f".to_string())), TokenizerState::F, token_str, ch),
|
||||
'i' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("i".to_string())), TokenizerState::I, token_str, ch),
|
||||
'g' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("g".to_string())), TokenizerState::G, token_str, ch),
|
||||
'l' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("l".to_string())), TokenizerState::L, token_str, ch),
|
||||
'n' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("n".to_string())), TokenizerState::N, token_str, ch),
|
||||
'o' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("o".to_string())), TokenizerState::O, token_str, ch),
|
||||
'r' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("r".to_string())), TokenizerState::R, token_str, ch),
|
||||
't' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("t".to_string())), TokenizerState::T, token_str, ch),
|
||||
'u' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("u".to_string())), TokenizerState::U, token_str, ch),
|
||||
'w' => tokenize_terminal(last_index, *index, token, state, Some(Token::Name("w".to_string())), TokenizerState::W, token_str, ch),
|
||||
',' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Comma), TokenizerState::Comma),
|
||||
'=' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Equals), TokenizerState::Equals),
|
||||
'(' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::RoundOpen), TokenizerState::RoundOpen),
|
||||
')' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::RoundClosed), TokenizerState::RoundClosed),
|
||||
'.' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Dot), TokenizerState::Dot),
|
||||
':' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Colon), TokenizerState::Colon),
|
||||
'{' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::CurlyOpen), TokenizerState::CurlyOpen),
|
||||
'}' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::CurlyClosed), TokenizerState::CurlyClosed),
|
||||
'[' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::SquareOpen), TokenizerState::SquareOpen),
|
||||
']' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::SquareClosed), TokenizerState::SquareClosed),
|
||||
'+' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Plus), TokenizerState::Plus),
|
||||
'~' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Tilde), TokenizerState::Tilde),
|
||||
'>' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Gt), TokenizerState::Gt),
|
||||
'<' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Lt), TokenizerState::Lt),
|
||||
'#' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Hash), TokenizerState::Hash),
|
||||
'|' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Pipe), TokenizerState::Pipe),
|
||||
'&' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Ampersand), TokenizerState::Ampersand),
|
||||
'%' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Percent), TokenizerState::Percent),
|
||||
'*' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Star), TokenizerState::Star),
|
||||
'/' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Slash), TokenizerState::Slash),
|
||||
';' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Semicolon), TokenizerState::Semicolon),
|
||||
'^' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Caret), TokenizerState::Caret),
|
||||
'0' => tokenize_terminal(last_index, *index, token, state, Some(Token::Numeral("0".to_string())), TokenizerState::Zero, token_str, ch),
|
||||
'"' =>
|
||||
{
|
||||
*token = None;
|
||||
*state = TokenizerState::Quote;
|
||||
}
|
||||
'\'' =>
|
||||
{
|
||||
*token = None;
|
||||
*state = TokenizerState::SingleQuote;
|
||||
}
|
||||
_ =>
|
||||
{
|
||||
if ch.is_whitespace() { }
|
||||
else if ch.is_ascii_alphabetic() || ch == '_'
|
||||
{
|
||||
tokenize_terminal(last_index, *index, token, state, Some(Token::Name(ch.to_string())), TokenizerState::Name, token_str, ch);
|
||||
}
|
||||
else if ch.is_numeric() && ch.is_ascii()
|
||||
{
|
||||
tokenize_terminal(last_index, *index, token, state, Some(Token::Numeral(ch.to_string())), TokenizerState::Number, token_str, ch);
|
||||
}
|
||||
else
|
||||
{
|
||||
todo!("State {:?}, Char {}", state, ch);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
TokenizerState::Quote =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'\\' =>
|
||||
{
|
||||
*state = TokenizerState::QuoteBackslash;
|
||||
}
|
||||
'"' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::StringLiteral(token_str.clone())), TokenizerState::String),
|
||||
_ =>
|
||||
{
|
||||
token_str.push(ch);
|
||||
}
|
||||
}
|
||||
}
|
||||
TokenizerState::QuoteBackslash =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'a' =>
|
||||
{
|
||||
token_str.push('\u{0007}');
|
||||
*state = TokenizerState::Quote;
|
||||
}
|
||||
'b' =>
|
||||
{
|
||||
token_str.push('\u{0008}');
|
||||
*state = TokenizerState::Quote;
|
||||
}
|
||||
't' =>
|
||||
{
|
||||
token_str.push('\t');
|
||||
*state = TokenizerState::Quote;
|
||||
}
|
||||
'n' | '\n' =>
|
||||
{
|
||||
token_str.push('\n');
|
||||
*state = TokenizerState::Quote;
|
||||
}
|
||||
'v' =>
|
||||
{
|
||||
token_str.push('\u{000b}');
|
||||
*state = TokenizerState::Quote;
|
||||
}
|
||||
'f' =>
|
||||
{
|
||||
token_str.push('\u{000c}');
|
||||
*state = TokenizerState::Quote;
|
||||
}
|
||||
'r' =>
|
||||
{
|
||||
token_str.push('\r');
|
||||
*state = TokenizerState::Quote;
|
||||
}
|
||||
'\\' =>
|
||||
{
|
||||
token_str.push('\\');
|
||||
*state = TokenizerState::Quote;
|
||||
}
|
||||
'"' =>
|
||||
{
|
||||
token_str.push('\"');
|
||||
*state = TokenizerState::Quote;
|
||||
}
|
||||
'\'' =>
|
||||
{
|
||||
token_str.push('\'');
|
||||
*state = TokenizerState::Quote;
|
||||
}
|
||||
'z' =>
|
||||
{
|
||||
*state = TokenizerState::QuoteBackslashZ;
|
||||
}
|
||||
_ => return Err("Unknown escape sequence"),
|
||||
}
|
||||
}
|
||||
TokenizerState::QuoteBackslashZ =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'\\' =>
|
||||
{
|
||||
*state = TokenizerState::QuoteBackslash;
|
||||
}
|
||||
'"' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::StringLiteral(token_str.clone())), TokenizerState::String),
|
||||
_ =>
|
||||
{
|
||||
if !ch.is_whitespace()
|
||||
{
|
||||
token_str.push(ch);
|
||||
*state = TokenizerState::Quote;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
TokenizerState::SingleQuote =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'\\' =>
|
||||
{
|
||||
*state = TokenizerState::SingleQuoteBackslash;
|
||||
}
|
||||
'\'' =>
|
||||
{
|
||||
*last_index = *index as i32;
|
||||
*token = Some(Token::StringLiteral(token_str.clone()));
|
||||
*state = TokenizerState::String;
|
||||
}
|
||||
_ =>
|
||||
{
|
||||
token_str.push(ch);
|
||||
}
|
||||
}
|
||||
}
|
||||
TokenizerState::SingleQuoteBackslash =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'a' =>
|
||||
{
|
||||
token_str.push('\u{0007}');
|
||||
*state = TokenizerState::SingleQuote;
|
||||
}
|
||||
'b' =>
|
||||
{
|
||||
token_str.push('\u{0008}');
|
||||
*state = TokenizerState::SingleQuote;
|
||||
}
|
||||
't' =>
|
||||
{
|
||||
token_str.push('\t');
|
||||
*state = TokenizerState::SingleQuote;
|
||||
}
|
||||
'n' | '\n' =>
|
||||
{
|
||||
token_str.push('\n');
|
||||
*state = TokenizerState::SingleQuote;
|
||||
}
|
||||
'v' =>
|
||||
{
|
||||
token_str.push('\u{000b}');
|
||||
*state = TokenizerState::SingleQuote;
|
||||
}
|
||||
'f' =>
|
||||
{
|
||||
token_str.push('\u{000c}');
|
||||
*state = TokenizerState::SingleQuote;
|
||||
}
|
||||
'r' =>
|
||||
{
|
||||
token_str.push('\r');
|
||||
*state = TokenizerState::SingleQuote;
|
||||
}
|
||||
'\\' =>
|
||||
{
|
||||
token_str.push('\\');
|
||||
*state = TokenizerState::SingleQuote;
|
||||
}
|
||||
'"' =>
|
||||
{
|
||||
token_str.push('\"');
|
||||
*state = TokenizerState::SingleQuote;
|
||||
}
|
||||
'\'' =>
|
||||
{
|
||||
token_str.push('\'');
|
||||
*state = TokenizerState::SingleQuote;
|
||||
}
|
||||
'z' =>
|
||||
{
|
||||
*state = TokenizerState::SingleQuoteBackslashZ;
|
||||
}
|
||||
_ => return Err("Unknown escape sequence"),
|
||||
}
|
||||
}
|
||||
TokenizerState::SingleQuoteBackslashZ =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'\\' =>
|
||||
{
|
||||
*state = TokenizerState::SingleQuoteBackslash;
|
||||
}
|
||||
'\'' =>
|
||||
{
|
||||
*last_index = *index as i32;
|
||||
*token = Some(Token::StringLiteral(token_str.clone()));
|
||||
*state = TokenizerState::String;
|
||||
}
|
||||
_ =>
|
||||
{
|
||||
if !ch.is_whitespace()
|
||||
{
|
||||
token_str.push(ch);
|
||||
*state = TokenizerState::SingleQuote;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
TokenizerState::String =>
|
||||
{
|
||||
let content = token_str.clone();
|
||||
tokenize_backtrack_custom_token(last_index, index, tokens, token, token_str, state, Token::StringLiteral(content))?;
|
||||
}
|
||||
TokenizerState::Name => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
TokenizerState::Zero =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'x' =>
|
||||
{
|
||||
token_str.push(ch);
|
||||
*token = None;
|
||||
*state = TokenizerState::HexNumberX;
|
||||
}
|
||||
_ =>
|
||||
{
|
||||
if ch.is_numeric() && ch.is_ascii()
|
||||
{
|
||||
*last_index = *index as i32;
|
||||
token_str.push(ch);
|
||||
*token = Some(Token::Numeral(token_str.clone()));
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenize_backtrack(last_index, index, tokens, token, token_str, state)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
TokenizerState::HexNumberX =>
|
||||
{
|
||||
if ch.is_ascii() && ch.is_numeric() || match ch
|
||||
{
|
||||
'A'..='F' | 'a'..='f' => true,
|
||||
_ => false,
|
||||
}
|
||||
{
|
||||
*last_index = *index as i32;
|
||||
token_str.push(ch);
|
||||
*token = Some(Token::Numeral(token_str.clone()));
|
||||
*state = TokenizerState::HexNumber;
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenize_backtrack(last_index, index, tokens, token, token_str, state)?;
|
||||
}
|
||||
}
|
||||
TokenizerState::HexNumber =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'p' =>
|
||||
{
|
||||
token_str.push(ch);
|
||||
*token = None;
|
||||
*state = TokenizerState::HexExpNumber;
|
||||
}
|
||||
_ =>
|
||||
{
|
||||
if ch.is_ascii() && ch.is_numeric() || match ch
|
||||
{
|
||||
'A'..='F' | 'a'..='f' => true,
|
||||
_ => false,
|
||||
}
|
||||
{
|
||||
*last_index = *index as i32;
|
||||
token_str.push(ch);
|
||||
*token = Some(Token::Numeral(token_str.clone()));
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenize_backtrack(last_index, index, tokens, token, token_str, state)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
TokenizerState::Number =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'e' =>
|
||||
{
|
||||
token_str.push(ch);
|
||||
*token = None;
|
||||
*state = TokenizerState::ExpNumber;
|
||||
}
|
||||
_ =>
|
||||
{
|
||||
if ch.is_numeric() && ch.is_ascii()
|
||||
{
|
||||
*last_index = *index as i32;
|
||||
token_str.push(ch);
|
||||
*token = Some(Token::Numeral(token_str.clone()));
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenize_backtrack(last_index, index, tokens, token, token_str, state)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
TokenizerState::Comma | TokenizerState::RoundOpen | TokenizerState::RoundClosed |
|
||||
TokenizerState::CurlyOpen | TokenizerState::CurlyClosed | TokenizerState::Plus |
|
||||
TokenizerState::TildeEquals | TokenizerState::EqualsEquals | TokenizerState::Hash |
|
||||
TokenizerState::GtEquals | TokenizerState::LtEquals | TokenizerState::SquareOpen |
|
||||
TokenizerState::SquareClosed | TokenizerState::Pipe | TokenizerState::Ampersand |
|
||||
TokenizerState::Percent | TokenizerState::Star | TokenizerState::Semicolon |
|
||||
TokenizerState::Caret | TokenizerState::DotDotDot | TokenizerState::GtGt |
|
||||
TokenizerState::LtLt | TokenizerState::SlashSlash => tokenize_backtrack(last_index, index, tokens, token, token_str, state)?,
|
||||
TokenizerState::Tilde =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'=' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::TildeEquals), TokenizerState::TildeEquals),
|
||||
_ => tokenize_backtrack(last_index, index, tokens, token, token_str, state)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Gt =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'>' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::GtGt), TokenizerState::GtGt),
|
||||
'=' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::GtEquals), TokenizerState::GtEquals),
|
||||
_ => tokenize_backtrack(last_index, index, tokens, token, token_str, state)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Lt =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'>' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::LtLt), TokenizerState::LtLt),
|
||||
'=' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::LtEquals), TokenizerState::LtEquals),
|
||||
_ => tokenize_backtrack(last_index, index, tokens, token, token_str, state)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Slash =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'/' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::SlashSlash), TokenizerState::SlashSlash),
|
||||
_ => tokenize_backtrack(last_index, index, tokens, token, token_str, state)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Dot =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'.' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::DotDot), TokenizerState::DotDot),
|
||||
_ => tokenize_backtrack(last_index, index, tokens, token, token_str, state)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::DotDot =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'.' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::DotDotDot), TokenizerState::DotDotDot),
|
||||
_ => tokenize_backtrack(last_index, index, tokens, token, token_str, state)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Colon =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
':' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::ColonColon), TokenizerState::ColonColon),
|
||||
_ => tokenize_backtrack(last_index, index, tokens, token, token_str, state)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Equals =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'=' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::EqualsEquals), TokenizerState::EqualsEquals),
|
||||
_ => tokenize_backtrack(last_index, index, tokens, token, token_str, state)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Minus =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'-' => tokenize_terminal_no_str(last_index, *index, token, state, None, TokenizerState::SmallCommentStart),
|
||||
_ => tokenize_backtrack(last_index, index, tokens, token, token_str, state)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::SmallCommentStart =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'[' =>
|
||||
{
|
||||
*token = None;
|
||||
*state = TokenizerState::BigCommentLongBracketStart;
|
||||
}
|
||||
'\n' =>
|
||||
{
|
||||
*state = TokenizerState::Start;
|
||||
*last_index = -1;
|
||||
}
|
||||
_ =>
|
||||
{
|
||||
*state = TokenizerState::SmallComment;
|
||||
}
|
||||
}
|
||||
}
|
||||
TokenizerState::SmallComment =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'\n' =>
|
||||
{
|
||||
*state = TokenizerState::Start;
|
||||
*last_index = -1;
|
||||
}
|
||||
_ => { }
|
||||
}
|
||||
}
|
||||
TokenizerState::BigCommentLongBracketStart =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'=' =>
|
||||
{
|
||||
*long_bracket_level += 1;
|
||||
}
|
||||
'[' =>
|
||||
{
|
||||
*state = TokenizerState::BigComment;
|
||||
}
|
||||
_ => return Err("Malformed long bracket at the beginning of a big comment"),
|
||||
}
|
||||
}
|
||||
TokenizerState::BigComment =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
']' =>
|
||||
{
|
||||
*state = TokenizerState::BigCommentLongBracketEnd;
|
||||
}
|
||||
_ => { }
|
||||
}
|
||||
}
|
||||
TokenizerState::BigCommentLongBracketEnd =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'=' =>
|
||||
{
|
||||
if *long_bracket_level == 0
|
||||
{
|
||||
return Err("Long bracket level too big when ending big comment");
|
||||
}
|
||||
*long_bracket_level -= 1;
|
||||
}
|
||||
']' =>
|
||||
{
|
||||
if *long_bracket_level != 0
|
||||
{
|
||||
return Err("Long bracket level too small when ending big comment");
|
||||
}
|
||||
*state = TokenizerState::Start;
|
||||
}
|
||||
_ => return Err("Malformed long bracket when ending big comment"),
|
||||
}
|
||||
}
|
||||
TokenizerState::A =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'n' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::An, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::An =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'd' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::And, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::And => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::And)?,
|
||||
TokenizerState::W =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'h' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Wh, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Wh =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'i' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Whi, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Whi =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'l' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Whil, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Whil =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'e' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::While, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::While => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::While)?,
|
||||
TokenizerState::B =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'r' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Br, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Br =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'e' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Bre, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Bre =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'a' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Brea, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Brea =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'k' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Break, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Break => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Break)?,
|
||||
TokenizerState::G =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'o' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Go, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Go =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
't' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Got, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Got =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'o' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Goto, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Goto => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Goto)?,
|
||||
TokenizerState::R =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'e' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Re, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Re =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
't' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Ret, token_str, ch),
|
||||
'p' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Rep, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Ret =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'u' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Retu, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Retu =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'r' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Retur, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Retur =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'n' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Return, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Return => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Return)?,
|
||||
TokenizerState::Rep =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'e' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Repe, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Repe =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'a' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Repea, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Repea =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
't' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Repeat, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Repeat => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Repeat)?,
|
||||
TokenizerState::N =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'i' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Ni, token_str, ch),
|
||||
'o' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::No, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::No =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
't' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Not, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Not => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Not)?,
|
||||
TokenizerState::Ni =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'l' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Nil, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Nil => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Nil)?,
|
||||
TokenizerState::T =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'h' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Th, token_str, ch),
|
||||
'r' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Tr, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Th =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'e' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::The, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::The =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'n' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Then, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Then => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Then)?,
|
||||
TokenizerState::Tr =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'u' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Tru, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Tru =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'e' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::True, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::True => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::True)?,
|
||||
TokenizerState::E =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'l' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::El, token_str, ch),
|
||||
'n' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::En, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::En =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'd' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::End, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::End => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::End)?,
|
||||
TokenizerState::El =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
's' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Els, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Els =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'e' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Else, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Else =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'i' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Elsei, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Else)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Elsei =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'f' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Elseif, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Elseif => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Elseif)?,
|
||||
TokenizerState::O =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'r' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Or, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Or => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Or)?,
|
||||
TokenizerState::D =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'o' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Do, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Do => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Do)?,
|
||||
TokenizerState::I =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'f' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::If, token_str, ch),
|
||||
'n' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::In, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::In => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::In)?,
|
||||
TokenizerState::If => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::If)?,
|
||||
TokenizerState::F =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'a' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Fa, token_str, ch),
|
||||
'o' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Fo, token_str, ch),
|
||||
'u' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Fu, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Fu =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'n' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Fun, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Fun =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'c' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Func, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Func =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
't' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Funct, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Funct =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'i' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Functi, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Functi =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'o' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Functio, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Functio =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'n' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Function, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Function => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Function)?,
|
||||
TokenizerState::Fa =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'l' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Fal, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Fal =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
's' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Fals, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Fals =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'e' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::False, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::False => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::False)?,
|
||||
TokenizerState::Fo =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'r' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::For, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::For => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::For)?,
|
||||
TokenizerState::L =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'o' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Lo, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Lo =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'c' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Loc, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Loc =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'a' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Loca, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Loca =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'l' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Local, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Local => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Local)?,
|
||||
TokenizerState::U =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'n' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Un, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Un =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
't' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Unt, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Unt =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'i' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Unti, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Unti =>
|
||||
{
|
||||
match ch
|
||||
{
|
||||
'l' => tokenize_terminal_no_token(last_index, *index, state, TokenizerState::Until, token_str, ch),
|
||||
_ => tokenize_alphanumeric_nonstart(last_index, index, tokens, token, token_str, state, ch)?,
|
||||
}
|
||||
}
|
||||
TokenizerState::Until => tokenize_alphanumeric_nonstart_custom(last_index, index, tokens, token, token_str, state, ch, Token::Until)?,
|
||||
_ => todo!("State: {:?}", state),
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
pub fn tokenize(file_content: &String) -> Result<Vec<Token>, &'static str>
|
||||
{
|
||||
let mut tokens: Vec<Token> = Vec::new();
|
||||
let mut state = TokenizerState::Start;
|
||||
let char_vec: Vec<char> = file_content.chars().collect();
|
||||
|
||||
let mut last_index: i32 = -1;
|
||||
let mut index = 0;
|
||||
let mut token: Option<Token> = None;
|
||||
let mut token_str: String = String::new();
|
||||
let mut long_bracket_level = 0;
|
||||
|
||||
while index < char_vec.len()
|
||||
{
|
||||
let ch = char_vec[index];
|
||||
tokenize_char(&mut state, ch, &mut last_index, &mut index, &mut token, &mut token_str, &mut tokens, &mut long_bracket_level)?;
|
||||
index += 1;
|
||||
}
|
||||
match state
|
||||
{
|
||||
TokenizerState::Name => tokenize_backtrack_name(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state)?,
|
||||
TokenizerState::End => tokenize_backtrack_custom_token(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, Token::End)?,
|
||||
TokenizerState::And => tokenize_backtrack_custom_token(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, Token::And)?,
|
||||
TokenizerState::Semicolon => tokenize_backtrack_custom_token(&mut last_index, &mut index, &mut tokens, &mut token, &mut token_str, &mut state, Token::Semicolon)?,
|
||||
TokenizerState::Number =>
|
||||
{
|
||||
if let Some(numeral_token) = token
|
||||
{
|
||||
if let Token::Numeral(_) = numeral_token
|
||||
{
|
||||
tokens.push(numeral_token);
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("In number state but current token is not a numeral")
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return Err("In number state but no current token")
|
||||
}
|
||||
}
|
||||
TokenizerState::Start =>
|
||||
{
|
||||
if token.is_some()
|
||||
{
|
||||
return Err("Finished tokenizing in the start state but the token was non-empty");
|
||||
}
|
||||
}
|
||||
_ => todo!("state: {:?} {:?}", state, token),
|
||||
}
|
||||
|
||||
return Ok(tokens);
|
||||
}
|
||||
1246
src/tokenizer.zig
1246
src/tokenizer.zig
@ -1,1246 +0,0 @@
|
||||
const types = @import("types.zig");
|
||||
const std = @import("std");
|
||||
const CodeRegion = @import("types.zig").CodeRegion;
|
||||
const CodeLocation = @import("types.zig").CodeLocation;
|
||||
|
||||
pub const TokenType = enum
|
||||
{
|
||||
Name,
|
||||
And, Break, Do, Else, Elseif, End,
|
||||
False, For, Function, Goto, If, In,
|
||||
Local, Nil, Not, Or, Repeat, Return,
|
||||
Then, True, Until, While,
|
||||
Plus, Minus, Star, Slash, Percent, Caret, Hash,
|
||||
Ampersand, Tilde, Pipe, LtLt, GtGt, SlashSlash,
|
||||
EqualsEquals, TildeEquals, LtEquals, GtEquals, Lt, Gt, Equals,
|
||||
RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, ColonColon,
|
||||
Semicolon, Colon, Comma, Dot, DotDot, DotDotDot,
|
||||
Numeral,
|
||||
StringLiteral,
|
||||
|
||||
};
|
||||
|
||||
const TokenData = union(enum)
|
||||
{
|
||||
string: []u8,
|
||||
numeral: types.Numeral,
|
||||
none,
|
||||
};
|
||||
|
||||
pub const Token = struct
|
||||
{
|
||||
tokenType: TokenType,
|
||||
tokenData: TokenData,
|
||||
region: CodeRegion,
|
||||
};
|
||||
|
||||
const TokenizerState = enum
|
||||
{
|
||||
Start,
|
||||
Quote, SingleQuote, Name, Number, Zero,
|
||||
A, B, D, E, F, G, I, L, N, O, R, T, U, W,
|
||||
Plus, Minus, Star, Slash, Percent, Caret, Hash,
|
||||
Ampersand, Tilde, Pipe, Lt, Gt, Equals, RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed,
|
||||
Colon, Semicolon, Comma, Dot,
|
||||
|
||||
An, Br, Do, El, En, Fa, Fo, Fu, Go, If, In, Lo, Ni, No, Or, Re, Th, Tr, Un, Wh,
|
||||
LtLt, GtGt, SlashSlash, EqualsEquals, TildeEquals, LtEquals, GtEquals, ColonColon, DotDot,
|
||||
SmallCommentStart, QuoteBackslash, SingleQuoteBackslash, String, HexNumberX, ExpNumber,
|
||||
|
||||
And, Bre, Els, End, Fal, For, Fun, Got, Loc, Nil, Not, Rep, Ret, The, Tru, Unt, Whi,
|
||||
DotDotDot, HexNumber, QuoteBackslashZ, SingleQuoteBackslashZ,
|
||||
BigCommentLongBracketStart, SmallComment,
|
||||
|
||||
Brea, Else, Fals, Func, Goto, Loca, Repe, Retu, Then, True, Unti, Whil, HexExpNumber,
|
||||
BigComment, BigCommentLongBracketEnd,
|
||||
|
||||
Break, Elsei, False, Funct, Local, Repea, Retur, Until, While,
|
||||
|
||||
Elseif, Functi, Repeat, Return,
|
||||
|
||||
Functio,
|
||||
|
||||
Function,
|
||||
};
|
||||
|
||||
fn tokenizeUpdateIndexAndState(lastIndex: *?usize, index: ?usize, state: *TokenizerState, newState: TokenizerState, region: *CodeRegion) void
|
||||
{
|
||||
lastIndex.* = index;
|
||||
state.* = newState;
|
||||
if(index == null)
|
||||
{
|
||||
region.*.start = null;
|
||||
region.*.length = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
if(region.*.start == null)
|
||||
{
|
||||
// TODO: There is no line/col info here and plumbing it to here would be pain.
|
||||
region.*.start = CodeLocation { .col = 0, .line = 0 };
|
||||
}
|
||||
region.*.length += 1;
|
||||
}
|
||||
}
|
||||
fn tokenizeTerminalBase(lastIndex: *?usize, index: ?usize, tokenType: *?TokenType, state: *TokenizerState, newTokenType: ?TokenType, newState: TokenizerState, region: *CodeRegion) void
|
||||
{
|
||||
tokenizeUpdateIndexAndState(lastIndex, index, state, newState, region);
|
||||
tokenType.* = newTokenType;
|
||||
}
|
||||
fn tokenizeTerminalStr(lastIndex: *?usize, index: usize, tokenType: *?TokenType, state: *TokenizerState, newTokenType: ?TokenType, newState: TokenizerState, tokenStr: *std.ArrayList(u8), ch: u8, region: *CodeRegion) !void
|
||||
{
|
||||
tokenizeTerminalBase(lastIndex, index, tokenType, state, newTokenType, newState, region);
|
||||
try tokenStr.append(ch);
|
||||
}
|
||||
fn tokenizeTerminalIntNum(lastIndex: *?usize, index: usize, tokenType: *?TokenType, state: *TokenizerState, newTokenType: TokenType, newState: TokenizerState, tokenNumeral: *?types.Numeral, ch: u8, region: *CodeRegion) !void
|
||||
{
|
||||
tokenizeTerminalBase(lastIndex, index, tokenType, state, newTokenType, newState, region);
|
||||
if(!std.ascii.isDigit(ch))
|
||||
{
|
||||
return error.NoDigit;
|
||||
}
|
||||
const digitValue = @as(i64, ch - '0');
|
||||
if(tokenNumeral.* == null)
|
||||
{
|
||||
tokenNumeral.* = types.Numeral { .Integer = digitValue };
|
||||
}
|
||||
else
|
||||
{
|
||||
switch(tokenNumeral.*.?)
|
||||
{
|
||||
.Integer => |*n| n.* = n.* * 10 + digitValue,
|
||||
.Float => return error.ExpectedIntGotFloat
|
||||
}
|
||||
}
|
||||
}
|
||||
fn tokenizeTerminalNoToken(lastIndex: *?usize, index: usize, state: *TokenizerState, newState: TokenizerState, tokenStr: *std.ArrayList(u8), ch: u8, region: *CodeRegion) !void
|
||||
{
|
||||
tokenizeUpdateIndexAndState(lastIndex, index, state, newState, region);
|
||||
try tokenStr.*.append(ch);
|
||||
}
|
||||
fn tokenizeBacktrack(lastIndex: *?usize, index: *usize, tokens: *std.ArrayList(Token), tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, state: *TokenizerState, allocator: std.mem.Allocator, region: *CodeRegion) !void
|
||||
{
|
||||
try tokenizeBacktrackCustomToken(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, tokenType.*.?, allocator, region);
|
||||
}
|
||||
fn tokenizeBacktrackCustomToken(lastIndex: *?usize, index: *usize, tokens: *std.ArrayList(Token), tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, state: *TokenizerState, newTokenType: TokenType, allocator: std.mem.Allocator, region: *CodeRegion) !void
|
||||
{
|
||||
if(lastIndex.* == null or tokenType.* == null)
|
||||
{
|
||||
return error.LexError;
|
||||
}
|
||||
if(newTokenType == TokenType.StringLiteral or newTokenType == TokenType.Name)
|
||||
{
|
||||
const content = try allocator.alloc(u8, tokenStr.*.items.len);
|
||||
@memcpy(content, tokenStr.*.items);
|
||||
try tokens.append(Token { .tokenType = newTokenType, .tokenData = TokenData { .string = content }, .region = region.* });
|
||||
}
|
||||
else
|
||||
{
|
||||
try tokens.append(Token { .tokenType = newTokenType, .region = region.*, .tokenData = if(tokenType.*.? == TokenType.Numeral) TokenData { .numeral = tokenNumeral.*.? }
|
||||
else TokenData.none
|
||||
});
|
||||
}
|
||||
tokenNumeral.* = null;
|
||||
index.* = lastIndex.*.?;
|
||||
tokenStr.*.clearAndFree();
|
||||
// region is reset in tokenizeTerminalBase since null is passed as index
|
||||
tokenizeTerminalBase(lastIndex, null, tokenType, state, null, TokenizerState.Start, region);
|
||||
}
|
||||
fn tokenizeAlphanumericNonstart(lastIndex: *?usize, index: *usize, tokens: *std.ArrayList(Token), tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, state: *TokenizerState, ch: u8, newTokenType: TokenType, allocator: std.mem.Allocator, region: *CodeRegion) !void
|
||||
{
|
||||
if(std.ascii.isAlphanumeric(ch) or ch == '_')
|
||||
{
|
||||
try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.Name, tokenStr, ch, region);
|
||||
}
|
||||
else
|
||||
{
|
||||
try tokenizeBacktrackCustomToken(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, newTokenType, allocator, region);
|
||||
}
|
||||
}
|
||||
fn tokenizeChar(state: *TokenizerState, ch: u8, lastIndex: *?usize, index: *usize, tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, tokens: *std.ArrayList(Token), longBracketLevel: *u32, region: *CodeRegion, allocator: std.mem.Allocator) !void
|
||||
{
|
||||
switch(state.*)
|
||||
{
|
||||
TokenizerState.Start =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'-' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Minus, TokenizerState.Minus, region),
|
||||
',' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Comma, TokenizerState.Comma, region),
|
||||
'=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Equals, TokenizerState.Equals, region),
|
||||
'(' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.RoundOpen, TokenizerState.RoundOpen, region),
|
||||
')' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.RoundClosed, TokenizerState.RoundClosed, region),
|
||||
'.' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Dot, TokenizerState.Dot, region),
|
||||
':' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Colon, TokenizerState.Colon, region),
|
||||
'{' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.CurlyOpen, TokenizerState.CurlyOpen, region),
|
||||
'}' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.CurlyClosed, TokenizerState.CurlyClosed, region),
|
||||
'[' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.SquareOpen, TokenizerState.SquareOpen, region),
|
||||
']' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.SquareClosed, TokenizerState.SquareClosed, region),
|
||||
'+' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Plus, TokenizerState.Plus, region),
|
||||
'~' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Tilde, TokenizerState.Tilde, region),
|
||||
'>' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Gt, TokenizerState.Gt, region),
|
||||
'<' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Lt, TokenizerState.Lt, region),
|
||||
'#' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Hash, TokenizerState.Hash, region),
|
||||
'|' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Pipe, TokenizerState.Pipe, region),
|
||||
'&' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Ampersand, TokenizerState.Ampersand, region),
|
||||
'%' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Percent, TokenizerState.Percent, region),
|
||||
'*' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Star, TokenizerState.Star, region),
|
||||
'/' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Slash, TokenizerState.Slash, region),
|
||||
';' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Semicolon, TokenizerState.Semicolon, region),
|
||||
'^' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Caret, TokenizerState.Caret, region),
|
||||
'a' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.A, tokenStr, ch, region),
|
||||
'b' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.B, tokenStr, ch, region),
|
||||
'd' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.D, tokenStr, ch, region),
|
||||
'e' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.E, tokenStr, ch, region),
|
||||
'f' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.F, tokenStr, ch, region),
|
||||
'i' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.I, tokenStr, ch, region),
|
||||
'g' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.G, tokenStr, ch, region),
|
||||
'l' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.L, tokenStr, ch, region),
|
||||
'n' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.N, tokenStr, ch, region),
|
||||
'o' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.O, tokenStr, ch, region),
|
||||
'r' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.R, tokenStr, ch, region),
|
||||
't' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.T, tokenStr, ch, region),
|
||||
'u' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.U, tokenStr, ch, region),
|
||||
'w' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.W, tokenStr, ch, region),
|
||||
'0' => try tokenizeTerminalIntNum(lastIndex, index.*, tokenType, state, TokenType.Numeral, TokenizerState.Zero, tokenNumeral, ch, region),
|
||||
'"' =>
|
||||
{
|
||||
tokenType.* = null;
|
||||
state.* = TokenizerState.Quote;
|
||||
},
|
||||
'\'' =>
|
||||
{
|
||||
tokenType.* = null;
|
||||
state.* = TokenizerState.SingleQuote;
|
||||
},
|
||||
else =>
|
||||
{
|
||||
if(std.ascii.isWhitespace(ch))
|
||||
{
|
||||
|
||||
}
|
||||
else if(std.ascii.isAlphabetic(ch) or ch == '_')
|
||||
{
|
||||
try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.Name, tokenStr, ch, region);
|
||||
}
|
||||
else if(std.ascii.isDigit(ch))
|
||||
{
|
||||
try tokenizeTerminalIntNum(lastIndex, index.*, tokenType, state, TokenType.Numeral, TokenizerState.Number, tokenNumeral, ch, region);
|
||||
}
|
||||
else
|
||||
{
|
||||
std.debug.print("{}: {c}\n", .{state.*, ch});
|
||||
return error.NotImplemented;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
TokenizerState.Quote =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'\\' => state.* = TokenizerState.QuoteBackslash,
|
||||
'"' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region),
|
||||
else => try tokenStr.*.append(ch),
|
||||
}
|
||||
},
|
||||
TokenizerState.QuoteBackslash =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'a' =>
|
||||
{
|
||||
try tokenStr.append('\u{0007}');
|
||||
state.* = TokenizerState.Quote;
|
||||
},
|
||||
'b' =>
|
||||
{
|
||||
try tokenStr.append('\u{0008}');
|
||||
state.* = TokenizerState.Quote;
|
||||
},
|
||||
't' =>
|
||||
{
|
||||
try tokenStr.append('\t');
|
||||
state.* = TokenizerState.Quote;
|
||||
},
|
||||
'n' | '\n' =>
|
||||
{
|
||||
try tokenStr.append('\n');
|
||||
state.* = TokenizerState.Quote;
|
||||
},
|
||||
'v' =>
|
||||
{
|
||||
try tokenStr.append('\u{000b}');
|
||||
state.* = TokenizerState.Quote;
|
||||
},
|
||||
'f' =>
|
||||
{
|
||||
try tokenStr.append('\u{000c}');
|
||||
state.* = TokenizerState.Quote;
|
||||
},
|
||||
'r' =>
|
||||
{
|
||||
try tokenStr.append('\r');
|
||||
state.* = TokenizerState.Quote;
|
||||
},
|
||||
'\\' =>
|
||||
{
|
||||
try tokenStr.append('\\');
|
||||
state.* = TokenizerState.Quote;
|
||||
},
|
||||
'"' =>
|
||||
{
|
||||
try tokenStr.append('\"');
|
||||
state.* = TokenizerState.Quote;
|
||||
},
|
||||
'\'' =>
|
||||
{
|
||||
try tokenStr.append('\'');
|
||||
state.* = TokenizerState.Quote;
|
||||
},
|
||||
'z' =>
|
||||
{
|
||||
state.* = TokenizerState.QuoteBackslashZ;
|
||||
},
|
||||
else => return error.UnknownEscapeSequence,
|
||||
}
|
||||
},
|
||||
TokenizerState.QuoteBackslashZ =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'\\' => state.* = TokenizerState.QuoteBackslash,
|
||||
'"' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region),
|
||||
else =>
|
||||
{
|
||||
if(!std.ascii.isWhitespace(ch))
|
||||
{
|
||||
try tokenStr.append(ch);
|
||||
state.* = TokenizerState.Quote;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Noop, https://www.lua.org/manual/5.4/manual.html#3.1:
|
||||
// "The escape sequence '\z' skips the following span of whitespace characters, including line breaks;"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
TokenizerState.SingleQuote =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'\\' => state.* = TokenizerState.SingleQuoteBackslash,
|
||||
'\'' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region),
|
||||
else => try tokenStr.append(ch),
|
||||
}
|
||||
},
|
||||
TokenizerState.SingleQuoteBackslash =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'a' =>
|
||||
{
|
||||
try tokenStr.append('\u{0007}');
|
||||
state.* = TokenizerState.SingleQuote;
|
||||
},
|
||||
'b' =>
|
||||
{
|
||||
try tokenStr.append('\u{0008}');
|
||||
state.* = TokenizerState.SingleQuote;
|
||||
},
|
||||
't' =>
|
||||
{
|
||||
try tokenStr.append('\t');
|
||||
state.* = TokenizerState.SingleQuote;
|
||||
},
|
||||
'n' | '\n' =>
|
||||
{
|
||||
try tokenStr.append('\n');
|
||||
state.* = TokenizerState.SingleQuote;
|
||||
},
|
||||
'v' =>
|
||||
{
|
||||
try tokenStr.append('\u{000b}');
|
||||
state.* = TokenizerState.SingleQuote;
|
||||
},
|
||||
'f' =>
|
||||
{
|
||||
try tokenStr.append('\u{000c}');
|
||||
state.* = TokenizerState.SingleQuote;
|
||||
},
|
||||
'r' =>
|
||||
{
|
||||
try tokenStr.append('\r');
|
||||
state.* = TokenizerState.SingleQuote;
|
||||
},
|
||||
'\\' =>
|
||||
{
|
||||
try tokenStr.append('\\');
|
||||
state.* = TokenizerState.SingleQuote;
|
||||
},
|
||||
'"' =>
|
||||
{
|
||||
try tokenStr.append('\"');
|
||||
state.* = TokenizerState.SingleQuote;
|
||||
},
|
||||
'\'' =>
|
||||
{
|
||||
try tokenStr.append('\'');
|
||||
state.* = TokenizerState.SingleQuote;
|
||||
},
|
||||
'z' =>
|
||||
{
|
||||
state.* = TokenizerState.SingleQuoteBackslashZ;
|
||||
},
|
||||
else => return error.UnknownEscapeSequence,
|
||||
}
|
||||
},
|
||||
TokenizerState.SingleQuoteBackslashZ =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'\\' => state.* = TokenizerState.SingleQuoteBackslash,
|
||||
'\'' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region),
|
||||
else =>
|
||||
{
|
||||
if(!std.ascii.isWhitespace(ch))
|
||||
{
|
||||
try tokenStr.append(ch);
|
||||
state.* = TokenizerState.SingleQuote;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Noop, https://www.lua.org/manual/5.4/manual.html#3.1:
|
||||
// "The escape sequence '\z' skips the following span of whitespace characters, including line breaks;"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
TokenizerState.String => try tokenizeBacktrackCustomToken(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, TokenType.StringLiteral, allocator, region),
|
||||
TokenizerState.Name => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
TokenizerState.Zero =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'x' =>
|
||||
{
|
||||
try tokenStr.*.append(ch);
|
||||
tokenType.* = null;
|
||||
state.* = TokenizerState.HexNumberX;
|
||||
},
|
||||
'.' => return error.NotImplemented,
|
||||
else =>
|
||||
{
|
||||
if(std.ascii.isDigit(ch))
|
||||
{
|
||||
const digitValue = @as(i64, ch - '0');
|
||||
lastIndex.* = index.*;
|
||||
if(tokenNumeral.* == null)
|
||||
{
|
||||
tokenNumeral.* = types.Numeral { .Integer = digitValue };
|
||||
tokenType.* = TokenType.Numeral;
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenNumeral.*.?.Integer = tokenNumeral.*.?.Integer * 10 + digitValue;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
TokenizerState.HexNumberX =>
|
||||
{
|
||||
if(std.ascii.isHex(ch))
|
||||
{
|
||||
lastIndex.* = index.*;
|
||||
tokenType.* = TokenType.Numeral;
|
||||
if(std.ascii.isDigit(ch))
|
||||
{
|
||||
tokenNumeral.* = types.Numeral { .Integer = @as(i64, ch - '0') };
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenNumeral.* = types.Numeral { .Integer = @as(i64, std.ascii.toLower(ch) - 'a') };
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region);
|
||||
}
|
||||
},
|
||||
TokenizerState.HexNumber =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'p' =>
|
||||
{
|
||||
try tokenStr.*.append(ch);
|
||||
tokenType.* = null;
|
||||
state.* = TokenizerState.HexExpNumber;
|
||||
},
|
||||
else =>
|
||||
{
|
||||
if(std.ascii.isHex(ch))
|
||||
{
|
||||
lastIndex.* = index.*;
|
||||
tokenType.* = TokenType.Numeral;
|
||||
if(std.ascii.isDigit(ch))
|
||||
{
|
||||
tokenNumeral.* = types.Numeral { .Integer = @as(i64, ch - '0') };
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenNumeral.* = types.Numeral { .Integer = @as(i64, std.ascii.toLower(ch) - 'a') };
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
TokenizerState.Number =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'e' =>
|
||||
{
|
||||
try tokenStr.*.append(ch);
|
||||
tokenType.* = null;
|
||||
state.* = TokenizerState.ExpNumber;
|
||||
},
|
||||
'.' => return error.NotImplemented,
|
||||
else =>
|
||||
{
|
||||
if(std.ascii.isDigit(ch))
|
||||
{
|
||||
const digitValue = @as(i64, ch - '0');
|
||||
lastIndex.* = index.*;
|
||||
if(tokenNumeral.* == null)
|
||||
{
|
||||
tokenNumeral.* = types.Numeral { .Integer = digitValue };
|
||||
tokenType.* = TokenType.Numeral;
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenNumeral.*.?.Integer = tokenNumeral.*.?.Integer * 10 + digitValue;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
},
|
||||
TokenizerState.Comma, TokenizerState.RoundOpen, TokenizerState.RoundClosed,
|
||||
TokenizerState.CurlyOpen, TokenizerState.CurlyClosed, TokenizerState.Plus,
|
||||
TokenizerState.TildeEquals, TokenizerState.EqualsEquals, TokenizerState.Hash,
|
||||
TokenizerState.GtEquals, TokenizerState.LtEquals, TokenizerState.SquareOpen,
|
||||
TokenizerState.SquareClosed, TokenizerState.Pipe, TokenizerState.Ampersand,
|
||||
TokenizerState.Percent, TokenizerState.Star, TokenizerState.Semicolon,
|
||||
TokenizerState.Caret, TokenizerState.DotDotDot, TokenizerState.GtGt,
|
||||
TokenizerState.LtLt, TokenizerState.SlashSlash => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
|
||||
TokenizerState.Tilde =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.TildeEquals, TokenizerState.TildeEquals, region),
|
||||
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Gt =>
|
||||
{
|
||||
switch (ch)
|
||||
{
|
||||
'>' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.GtGt, TokenizerState.GtGt, region),
|
||||
'=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.GtEquals, TokenizerState.GtEquals, region),
|
||||
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Lt =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'<' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.LtLt, TokenizerState.LtLt, region),
|
||||
'=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.LtEquals, TokenizerState.LtEquals, region),
|
||||
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Slash =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'/' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.SlashSlash, TokenizerState.SlashSlash, region),
|
||||
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Dot =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'.' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.DotDot, TokenizerState.DotDot, region),
|
||||
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.DotDot =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'.' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.DotDotDot, TokenizerState.DotDotDot, region),
|
||||
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Colon =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
':' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.ColonColon, TokenizerState.ColonColon, region),
|
||||
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Equals =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.EqualsEquals, TokenizerState.EqualsEquals, region),
|
||||
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Minus =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'-' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, null, TokenizerState.SmallCommentStart, region),
|
||||
else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.SmallCommentStart =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'[' =>
|
||||
{
|
||||
tokenType.* = null;
|
||||
state.* = TokenizerState.BigCommentLongBracketStart;
|
||||
},
|
||||
'\n' =>
|
||||
{
|
||||
state.* = TokenizerState.Start;
|
||||
lastIndex.* = null;
|
||||
},
|
||||
else =>
|
||||
{
|
||||
state.* = TokenizerState.SmallComment;
|
||||
},
|
||||
}
|
||||
},
|
||||
TokenizerState.SmallComment =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'\n' =>
|
||||
{
|
||||
state.* = TokenizerState.Start;
|
||||
lastIndex.* = null;
|
||||
},
|
||||
else => { }
|
||||
}
|
||||
},
|
||||
TokenizerState.BigCommentLongBracketStart =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'=' =>
|
||||
{
|
||||
longBracketLevel.* += 1;
|
||||
},
|
||||
'[' =>
|
||||
{
|
||||
state.* = TokenizerState.BigComment;
|
||||
},
|
||||
else => return error.LongBracketMalformedStartBigComment,
|
||||
}
|
||||
},
|
||||
TokenizerState.BigComment =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
']' =>
|
||||
{
|
||||
state.* = TokenizerState.BigCommentLongBracketEnd;
|
||||
},
|
||||
else => { },
|
||||
}
|
||||
},
|
||||
TokenizerState.BigCommentLongBracketEnd =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'=' =>
|
||||
{
|
||||
if(longBracketLevel.* == 0)
|
||||
{
|
||||
return error.LongBracketLevelTooBigEndBigComment;
|
||||
}
|
||||
longBracketLevel.* -= 1;
|
||||
},
|
||||
']' =>
|
||||
{
|
||||
if(longBracketLevel.* != 0)
|
||||
{
|
||||
return error.LongBracketLevelTooSmallEndBigComment;
|
||||
}
|
||||
state.* = TokenizerState.Start;
|
||||
},
|
||||
else => return error.LongBracketMalformedSmallEndBigComment,
|
||||
}
|
||||
},
|
||||
TokenizerState.A =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.An, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.An =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'd' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.And, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.And => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.And, allocator, region),
|
||||
TokenizerState.W =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'h' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Wh, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Wh =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Whi, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Whi =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Whil, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Whil =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.While, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.While => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.While, allocator, region),
|
||||
TokenizerState.B =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Br, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Br =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Bre, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Bre =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Brea, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Brea =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'k' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Break, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Break => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Break, allocator, region),
|
||||
TokenizerState.G =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Go, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Go =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Got, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Got =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Goto, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Goto => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Goto, allocator, region),
|
||||
TokenizerState.R =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Re, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Re =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Ret, tokenStr, ch, region),
|
||||
'p' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Rep, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Ret =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'u' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Retu, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Retu =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Retur, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Retur =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Return, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Return => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Return, allocator, region),
|
||||
TokenizerState.Rep =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Repe, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Repe =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Repea, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Repea =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Repeat, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Repeat => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Repeat, allocator, region),
|
||||
TokenizerState.N =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Ni, tokenStr, ch, region),
|
||||
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.No, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.No =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Not, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Not => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Not, allocator, region),
|
||||
TokenizerState.Ni =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Nil, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Nil => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Nil, allocator, region),
|
||||
TokenizerState.T =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'h' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Th, tokenStr, ch, region),
|
||||
'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Tr, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Th =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.The, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.The =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Then, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Then => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Then, allocator, region),
|
||||
TokenizerState.Tr =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'u' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Tru, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Tru =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.True, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.True => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.True, allocator, region),
|
||||
TokenizerState.E =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.El, tokenStr, ch, region),
|
||||
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.En, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.En =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'd' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.End, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.End => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.End, allocator, region),
|
||||
TokenizerState.El =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
's' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Els, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Els =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Else, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Else =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Elsei, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Else, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Elsei =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'f' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Elseif, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Elseif => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Elseif, allocator, region),
|
||||
TokenizerState.O =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Or, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Or => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Or, allocator, region),
|
||||
TokenizerState.D =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Do, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Do => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Do, allocator, region),
|
||||
TokenizerState.I =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'f' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.If, tokenStr, ch, region),
|
||||
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.In, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.In => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.In, allocator, region),
|
||||
TokenizerState.If => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.If, allocator, region),
|
||||
TokenizerState.F =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fa, tokenStr, ch, region),
|
||||
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fo, tokenStr, ch, region),
|
||||
'u' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fu, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Fu =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fun, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Fun =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'c' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Func, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Func =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Funct, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Funct =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Functi, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Functi =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Functio, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Functio =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Function, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Function => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Function, allocator, region),
|
||||
TokenizerState.Fa =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fal, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Fal =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
's' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fals, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Fals =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.False, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.False => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.False, allocator, region),
|
||||
TokenizerState.Fo =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.For, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.For => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.For, allocator, region),
|
||||
TokenizerState.L =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Lo, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Lo =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'c' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Loc, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Loc =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Loca, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Loca =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Local, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Local => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Local, allocator, region),
|
||||
TokenizerState.U =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Un, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Un =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Unt, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Unt =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Unti, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Unti =>
|
||||
{
|
||||
switch(ch)
|
||||
{
|
||||
'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Until, tokenStr, ch, region),
|
||||
else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region),
|
||||
}
|
||||
},
|
||||
TokenizerState.Until => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Until, allocator, region),
|
||||
else =>
|
||||
{
|
||||
std.debug.print("{}\n", . {state.*});
|
||||
return error.NotImplemented;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn tokenize(fileContent: []u8, allocator: std.mem.Allocator) ![]Token
|
||||
{
|
||||
var tokens = std.ArrayList(Token).init(allocator);
|
||||
var state: TokenizerState = TokenizerState.Start;
|
||||
var lastIndex: ?usize = null;
|
||||
var index: usize = 0;
|
||||
var tokenType: ?TokenType = null;
|
||||
var tokenStr = std.ArrayList(u8).init(allocator);
|
||||
defer tokenStr.deinit();
|
||||
var tokenNumeral: ?types.Numeral = null;
|
||||
var longBracketLevel: u32 = 0;
|
||||
var region = CodeRegion { .start = null, .length = 0 };
|
||||
|
||||
while(index < fileContent.len)
|
||||
{
|
||||
const ch = fileContent[index];
|
||||
try tokenizeChar(&state, ch, &lastIndex, &index, &tokenType, &tokenStr, &tokenNumeral, &tokens, &longBracketLevel, ®ion, allocator);
|
||||
if(region.start != null and region.start.?.col == 0 and region.start.?.line == 0)
|
||||
{
|
||||
region.start = calculatePoint(fileContent, index);
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
if(longBracketLevel != 0)
|
||||
{
|
||||
return error.UnbalancedLongBracketLevel;
|
||||
}
|
||||
try tokenizeChar(&state, '\n', &lastIndex, &index, &tokenType, &tokenStr, &tokenNumeral, &tokens, &longBracketLevel, ®ion, allocator);
|
||||
if(region.start != null and region.start.?.col == 0 and region.start.?.line == 0)
|
||||
{
|
||||
region.start = calculatePoint(fileContent, index);
|
||||
}
|
||||
return tokens.toOwnedSlice();
|
||||
}
|
||||
|
||||
fn calculatePoint(fileContent: []u8, index: usize) CodeLocation
|
||||
{
|
||||
var ret = CodeLocation { .col = 1, .line = 1 };
|
||||
for(0..index) |i|
|
||||
{
|
||||
ret.col += 1;
|
||||
if(fileContent[i] == '\n')
|
||||
{
|
||||
ret.line += 1;
|
||||
ret.col = 1;
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@ -1,142 +0,0 @@
|
||||
const std = @import("std");
|
||||
const parser = @import("parser.zig");
|
||||
const types = @import("types.zig");
|
||||
|
||||
pub fn interpret(root: parser.ChunkNode, allocator: std.mem.Allocator) !void
|
||||
{
|
||||
var _ENV = types.Table { .entries= std.ArrayList(types.TableEntry).init(allocator) };
|
||||
try walkChunk(root, &_ENV, allocator);
|
||||
}
|
||||
|
||||
fn walkChunk(node: parser.ChunkNode, environment: *types.Table, allocator: std.mem.Allocator) !void
|
||||
{
|
||||
try walkBlock(node.block, environment, allocator);
|
||||
}
|
||||
|
||||
fn walkBlock(node: parser.BlockNode, environment: *types.Table, allocator: std.mem.Allocator) !void
|
||||
{
|
||||
for(node.stats.items) |stat|
|
||||
{
|
||||
try walkStat(stat, environment, allocator);
|
||||
}
|
||||
if(node.retstat != null)
|
||||
{
|
||||
try walkRetstat(node.retstat.?, environment, allocator);
|
||||
}
|
||||
}
|
||||
|
||||
fn walkStat(node: parser.StatNode, environment: *types.Table, allocator: std.mem.Allocator) !void
|
||||
{
|
||||
switch(node)
|
||||
{
|
||||
.Assignment => |assignmentNode|
|
||||
{
|
||||
return try walkAssignmentNode(assignmentNode, environment, allocator);
|
||||
},
|
||||
else =>
|
||||
{
|
||||
std.debug.print("{any}\n", .{node});
|
||||
return error.NotImplemented;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn walkRetstat(node: parser.RetstatNode, environment: *types.Table, allocator: std.mem.Allocator) !void
|
||||
{
|
||||
_ = node;
|
||||
_ = environment;
|
||||
_ = allocator;
|
||||
return error.NotImplemented;
|
||||
}
|
||||
|
||||
fn walkAssignmentNode(node: parser.AssignmentNode, environment: *types.Table, allocator: std.mem.Allocator) !void
|
||||
{
|
||||
const results = try walkExplist(node.rhs, environment, allocator);
|
||||
var i: usize = 0;
|
||||
_ = results;
|
||||
_ = i;
|
||||
for(node.lhs.vars.items) |variable|
|
||||
{
|
||||
switch(variable)
|
||||
{
|
||||
.Indexed => |indexedNode|
|
||||
{
|
||||
_ = indexedNode;
|
||||
return error.NotImplemented;
|
||||
},
|
||||
else => return error.NotImplemented,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn walkExplist(node: parser.ExplistNode, environment: *types.Table, allocator: std.mem.Allocator) ![]types.Value
|
||||
{
|
||||
var results = std.ArrayList(types.Value).init(allocator);
|
||||
for(node.exps.items) |exp|
|
||||
{
|
||||
try results.append(try walkExp(exp, environment, allocator, false));
|
||||
}
|
||||
return results.toOwnedSlice();
|
||||
}
|
||||
|
||||
fn walkExp(node: parser.ExpNode, environment: *types.Table, allocator: std.mem.Allocator, isVariadicFunction: bool) !types.Value
|
||||
{
|
||||
switch(node)
|
||||
{
|
||||
.Nil => return types.Value.Nil,
|
||||
.False => return types.Value { .Bool = false },
|
||||
.True => return types.Value { .Bool = true },
|
||||
.Numeral => |numeral| return types.Value { .Numeral = numeral },
|
||||
.LiteralString => |string| return types.Value { .String = string },
|
||||
.Varargs =>
|
||||
{
|
||||
if(isVariadicFunction)
|
||||
{
|
||||
return error.NotImplemented;
|
||||
}
|
||||
else
|
||||
{
|
||||
return error.UseVarargsOutsideVariadicFunction;
|
||||
}
|
||||
},
|
||||
.Suffixexp => |suffixExp|
|
||||
{
|
||||
return walkSuffixexp(suffixExp.*, environment, allocator);
|
||||
},
|
||||
else =>
|
||||
{
|
||||
std.debug.print("{}\n", .{node});
|
||||
return error.NotImplemented;
|
||||
}
|
||||
}
|
||||
return error.NotImplemented;
|
||||
}
|
||||
|
||||
fn walkSuffixexp(node: parser.SuffixexpNode, environment: *types.Table, allocator: std.mem.Allocator) !types.Value
|
||||
{
|
||||
_ = allocator;
|
||||
switch(node)
|
||||
{
|
||||
.Normal => |normal|
|
||||
{
|
||||
switch(normal.firstPart)
|
||||
{
|
||||
.Name => |name|
|
||||
{
|
||||
std.debug.print("name: {s}\n", .{name});
|
||||
std.debug.print("val: {!}\n", .{environment.get(types.Value { .String = name })});
|
||||
},
|
||||
else =>
|
||||
{
|
||||
std.debug.print("{}\n", .{normal.firstPart});
|
||||
}
|
||||
}
|
||||
return error.NotImplemented;
|
||||
},
|
||||
else =>
|
||||
{
|
||||
std.debug.print("{}\n", .{node});
|
||||
return error.NotImplemented;
|
||||
}
|
||||
}
|
||||
}
|
||||
194
src/types.zig
194
src/types.zig
@ -1,194 +0,0 @@
|
||||
const std = @import("std");
|
||||
|
||||
pub const NumeralTag = enum
|
||||
{
|
||||
Integer,
|
||||
Float,
|
||||
};
|
||||
|
||||
pub fn isFloatExactInt(value: f64) bool
|
||||
{
|
||||
return value == 0 or (std.math.isNormal(value) and @floor(value) == value);
|
||||
}
|
||||
|
||||
pub const Numeral = union(NumeralTag)
|
||||
{
|
||||
Integer: i64,
|
||||
Float: f64,
|
||||
|
||||
pub fn rawEqual(self: Numeral, other: Numeral) bool
|
||||
{
|
||||
if(@as(NumeralTag, self) == @as(NumeralTag, other))
|
||||
{
|
||||
switch(self)
|
||||
{
|
||||
.Float => |value| return value == other.Float,
|
||||
.Integer => |value| return value == other.Integer,
|
||||
}
|
||||
}
|
||||
// Other is the respective other type
|
||||
switch(self)
|
||||
{
|
||||
.Float => |value|
|
||||
{
|
||||
return isFloatExactInt(value) and @as(u64, @intFromFloat(value)) == other.Integer;
|
||||
},
|
||||
.Integer => |value|
|
||||
{
|
||||
return isFloatExactInt(other.Float) and @as(u64, @intFromFloat(other.Float)) == value;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
test "float int equality"
|
||||
{
|
||||
const a = Numeral { .Float = 12.0 };
|
||||
const b = Numeral { .Integer = 12 };
|
||||
try std.testing.expect(a.rawEqual(b));
|
||||
try std.testing.expect(b.rawEqual(a));
|
||||
try std.testing.expect(a.rawEqual(a));
|
||||
try std.testing.expect(b.rawEqual(b));
|
||||
const c = Numeral { .Float = (0.2 + 0.1) * 10.0 };
|
||||
const d = Numeral { .Integer = 3 };
|
||||
try std.testing.expect(c.rawEqual(d));
|
||||
try std.testing.expect(d.rawEqual(c));
|
||||
try std.testing.expect(c.rawEqual(c));
|
||||
try std.testing.expect(d.rawEqual(d));
|
||||
const e = Numeral { .Float = 3.2 };
|
||||
try std.testing.expect(!a.rawEqual(e));
|
||||
try std.testing.expect(!b.rawEqual(e));
|
||||
try std.testing.expect(!c.rawEqual(e));
|
||||
try std.testing.expect(!d.rawEqual(e));
|
||||
try std.testing.expect(!e.rawEqual(a));
|
||||
try std.testing.expect(!e.rawEqual(b));
|
||||
try std.testing.expect(!e.rawEqual(c));
|
||||
try std.testing.expect(!e.rawEqual(d));
|
||||
}
|
||||
|
||||
pub const TableEntry = struct
|
||||
{
|
||||
key: Value,
|
||||
value: Value,
|
||||
};
|
||||
|
||||
pub const Table = struct
|
||||
{
|
||||
entries: std.ArrayList(TableEntry),
|
||||
|
||||
pub fn get(self: Table, key: Value) Value
|
||||
{
|
||||
if(@as(ValueTag, key) == ValueTag.Nil)
|
||||
{
|
||||
return Value.Nil;
|
||||
}
|
||||
for (self.entries.items) |entry|
|
||||
{
|
||||
if(entry.key.rawEqual(key))
|
||||
{
|
||||
return entry.value;
|
||||
}
|
||||
}
|
||||
return Value.Nil;
|
||||
}
|
||||
};
|
||||
|
||||
pub const ValueTag = enum
|
||||
{
|
||||
Nil,
|
||||
Bool,
|
||||
Numeral,
|
||||
String,
|
||||
Table,
|
||||
};
|
||||
pub const Value = union(ValueTag)
|
||||
{
|
||||
Nil,
|
||||
Bool: bool,
|
||||
Numeral: Numeral,
|
||||
String: []const u8,
|
||||
Table: *Table,
|
||||
|
||||
pub fn rawEqual(self: Value, other: Value) bool
|
||||
{
|
||||
if(@as(ValueTag, self) != @as(ValueTag, other))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
switch(self)
|
||||
{
|
||||
.Nil => return true,
|
||||
.Bool => |value| return value == other.Bool,
|
||||
.Numeral => |value| return value.rawEqual(other.Numeral),
|
||||
.String => |value| return std.mem.eql(u8, value, other.String),
|
||||
.Table => |value| return value == other.Table,
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
test "Value equalities"
|
||||
{
|
||||
const a = Value { .Bool = true };
|
||||
const b = Value { .Numeral = Numeral { .Integer = 1 } };
|
||||
// true != 1
|
||||
try std.testing.expect(!a.rawEqual(b));
|
||||
// 1 != true
|
||||
try std.testing.expect(!b.rawEqual(a));
|
||||
const c = Value { .Bool = false };
|
||||
// true != false
|
||||
try std.testing.expect(!a.rawEqual(c));
|
||||
// false!= true
|
||||
try std.testing.expect(!c.rawEqual(a));
|
||||
const d = Value { .Bool = true };
|
||||
// true == true
|
||||
try std.testing.expect(a.rawEqual(d));
|
||||
// true == true
|
||||
try std.testing.expect(d.rawEqual(a));
|
||||
const e = Value { .String = "foo" };
|
||||
const f = Value { .String = "bar" };
|
||||
// foo != bar
|
||||
try std.testing.expect(!e.rawEqual(f));
|
||||
// bar != foo
|
||||
try std.testing.expect(!f.rawEqual(e));
|
||||
const g = Value { .String = "foo" };
|
||||
// foo != foo
|
||||
try std.testing.expect(e.rawEqual(g));
|
||||
// foo != foo
|
||||
try std.testing.expect(g.rawEqual(e));
|
||||
var table = Table { .entries = std.ArrayList(TableEntry).init(std.testing.allocator) };
|
||||
const h = Value { .Table = &table };
|
||||
var table2 = Table { .entries = std.ArrayList(TableEntry).init(std.testing.allocator) };
|
||||
const i = Value { .Table = &table2 };
|
||||
const j = Value { .Table = &table2 };
|
||||
|
||||
try std.testing.expect(h.rawEqual(h));
|
||||
try std.testing.expect(i.rawEqual(i));
|
||||
try std.testing.expect(!h.rawEqual(i));
|
||||
try std.testing.expect(!i.rawEqual(h));
|
||||
try std.testing.expect(i.rawEqual(j));
|
||||
try std.testing.expect(!h.rawEqual(j));
|
||||
}
|
||||
|
||||
test "Table get"
|
||||
{
|
||||
var a = Table { .entries = std.ArrayList(TableEntry).init(std.testing.allocator) };
|
||||
defer a.entries.deinit();
|
||||
try a.entries.append(TableEntry { .key = Value { .Bool = true }, .value = Value { .String = "foo" } });
|
||||
try std.testing.expectEqualStrings(a.get(Value { .Bool = true }).String, "foo");
|
||||
try std.testing.expectEqual(a.get(Value.Nil), Value.Nil);
|
||||
try std.testing.expectEqual(a.get(Value { .Numeral = Numeral { .Integer = 12 } }), Value.Nil);
|
||||
var c = a.get(Value { .Bool = true });
|
||||
c.String = "bar";
|
||||
try std.testing.expectEqual(a.get(Value { .Bool = true }).String, "foo");
|
||||
}
|
||||
|
||||
pub const CodeRegion = struct
|
||||
{
|
||||
start: ?CodeLocation,
|
||||
length: usize,
|
||||
};
|
||||
pub const CodeLocation = struct
|
||||
{
|
||||
line: usize,
|
||||
col: usize,
|
||||
};
|
||||
@ -1,8 +0,0 @@
|
||||
a, b = 12, test(32, 4)
|
||||
local t=(string.find(originalField.af,'m') and originalField.tableAction) or c.tableAction or originalField.tableAction or tableActionGeneric
|
||||
b = {["a"] = 23}
|
||||
for i=0, 10 do b[i] = 2^23 end
|
||||
print("asdf")
|
||||
function test(a, b)
|
||||
return 42 + a / b
|
||||
end
|
||||
Loading…
x
Reference in New Issue
Block a user