Fix last token not getting tokenized

Since hashtables are a hassle with strings and complex datastructures
like the table they are an arraylist for now.

Also implement rawEquals for numerals and values to make table.get work

.gitignore

@@ -1,2 +1,4 @@
 zig-out/
 zig-cache/
+.vscode/
+target/

src/parser.zig

@@ -1496,7 +1496,7 @@ pub fn parse(tokens: []Token, allocator: *std.heap.ArenaAllocator) !ChunkNode
 	var i: usize = 0;
 	const maybeParsedChunk = parseChunk(tokens, &i, allocator) catch |err|
 	{
-		std.debug.print("{any}: data: {any}, type: {any}\n", .{tokens[i].region, tokens[i].tokenData, tokens[i].tokenType});
+		//std.debug.print("{any}: data: {any}, type: {any}\n", .{tokens[i].region, tokens[i].tokenData, tokens[i].tokenType});
 		return err;
 	};
 	return maybeParsedChunk;

src/tokenizer.zig

@@ -225,7 +225,7 @@ fn tokenizeChar(state: *TokenizerState, ch: u8, lastIndex: *?usize, index: *usiz
 					}
 					else if(std.ascii.isDigit(ch))
 					{
-						try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Numeral, TokenizerState.Name, tokenStr, ch, region);
+						try tokenizeTerminalIntNum(lastIndex, index.*, tokenType, state, TokenType.Numeral, TokenizerState.Number, tokenNumeral, ch, region);
 					}
 					else
 					{
@@ -1218,6 +1218,15 @@ pub fn tokenize(fileContent: []u8, allocator: std.mem.Allocator) ![]Token
 		}
 		index += 1;
 	}
+	if(longBracketLevel != 0)
+	{
+		return error.UnbalancedLongBracketLevel;
+	}
+	try tokenizeChar(&state, '\n', &lastIndex, &index, &tokenType, &tokenStr, &tokenNumeral, &tokens, &longBracketLevel, &region, allocator);
+	if(region.start != null and region.start.?.col == 0 and region.start.?.line == 0)
+	{
+		region.start = calculatePoint(fileContent, index);
+	}
 	return tokens.toOwnedSlice();
 }
 

src/treewalker.zig

@@ -4,7 +4,7 @@ const types = @import("types.zig");
 
 pub fn interpret(root: parser.ChunkNode, allocator: std.mem.Allocator) !void
 {
-	var _ENV = types.Table { .items = std.HashMap(types.Value, types.Value, std.hash_map.AutoContext(types.Value), std.hash_map.default_max_load_percentage).init(allocator) };
+	var _ENV = types.Table { .entries= std.ArrayList(types.TableEntry).init(allocator) };
 	try walkChunk(root, &_ENV, allocator);
 }
 

src/types.zig

@@ -1,34 +1,187 @@
 const std = @import("std");
 
-pub const Numeral = union(enum)
+pub const NumeralTag = enum
+{
+	Integer,
+	Float,
+};
+
+pub fn isFloatExactInt(value: f64) bool
+{
+	return value == 0 or (std.math.isNormal(value) and @floor(value) == value);
+}
+
+pub const Numeral = union(NumeralTag)
 {
 	Integer: i64,
 	Float: f64,
+
+	pub fn rawEqual(self: Numeral, other: Numeral) bool
+	{
+		if(@as(NumeralTag, self) == @as(NumeralTag, other))
+		{
+			switch(self)
+			{
+				.Float => |value| return value == other.Float,
+				.Integer =>  |value| return value == other.Integer,
+			}
+		}
+		// Other is the respective other type
+		switch(self)
+		{
+			.Float => |value|
+			{
+				return isFloatExactInt(value) and @as(u64, @intFromFloat(value)) == other.Integer;
+			},
+			.Integer => |value|
+			{
+				return isFloatExactInt(other.Float) and @as(u64, @intFromFloat(other.Float)) == value;
+			}
+		}
+	}
 };
+
+test "float int equality"
+{
+	const a = Numeral { .Float = 12.0 };
+	const b = Numeral { .Integer = 12 };
+	try std.testing.expect(a.rawEqual(b));
+	try std.testing.expect(b.rawEqual(a));
+	try std.testing.expect(a.rawEqual(a));
+	try std.testing.expect(b.rawEqual(b));
+	const c = Numeral { .Float = (0.2 + 0.1) * 10.0 };
+	const d = Numeral { .Integer = 3 };
+	try std.testing.expect(c.rawEqual(d));
+	try std.testing.expect(d.rawEqual(c));
+	try std.testing.expect(c.rawEqual(c));
+	try std.testing.expect(d.rawEqual(d));
+	const e = Numeral { .Float = 3.2 };
+	try std.testing.expect(!a.rawEqual(e));
+	try std.testing.expect(!b.rawEqual(e));
+	try std.testing.expect(!c.rawEqual(e));
+	try std.testing.expect(!d.rawEqual(e));
+	try std.testing.expect(!e.rawEqual(a));
+	try std.testing.expect(!e.rawEqual(b));
+	try std.testing.expect(!e.rawEqual(c));
+	try std.testing.expect(!e.rawEqual(d));
+}
+
+pub const TableEntry = struct
+{
+	key: Value,
+	value: Value,
+};
+
 pub const Table = struct
 {
-	items: std.HashMap(Value, Value, std.hash_map.AutoContext(Value), std.hash_map.default_max_load_percentage),
+	entries: std.ArrayList(TableEntry),
 
-	pub fn insert(self: *const Table, key: Value, value: Value) !bool
+	pub fn get(self: Table, key: Value) Value
 	{
-		self.items.put(key, value);
-	}
-
-	pub fn get(self: *const Table, key: Value) !Value
-	{
-		const value = self.items.get(key);
-		return if(value == null) Value.Nil else value.?;
+		if(@as(ValueTag, key) == ValueTag.Nil)
+		{
+			return Value.Nil;
+		}
+		for (self.entries.items) |entry|
+		{
+			if(entry.key.rawEqual(key))
+			{
+				return entry.value;
+			}
+		}
+		return Value.Nil;
 	}
 };
-pub const Value = union(enum)
+
+pub const ValueTag = enum
+{
+	Nil,
+	Bool,
+	Numeral,
+	String,
+	Table,
+};
+pub const Value = union(ValueTag)
 {
 	Nil,
 	Bool: bool,
 	Numeral: Numeral,
 	String: []const u8,
 	Table: *Table,
+
+	pub fn rawEqual(self: Value, other: Value) bool
+	{
+		if(@as(ValueTag, self) != @as(ValueTag, other))
+		{
+			return false;
+		}
+		switch(self)
+		{
+			.Nil => return true,
+			.Bool => |value| return value == other.Bool,
+			.Numeral => |value| return value.rawEqual(other.Numeral),
+			.String => |value| return std.mem.eql(u8, value, other.String),
+			.Table => |value| return value == other.Table,
+		}
+	}
 };
 
+test "Value equalities"
+{
+	const a = Value { .Bool = true };
+	const b = Value { .Numeral = Numeral { .Integer = 1 } };
+	// true != 1
+	try std.testing.expect(!a.rawEqual(b));
+	// 1 != true
+	try std.testing.expect(!b.rawEqual(a));
+	const c = Value { .Bool = false };
+	// true != false
+	try std.testing.expect(!a.rawEqual(c));
+	// false!= true
+	try std.testing.expect(!c.rawEqual(a));
+	const d = Value { .Bool = true };
+	// true == true
+	try std.testing.expect(a.rawEqual(d));
+	// true == true
+	try std.testing.expect(d.rawEqual(a));
+	const e = Value { .String = "foo" };
+	const f = Value { .String = "bar" };
+	// foo != bar
+	try std.testing.expect(!e.rawEqual(f));
+	// bar != foo
+	try std.testing.expect(!f.rawEqual(e));
+	const g = Value { .String = "foo" };
+	// foo != foo
+	try std.testing.expect(e.rawEqual(g));
+	// foo != foo
+	try std.testing.expect(g.rawEqual(e));
+	var table = Table { .entries = std.ArrayList(TableEntry).init(std.testing.allocator) };
+	const h = Value { .Table = &table };
+	var table2 = Table { .entries = std.ArrayList(TableEntry).init(std.testing.allocator) };
+	const i = Value { .Table = &table2 };
+	const j = Value { .Table = &table2 };
+
+	try std.testing.expect(h.rawEqual(h));
+	try std.testing.expect(i.rawEqual(i));
+	try std.testing.expect(!h.rawEqual(i));
+	try std.testing.expect(!i.rawEqual(h));
+	try std.testing.expect(i.rawEqual(j));
+	try std.testing.expect(!h.rawEqual(j));
+}
+
+test "Table get"
+{
+	var a = Table { .entries = std.ArrayList(TableEntry).init(std.testing.allocator) };
+	defer a.entries.deinit();
+	try a.entries.append(TableEntry { .key = Value { .Bool = true }, .value = Value { .String = "foo" } });
+	try std.testing.expectEqualStrings(a.get(Value { .Bool = true }).String, "foo");
+	try std.testing.expectEqual(a.get(Value.Nil), Value.Nil);
+	try std.testing.expectEqual(a.get(Value { .Numeral = Numeral { .Integer = 12 } }), Value.Nil);
+	var c = a.get(Value { .Bool = true });
+	c.String = "bar";
+	try std.testing.expectEqual(a.get(Value { .Bool = true }).String, "foo");
+}
+
 pub const CodeRegion = struct
 {
 	start: ?CodeLocation,

test/test.lua

@@ -4,5 +4,5 @@ b = {["a"] = 23}
 for i=0, 10 do b[i] = 2^23 end
 print("asdf")
 function test(a, b)
-	return 12 + a / b
+	return 42 + a / b
 end