Fix last token not getting tokenized

src/parser.zig

@@ -1496,7 +1496,7 @@ pub fn parse(tokens: []Token, allocator: *std.heap.ArenaAllocator) !ChunkNode
 	var i: usize = 0;
 	const maybeParsedChunk = parseChunk(tokens, &i, allocator) catch |err|
 	{
-		std.debug.print("{any}: data: {any}, type: {any}\n", .{tokens[i].region, tokens[i].tokenData, tokens[i].tokenType});
+		//std.debug.print("{any}: data: {any}, type: {any}\n", .{tokens[i].region, tokens[i].tokenData, tokens[i].tokenType});
 		return err;
 	};
 	return maybeParsedChunk;

src/tokenizer.zig

@@ -225,7 +225,7 @@ fn tokenizeChar(state: *TokenizerState, ch: u8, lastIndex: *?usize, index: *usiz
 					}
 					else if(std.ascii.isDigit(ch))
 					{
-						try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Numeral, TokenizerState.Name, tokenStr, ch, region);
+						try tokenizeTerminalIntNum(lastIndex, index.*, tokenType, state, TokenType.Numeral, TokenizerState.Number, tokenNumeral, ch, region);
 					}
 					else
 					{
@@ -1218,6 +1218,15 @@ pub fn tokenize(fileContent: []u8, allocator: std.mem.Allocator) ![]Token
 		}
 		index += 1;
 	}
+	if(longBracketLevel != 0)
+	{
+		return error.UnbalancedLongBracketLevel;
+	}
+	try tokenizeChar(&state, '\n', &lastIndex, &index, &tokenType, &tokenStr, &tokenNumeral, &tokens, &longBracketLevel, &region, allocator);
+	if(region.start != null and region.start.?.col == 0 and region.start.?.line == 0)
+	{
+		region.start = calculatePoint(fileContent, index);
+	}
 	return tokens.toOwnedSlice();
 }
 

test/test.lua

@@ -4,5 +4,5 @@ b = {["a"] = 23}
 for i=0, 10 do b[i] = 2^23 end
 print("asdf")
 function test(a, b)
-	return 12 + a / b
+	return 42 + a / b
 end