From 7889f4c27ac6c7061bcea06d3d0cca71c0079dee Mon Sep 17 00:00:00 2001
From: 0x4261756D <38735823+0x4261756D@users.noreply.github.com>
Date: Mon, 15 Jan 2024 21:28:40 +0100
Subject: [PATCH] Fix last token not getting tokenized

---
 src/tokenizer.zig | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/tokenizer.zig b/src/tokenizer.zig
index 92ed10d..f0863dd 100644
--- a/src/tokenizer.zig
+++ b/src/tokenizer.zig
@@ -1218,6 +1218,15 @@ pub fn tokenize(fileContent: []u8, allocator: std.mem.Allocator) ![]Token
 		}
 		index += 1;
 	}
+	if(longBracketLevel != 0)
+	{
+		return error.UnbalancedLongBracketLevel;
+	}
+	try tokenizeChar(&state, '\n', &lastIndex, &index, &tokenType, &tokenStr, &tokenNumeral, &tokens, &longBracketLevel, &region, allocator);
+	if(region.start != null and region.start.?.col == 0 and region.start.?.line == 0)
+	{
+		region.start = calculatePoint(fileContent, index);
+	}
 	return tokens.toOwnedSlice();
 }