0x4261756D/luaaaaah
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix last token not getting tokenized

Browse Source
This commit is contained in:
0x4261756D 2024-01-15 21:28:40 +01:00
parent 23269baa0b
commit 7889f4c27a
1 changed files with 9 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/tokenizer.zig
View File

@ -1218,6 +1218,15 @@ pub fn tokenize(fileContent: []u8, allocator: std.mem.Allocator) ![]Token
} }
index += 1; index += 1;
} }
if(longBracketLevel != 0)
{
return error.UnbalancedLongBracketLevel;
}
try tokenizeChar(&state, '\n', &lastIndex, &index, &tokenType, &tokenStr, &tokenNumeral, &tokens, &longBracketLevel, &region, allocator);
if(region.start != null and region.start.?.col == 0 and region.start.?.line == 0)
{
region.start = calculatePoint(fileContent, index);
}
return tokens.toOwnedSlice(); return tokens.toOwnedSlice();
} }