Compare commits

..

1 Commits

Author SHA1 Message Date
7889f4c27a Fix last token not getting tokenized 2024-01-15 21:28:40 +01:00

View File

@ -1218,6 +1218,15 @@ pub fn tokenize(fileContent: []u8, allocator: std.mem.Allocator) ![]Token
}
index += 1;
}
if(longBracketLevel != 0)
{
return error.UnbalancedLongBracketLevel;
}
try tokenizeChar(&state, '\n', &lastIndex, &index, &tokenType, &tokenStr, &tokenNumeral, &tokens, &longBracketLevel, &region, allocator);
if(region.start != null and region.start.?.col == 0 and region.start.?.line == 0)
{
region.start = calculatePoint(fileContent, index);
}
return tokens.toOwnedSlice();
}