const types = @import("types.zig"); const std = @import("std"); const CodeRegion = @import("types.zig").CodeRegion; const CodeLocation = @import("types.zig").CodeLocation; pub const TokenType = enum { Name, And, Break, Do, Else, Elseif, End, False, For, Function, Goto, If, In, Local, Nil, Not, Or, Repeat, Return, Then, True, Until, While, Plus, Minus, Star, Slash, Percent, Caret, Hash, Ampersand, Tilde, Pipe, LtLt, GtGt, SlashSlash, EqualsEquals, TildeEquals, LtEquals, GtEquals, Lt, Gt, Equals, RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, ColonColon, Semicolon, Colon, Comma, Dot, DotDot, DotDotDot, Numeral, StringLiteral, }; const TokenData = union(enum) { string: []u8, numeral: types.Numeral, none, }; pub const Token = struct { tokenType: TokenType, tokenData: TokenData, region: CodeRegion, }; const TokenizerState = enum { Start, Quote, SingleQuote, Name, Number, Zero, A, B, D, E, F, G, I, L, N, O, R, T, U, W, Plus, Minus, Star, Slash, Percent, Caret, Hash, Ampersand, Tilde, Pipe, Lt, Gt, Equals, RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, Colon, Semicolon, Comma, Dot, An, Br, Do, El, En, Fa, Fo, Fu, Go, If, In, Lo, Ni, No, Or, Re, Th, Tr, Un, Wh, LtLt, GtGt, SlashSlash, EqualsEquals, TildeEquals, LtEquals, GtEquals, ColonColon, DotDot, SmallCommentStart, QuoteBackslash, SingleQuoteBackslash, String, HexNumberX, ExpNumber, And, Bre, Els, End, Fal, For, Fun, Got, Loc, Nil, Not, Rep, Ret, The, Tru, Unt, Whi, DotDotDot, HexNumber, QuoteBackslashZ, SingleQuoteBackslashZ, BigCommentLongBracketStart, SmallComment, Brea, Else, Fals, Func, Goto, Loca, Repe, Retu, Then, True, Unti, Whil, HexExpNumber, BigComment, BigCommentLongBracketEnd, Break, Elsei, False, Funct, Local, Repea, Retur, Until, While, Elseif, Functi, Repeat, Return, Functio, Function, }; fn tokenizeUpdateIndexAndState(lastIndex: *?usize, index: ?usize, state: *TokenizerState, newState: TokenizerState, region: *CodeRegion) void { lastIndex.* = index; state.* = newState; if(index == null) { region.*.start = null; region.*.length = 0; } else { if(region.*.start == null) { // TODO: There is no line/col info here and plumbing it to here would be pain. region.*.start = CodeLocation { .col = 0, .line = 0 }; } region.*.length += 1; } } fn tokenizeTerminalBase(lastIndex: *?usize, index: ?usize, tokenType: *?TokenType, state: *TokenizerState, newTokenType: ?TokenType, newState: TokenizerState, region: *CodeRegion) void { tokenizeUpdateIndexAndState(lastIndex, index, state, newState, region); tokenType.* = newTokenType; } fn tokenizeTerminalStr(lastIndex: *?usize, index: usize, tokenType: *?TokenType, state: *TokenizerState, newTokenType: ?TokenType, newState: TokenizerState, tokenStr: *std.ArrayList(u8), ch: u8, region: *CodeRegion) !void { tokenizeTerminalBase(lastIndex, index, tokenType, state, newTokenType, newState, region); try tokenStr.append(ch); } fn tokenizeTerminalIntNum(lastIndex: *?usize, index: usize, tokenType: *?TokenType, state: *TokenizerState, newTokenType: TokenType, newState: TokenizerState, tokenNumeral: *?types.Numeral, ch: u8, region: *CodeRegion) !void { tokenizeTerminalBase(lastIndex, index, tokenType, state, newTokenType, newState, region); if(!std.ascii.isDigit(ch)) { return error.NoDigit; } const digitValue = @as(i64, ch - '0'); if(tokenNumeral.* == null) { tokenNumeral.* = types.Numeral { .Integer = digitValue }; } else { switch(tokenNumeral.*.?) { .Integer => |*n| n.* = n.* * 10 + digitValue, .Float => return error.ExpectedIntGotFloat } } } fn tokenizeTerminalNoToken(lastIndex: *?usize, index: usize, state: *TokenizerState, newState: TokenizerState, tokenStr: *std.ArrayList(u8), ch: u8, region: *CodeRegion) !void { tokenizeUpdateIndexAndState(lastIndex, index, state, newState, region); try tokenStr.*.append(ch); } fn tokenizeBacktrack(lastIndex: *?usize, index: *usize, tokens: *std.ArrayList(Token), tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, state: *TokenizerState, allocator: std.mem.Allocator, region: *CodeRegion) !void { try tokenizeBacktrackCustomToken(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, tokenType.*.?, allocator, region); } fn tokenizeBacktrackCustomToken(lastIndex: *?usize, index: *usize, tokens: *std.ArrayList(Token), tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, state: *TokenizerState, newTokenType: TokenType, allocator: std.mem.Allocator, region: *CodeRegion) !void { if(lastIndex.* == null or tokenType.* == null) { return error.LexError; } if(newTokenType == TokenType.StringLiteral or newTokenType == TokenType.Name) { const content = try allocator.alloc(u8, tokenStr.*.items.len); @memcpy(content, tokenStr.*.items); try tokens.append(Token { .tokenType = newTokenType, .tokenData = TokenData { .string = content }, .region = region.* }); } else { try tokens.append(Token { .tokenType = newTokenType, .region = region.*, .tokenData = if(tokenType.*.? == TokenType.Numeral) TokenData { .numeral = tokenNumeral.*.? } else TokenData.none }); } tokenNumeral.* = null; index.* = lastIndex.*.?; tokenStr.*.clearAndFree(); // region is reset in tokenizeTerminalBase since null is passed as index tokenizeTerminalBase(lastIndex, null, tokenType, state, null, TokenizerState.Start, region); } fn tokenizeAlphanumericNonstart(lastIndex: *?usize, index: *usize, tokens: *std.ArrayList(Token), tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, state: *TokenizerState, ch: u8, newTokenType: TokenType, allocator: std.mem.Allocator, region: *CodeRegion) !void { if(std.ascii.isAlphanumeric(ch) or ch == '_') { try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.Name, tokenStr, ch, region); } else { try tokenizeBacktrackCustomToken(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, newTokenType, allocator, region); } } fn tokenizeChar(state: *TokenizerState, ch: u8, lastIndex: *?usize, index: *usize, tokenType: *?TokenType, tokenStr: *std.ArrayList(u8), tokenNumeral: *?types.Numeral, tokens: *std.ArrayList(Token), longBracketLevel: *u32, region: *CodeRegion, allocator: std.mem.Allocator) !void { switch(state.*) { TokenizerState.Start => { switch(ch) { '-' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Minus, TokenizerState.Minus, region), ',' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Comma, TokenizerState.Comma, region), '=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Equals, TokenizerState.Equals, region), '(' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.RoundOpen, TokenizerState.RoundOpen, region), ')' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.RoundClosed, TokenizerState.RoundClosed, region), '.' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Dot, TokenizerState.Dot, region), ':' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Colon, TokenizerState.Colon, region), '{' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.CurlyOpen, TokenizerState.CurlyOpen, region), '}' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.CurlyClosed, TokenizerState.CurlyClosed, region), '[' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.SquareOpen, TokenizerState.SquareOpen, region), ']' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.SquareClosed, TokenizerState.SquareClosed, region), '+' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Plus, TokenizerState.Plus, region), '~' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Tilde, TokenizerState.Tilde, region), '>' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Gt, TokenizerState.Gt, region), '<' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Lt, TokenizerState.Lt, region), '#' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Hash, TokenizerState.Hash, region), '|' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Pipe, TokenizerState.Pipe, region), '&' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Ampersand, TokenizerState.Ampersand, region), '%' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Percent, TokenizerState.Percent, region), '*' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Star, TokenizerState.Star, region), '/' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Slash, TokenizerState.Slash, region), ';' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Semicolon, TokenizerState.Semicolon, region), '^' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.Caret, TokenizerState.Caret, region), 'a' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.A, tokenStr, ch, region), 'b' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.B, tokenStr, ch, region), 'd' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.D, tokenStr, ch, region), 'e' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.E, tokenStr, ch, region), 'f' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.F, tokenStr, ch, region), 'i' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.I, tokenStr, ch, region), 'g' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.G, tokenStr, ch, region), 'l' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.L, tokenStr, ch, region), 'n' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.N, tokenStr, ch, region), 'o' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.O, tokenStr, ch, region), 'r' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.R, tokenStr, ch, region), 't' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.T, tokenStr, ch, region), 'u' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.U, tokenStr, ch, region), 'w' => try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.W, tokenStr, ch, region), '0' => try tokenizeTerminalIntNum(lastIndex, index.*, tokenType, state, TokenType.Numeral, TokenizerState.Zero, tokenNumeral, ch, region), '"' => { tokenType.* = null; state.* = TokenizerState.Quote; }, '\'' => { tokenType.* = null; state.* = TokenizerState.SingleQuote; }, else => { if(std.ascii.isWhitespace(ch)) { } else if(std.ascii.isAlphabetic(ch) or ch == '_') { try tokenizeTerminalStr(lastIndex, index.*, tokenType, state, TokenType.Name, TokenizerState.Name, tokenStr, ch, region); } else if(std.ascii.isDigit(ch)) { try tokenizeTerminalIntNum(lastIndex, index.*, tokenType, state, TokenType.Numeral, TokenizerState.Number, tokenNumeral, ch, region); } else { std.debug.print("{}: {c}\n", .{state.*, ch}); return error.NotImplemented; } } } }, TokenizerState.Quote => { switch(ch) { '\\' => state.* = TokenizerState.QuoteBackslash, '"' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region), else => try tokenStr.*.append(ch), } }, TokenizerState.QuoteBackslash => { switch(ch) { 'a' => { try tokenStr.append('\u{0007}'); state.* = TokenizerState.Quote; }, 'b' => { try tokenStr.append('\u{0008}'); state.* = TokenizerState.Quote; }, 't' => { try tokenStr.append('\t'); state.* = TokenizerState.Quote; }, 'n' | '\n' => { try tokenStr.append('\n'); state.* = TokenizerState.Quote; }, 'v' => { try tokenStr.append('\u{000b}'); state.* = TokenizerState.Quote; }, 'f' => { try tokenStr.append('\u{000c}'); state.* = TokenizerState.Quote; }, 'r' => { try tokenStr.append('\r'); state.* = TokenizerState.Quote; }, '\\' => { try tokenStr.append('\\'); state.* = TokenizerState.Quote; }, '"' => { try tokenStr.append('\"'); state.* = TokenizerState.Quote; }, '\'' => { try tokenStr.append('\''); state.* = TokenizerState.Quote; }, 'z' => { state.* = TokenizerState.QuoteBackslashZ; }, else => return error.UnknownEscapeSequence, } }, TokenizerState.QuoteBackslashZ => { switch(ch) { '\\' => state.* = TokenizerState.QuoteBackslash, '"' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region), else => { if(!std.ascii.isWhitespace(ch)) { try tokenStr.append(ch); state.* = TokenizerState.Quote; } else { // Noop, https://www.lua.org/manual/5.4/manual.html#3.1: // "The escape sequence '\z' skips the following span of whitespace characters, including line breaks;" } } } }, TokenizerState.SingleQuote => { switch(ch) { '\\' => state.* = TokenizerState.SingleQuoteBackslash, '\'' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region), else => try tokenStr.append(ch), } }, TokenizerState.SingleQuoteBackslash => { switch(ch) { 'a' => { try tokenStr.append('\u{0007}'); state.* = TokenizerState.SingleQuote; }, 'b' => { try tokenStr.append('\u{0008}'); state.* = TokenizerState.SingleQuote; }, 't' => { try tokenStr.append('\t'); state.* = TokenizerState.SingleQuote; }, 'n' | '\n' => { try tokenStr.append('\n'); state.* = TokenizerState.SingleQuote; }, 'v' => { try tokenStr.append('\u{000b}'); state.* = TokenizerState.SingleQuote; }, 'f' => { try tokenStr.append('\u{000c}'); state.* = TokenizerState.SingleQuote; }, 'r' => { try tokenStr.append('\r'); state.* = TokenizerState.SingleQuote; }, '\\' => { try tokenStr.append('\\'); state.* = TokenizerState.SingleQuote; }, '"' => { try tokenStr.append('\"'); state.* = TokenizerState.SingleQuote; }, '\'' => { try tokenStr.append('\''); state.* = TokenizerState.SingleQuote; }, 'z' => { state.* = TokenizerState.SingleQuoteBackslashZ; }, else => return error.UnknownEscapeSequence, } }, TokenizerState.SingleQuoteBackslashZ => { switch(ch) { '\\' => state.* = TokenizerState.SingleQuoteBackslash, '\'' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.StringLiteral, TokenizerState.String, region), else => { if(!std.ascii.isWhitespace(ch)) { try tokenStr.append(ch); state.* = TokenizerState.SingleQuote; } else { // Noop, https://www.lua.org/manual/5.4/manual.html#3.1: // "The escape sequence '\z' skips the following span of whitespace characters, including line breaks;" } } } }, TokenizerState.String => try tokenizeBacktrackCustomToken(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, TokenType.StringLiteral, allocator, region), TokenizerState.Name => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), TokenizerState.Zero => { switch(ch) { 'x' => { try tokenStr.*.append(ch); tokenType.* = null; state.* = TokenizerState.HexNumberX; }, '.' => return error.NotImplemented, else => { if(std.ascii.isDigit(ch)) { const digitValue = @as(i64, ch - '0'); lastIndex.* = index.*; if(tokenNumeral.* == null) { tokenNumeral.* = types.Numeral { .Integer = digitValue }; tokenType.* = TokenType.Numeral; } else { tokenNumeral.*.?.Integer = tokenNumeral.*.?.Integer * 10 + digitValue; } } else { try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region); } } } }, TokenizerState.HexNumberX => { if(std.ascii.isHex(ch)) { lastIndex.* = index.*; tokenType.* = TokenType.Numeral; if(std.ascii.isDigit(ch)) { tokenNumeral.* = types.Numeral { .Integer = @as(i64, ch - '0') }; } else { tokenNumeral.* = types.Numeral { .Integer = @as(i64, std.ascii.toLower(ch) - 'a') }; } } else { try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region); } }, TokenizerState.HexNumber => { switch(ch) { 'p' => { try tokenStr.*.append(ch); tokenType.* = null; state.* = TokenizerState.HexExpNumber; }, else => { if(std.ascii.isHex(ch)) { lastIndex.* = index.*; tokenType.* = TokenType.Numeral; if(std.ascii.isDigit(ch)) { tokenNumeral.* = types.Numeral { .Integer = @as(i64, ch - '0') }; } else { tokenNumeral.* = types.Numeral { .Integer = @as(i64, std.ascii.toLower(ch) - 'a') }; } } else { try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region); } } } }, TokenizerState.Number => { switch(ch) { 'e' => { try tokenStr.*.append(ch); tokenType.* = null; state.* = TokenizerState.ExpNumber; }, '.' => return error.NotImplemented, else => { if(std.ascii.isDigit(ch)) { const digitValue = @as(i64, ch - '0'); lastIndex.* = index.*; if(tokenNumeral.* == null) { tokenNumeral.* = types.Numeral { .Integer = digitValue }; tokenType.* = TokenType.Numeral; } else { tokenNumeral.*.?.Integer = tokenNumeral.*.?.Integer * 10 + digitValue; } } else { try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region); } } } }, TokenizerState.Comma, TokenizerState.RoundOpen, TokenizerState.RoundClosed, TokenizerState.CurlyOpen, TokenizerState.CurlyClosed, TokenizerState.Plus, TokenizerState.TildeEquals, TokenizerState.EqualsEquals, TokenizerState.Hash, TokenizerState.GtEquals, TokenizerState.LtEquals, TokenizerState.SquareOpen, TokenizerState.SquareClosed, TokenizerState.Pipe, TokenizerState.Ampersand, TokenizerState.Percent, TokenizerState.Star, TokenizerState.Semicolon, TokenizerState.Caret, TokenizerState.DotDotDot, TokenizerState.GtGt, TokenizerState.LtLt, TokenizerState.SlashSlash => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), TokenizerState.Tilde => { switch(ch) { '=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.TildeEquals, TokenizerState.TildeEquals, region), else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), } }, TokenizerState.Gt => { switch (ch) { '>' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.GtGt, TokenizerState.GtGt, region), '=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.GtEquals, TokenizerState.GtEquals, region), else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), } }, TokenizerState.Lt => { switch(ch) { '<' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.LtLt, TokenizerState.LtLt, region), '=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.LtEquals, TokenizerState.LtEquals, region), else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), } }, TokenizerState.Slash => { switch(ch) { '/' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.SlashSlash, TokenizerState.SlashSlash, region), else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), } }, TokenizerState.Dot => { switch(ch) { '.' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.DotDot, TokenizerState.DotDot, region), else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), } }, TokenizerState.DotDot => { switch(ch) { '.' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.DotDotDot, TokenizerState.DotDotDot, region), else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), } }, TokenizerState.Colon => { switch(ch) { ':' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.ColonColon, TokenizerState.ColonColon, region), else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), } }, TokenizerState.Equals => { switch(ch) { '=' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, TokenType.EqualsEquals, TokenizerState.EqualsEquals, region), else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), } }, TokenizerState.Minus => { switch(ch) { '-' => tokenizeTerminalBase(lastIndex, index.*, tokenType, state, null, TokenizerState.SmallCommentStart, region), else => try tokenizeBacktrack(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, allocator, region), } }, TokenizerState.SmallCommentStart => { switch(ch) { '[' => { tokenType.* = null; state.* = TokenizerState.BigCommentLongBracketStart; }, '\n' => { state.* = TokenizerState.Start; lastIndex.* = null; }, else => { state.* = TokenizerState.SmallComment; }, } }, TokenizerState.SmallComment => { switch(ch) { '\n' => { state.* = TokenizerState.Start; lastIndex.* = null; }, else => { } } }, TokenizerState.BigCommentLongBracketStart => { switch(ch) { '=' => { longBracketLevel.* += 1; }, '[' => { state.* = TokenizerState.BigComment; }, else => return error.LongBracketMalformedStartBigComment, } }, TokenizerState.BigComment => { switch(ch) { ']' => { state.* = TokenizerState.BigCommentLongBracketEnd; }, else => { }, } }, TokenizerState.BigCommentLongBracketEnd => { switch(ch) { '=' => { if(longBracketLevel.* == 0) { return error.LongBracketLevelTooBigEndBigComment; } longBracketLevel.* -= 1; }, ']' => { if(longBracketLevel.* != 0) { return error.LongBracketLevelTooSmallEndBigComment; } state.* = TokenizerState.Start; }, else => return error.LongBracketMalformedSmallEndBigComment, } }, TokenizerState.A => { switch(ch) { 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.An, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.An => { switch(ch) { 'd' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.And, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.And => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.And, allocator, region), TokenizerState.W => { switch(ch) { 'h' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Wh, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Wh => { switch(ch) { 'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Whi, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Whi => { switch(ch) { 'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Whil, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Whil => { switch(ch) { 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.While, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.While => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.While, allocator, region), TokenizerState.B => { switch(ch) { 'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Br, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Br => { switch(ch) { 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Bre, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Bre => { switch(ch) { 'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Brea, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Brea => { switch(ch) { 'k' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Break, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Break => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Break, allocator, region), TokenizerState.G => { switch(ch) { 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Go, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Go => { switch(ch) { 't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Got, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Got => { switch(ch) { 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Goto, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Goto => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Goto, allocator, region), TokenizerState.R => { switch(ch) { 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Re, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Re => { switch(ch) { 't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Ret, tokenStr, ch, region), 'p' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Rep, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Ret => { switch(ch) { 'u' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Retu, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Retu => { switch(ch) { 'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Retur, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Retur => { switch(ch) { 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Return, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Return => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Return, allocator, region), TokenizerState.Rep => { switch(ch) { 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Repe, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Repe => { switch(ch) { 'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Repea, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Repea => { switch(ch) { 't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Repeat, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Repeat => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Repeat, allocator, region), TokenizerState.N => { switch(ch) { 'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Ni, tokenStr, ch, region), 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.No, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.No => { switch(ch) { 't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Not, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Not => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Not, allocator, region), TokenizerState.Ni => { switch(ch) { 'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Nil, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Nil => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Nil, allocator, region), TokenizerState.T => { switch(ch) { 'h' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Th, tokenStr, ch, region), 'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Tr, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Th => { switch(ch) { 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.The, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.The => { switch(ch) { 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Then, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Then => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Then, allocator, region), TokenizerState.Tr => { switch(ch) { 'u' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Tru, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Tru => { switch(ch) { 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.True, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.True => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.True, allocator, region), TokenizerState.E => { switch(ch) { 'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.El, tokenStr, ch, region), 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.En, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.En => { switch(ch) { 'd' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.End, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.End => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.End, allocator, region), TokenizerState.El => { switch(ch) { 's' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Els, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Els => { switch(ch) { 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Else, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Else => { switch(ch) { 'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Elsei, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Else, allocator, region), } }, TokenizerState.Elsei => { switch(ch) { 'f' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Elseif, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Elseif => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Elseif, allocator, region), TokenizerState.O => { switch(ch) { 'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Or, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Or => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Or, allocator, region), TokenizerState.D => { switch(ch) { 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Do, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Do => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Do, allocator, region), TokenizerState.I => { switch(ch) { 'f' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.If, tokenStr, ch, region), 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.In, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.In => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.In, allocator, region), TokenizerState.If => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.If, allocator, region), TokenizerState.F => { switch(ch) { 'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fa, tokenStr, ch, region), 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fo, tokenStr, ch, region), 'u' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fu, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Fu => { switch(ch) { 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fun, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Fun => { switch(ch) { 'c' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Func, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Func => { switch(ch) { 't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Funct, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Funct => { switch(ch) { 'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Functi, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Functi => { switch(ch) { 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Functio, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Functio => { switch(ch) { 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Function, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Function => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Function, allocator, region), TokenizerState.Fa => { switch(ch) { 'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fal, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Fal => { switch(ch) { 's' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Fals, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Fals => { switch(ch) { 'e' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.False, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.False => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.False, allocator, region), TokenizerState.Fo => { switch(ch) { 'r' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.For, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.For => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.For, allocator, region), TokenizerState.L => { switch(ch) { 'o' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Lo, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Lo => { switch(ch) { 'c' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Loc, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Loc => { switch(ch) { 'a' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Loca, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Loca => { switch(ch) { 'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Local, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Local => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Local, allocator, region), TokenizerState.U => { switch(ch) { 'n' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Un, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Un => { switch(ch) { 't' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Unt, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Unt => { switch(ch) { 'i' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Unti, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Unti => { switch(ch) { 'l' => try tokenizeTerminalNoToken(lastIndex, index.*, state, TokenizerState.Until, tokenStr, ch, region), else => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Name, allocator, region), } }, TokenizerState.Until => try tokenizeAlphanumericNonstart(lastIndex, index, tokens, tokenType, tokenStr, tokenNumeral, state, ch, TokenType.Until, allocator, region), else => { std.debug.print("{}\n", . {state.*}); return error.NotImplemented; } } } pub fn tokenize(fileContent: []u8, allocator: std.mem.Allocator) ![]Token { var tokens = std.ArrayList(Token).init(allocator); var state: TokenizerState = TokenizerState.Start; var lastIndex: ?usize = null; var index: usize = 0; var tokenType: ?TokenType = null; var tokenStr = std.ArrayList(u8).init(allocator); defer tokenStr.deinit(); var tokenNumeral: ?types.Numeral = null; var longBracketLevel: u32 = 0; var region = CodeRegion { .start = null, .length = 0 }; while(index < fileContent.len) { const ch = fileContent[index]; try tokenizeChar(&state, ch, &lastIndex, &index, &tokenType, &tokenStr, &tokenNumeral, &tokens, &longBracketLevel, ®ion, allocator); if(region.start != null and region.start.?.col == 0 and region.start.?.line == 0) { region.start = calculatePoint(fileContent, index); } index += 1; } if(longBracketLevel != 0) { return error.UnbalancedLongBracketLevel; } try tokenizeChar(&state, '\n', &lastIndex, &index, &tokenType, &tokenStr, &tokenNumeral, &tokens, &longBracketLevel, ®ion, allocator); if(region.start != null and region.start.?.col == 0 and region.start.?.line == 0) { region.start = calculatePoint(fileContent, index); } return tokens.toOwnedSlice(); } fn calculatePoint(fileContent: []u8, index: usize) CodeLocation { var ret = CodeLocation { .col = 1, .line = 1 }; for(0..index) |i| { ret.col += 1; if(fileContent[i] == '\n') { ret.line += 1; ret.col = 1; } } return ret; }