Add terribly slow cyk parser
This commit is contained in:
0x4261756D
parent
5a2171b023
commit
e53734dcca
698
src/grammar.rs
698
src/grammar.rs
@ -6,7 +6,7 @@ pub enum Rule
|
|||||||
NonTerminal(u8, u8, u8)
|
NonTerminal(u8, u8, u8)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub const NONTERMS: [&str; 115] =
|
pub const NONTERMINAL_NAMES: [&str; 115] =
|
||||||
[
|
[
|
||||||
"stat__15",
|
"stat__15",
|
||||||
"funcbody__50",
|
"funcbody__50",
|
||||||
@ -125,350 +125,356 @@ pub const NONTERMS: [&str; 115] =
|
|||||||
"retstat__47",
|
"retstat__47",
|
||||||
];
|
];
|
||||||
|
|
||||||
pub const GRAMMAR: [Rule; 344] =
|
pub const TERMINAL_RULES: [(u8, Token); 125] =
|
||||||
[
|
[
|
||||||
Rule::Terminal(57, Token::RoundOpen),
|
(57, Token::RoundOpen),
|
||||||
Rule::Terminal(34, Token::RoundClosed),
|
(34, Token::RoundClosed),
|
||||||
Rule::Terminal(8, Token::Comma),
|
(8, Token::Comma),
|
||||||
Rule::Terminal(66, Token::DotDotDot),
|
(66, Token::DotDotDot),
|
||||||
Rule::Terminal(69, Token::Dot),
|
(69, Token::Dot),
|
||||||
Rule::Terminal(39, Token::ColonColon),
|
(39, Token::ColonColon),
|
||||||
Rule::Terminal(99, Token::Colon),
|
(99, Token::Colon),
|
||||||
Rule::Terminal(15, Token::Semicolon),
|
(15, Token::Semicolon),
|
||||||
Rule::Terminal(20, Token::Lt),
|
(20, Token::Lt),
|
||||||
Rule::Terminal(98, Token::Equals),
|
(98, Token::Equals),
|
||||||
Rule::Terminal(6, Token::Gt),
|
(6, Token::Gt),
|
||||||
Rule::Terminal(81, Token::Name(String::new())),
|
(81, Token::Name(String::new())),
|
||||||
Rule::NonTerminal(80, 22, 29),
|
(80, Token::Return),
|
||||||
Rule::NonTerminal(80, 68, 114),
|
(80, Token::Semicolon),
|
||||||
Rule::Terminal(80, Token::Return),
|
(80, Token::Break),
|
||||||
Rule::NonTerminal(80, 40, 22),
|
(43, Token::SquareOpen),
|
||||||
Rule::Terminal(80, Token::Semicolon),
|
(42, Token::SquareClosed),
|
||||||
Rule::Terminal(80, Token::Break),
|
(87, Token::StringLiteral(String::new())),
|
||||||
Rule::NonTerminal(80, 82, 81),
|
(105, Token::RoundClosed),
|
||||||
Rule::NonTerminal(80, 79, 44),
|
(72, Token::Name(String::new())),
|
||||||
Rule::NonTerminal(80, 47, 25),
|
(65, Token::Plus),
|
||||||
Rule::NonTerminal(80, 63, 46),
|
(65, Token::Minus),
|
||||||
Rule::NonTerminal(80, 104, 59),
|
(65, Token::Star),
|
||||||
Rule::NonTerminal(80, 85, 30),
|
(65, Token::Slash),
|
||||||
Rule::NonTerminal(80, 75, 89),
|
(65, Token::SlashSlash),
|
||||||
Rule::NonTerminal(80, 33, 0),
|
(65, Token::Caret),
|
||||||
Rule::NonTerminal(80, 58, 93),
|
(65, Token::Percent),
|
||||||
Rule::NonTerminal(80, 85, 31),
|
(65, Token::Ampersand),
|
||||||
Rule::NonTerminal(80, 33, 11),
|
(65, Token::Pipe),
|
||||||
Rule::NonTerminal(80, 71, 87),
|
(65, Token::GtGt),
|
||||||
Rule::NonTerminal(80, 71, 73),
|
(65, Token::LtLt),
|
||||||
Rule::NonTerminal(80, 39, 51),
|
(65, Token::DotDot),
|
||||||
Rule::Terminal(43, Token::SquareOpen),
|
(65, Token::Lt),
|
||||||
Rule::Terminal(42, Token::SquareClosed),
|
(65, Token::LtEquals),
|
||||||
Rule::Terminal(87, Token::StringLiteral(String::new())),
|
(65, Token::Gt),
|
||||||
Rule::NonTerminal(87, 57, 105),
|
(65, Token::GtEquals),
|
||||||
Rule::NonTerminal(87, 23, 78),
|
(65, Token::EqualsEquals),
|
||||||
Rule::NonTerminal(105, 21, 34),
|
(65, Token::TildeEquals),
|
||||||
Rule::Terminal(105, Token::RoundClosed),
|
(65, Token::And),
|
||||||
Rule::NonTerminal(44, 98, 21),
|
(65, Token::Or),
|
||||||
Rule::NonTerminal(72, 81, 95),
|
(7, Token::Return),
|
||||||
Rule::Terminal(72, Token::Name(String::new())),
|
(7, Token::Semicolon),
|
||||||
Rule::NonTerminal(95, 103, 27),
|
(7, Token::Break),
|
||||||
Rule::NonTerminal(95, 20, 60),
|
(47, Token::Do),
|
||||||
Rule::NonTerminal(95, 112, 27),
|
(18, Token::Else),
|
||||||
Rule::NonTerminal(95, 8, 36),
|
(50, Token::Else),
|
||||||
Rule::NonTerminal(103, 20, 60),
|
(84, Token::Elseif),
|
||||||
Rule::NonTerminal(60, 81, 6),
|
(49, Token::Then),
|
||||||
Rule::Terminal(65, Token::Plus),
|
(94, Token::End),
|
||||||
Rule::Terminal(65, Token::Minus),
|
(55, Token::Nil),
|
||||||
Rule::Terminal(65, Token::Star),
|
(55, Token::False),
|
||||||
Rule::Terminal(65, Token::Slash),
|
(55, Token::True),
|
||||||
Rule::Terminal(65, Token::SlashSlash),
|
(55, Token::Numeral(String::new())),
|
||||||
Rule::Terminal(65, Token::Caret),
|
(55, Token::StringLiteral(String::new())),
|
||||||
Rule::Terminal(65, Token::Percent),
|
(55, Token::DotDotDot),
|
||||||
Rule::Terminal(65, Token::Ampersand),
|
(55, Token::Name(String::new())),
|
||||||
Rule::Terminal(65, Token::Pipe),
|
(21, Token::Nil),
|
||||||
Rule::Terminal(65, Token::GtGt),
|
(21, Token::False),
|
||||||
Rule::Terminal(65, Token::LtLt),
|
(21, Token::True),
|
||||||
Rule::Terminal(65, Token::DotDot),
|
(21, Token::Numeral(String::new())),
|
||||||
Rule::Terminal(65, Token::Lt),
|
(21, Token::StringLiteral(String::new())),
|
||||||
Rule::Terminal(65, Token::LtEquals),
|
(21, Token::DotDotDot),
|
||||||
Rule::Terminal(65, Token::Gt),
|
(21, Token::Name(String::new())),
|
||||||
Rule::Terminal(65, Token::GtEquals),
|
(32, Token::Nil),
|
||||||
Rule::Terminal(65, Token::EqualsEquals),
|
(32, Token::False),
|
||||||
Rule::Terminal(65, Token::TildeEquals),
|
(32, Token::True),
|
||||||
Rule::Terminal(65, Token::And),
|
(32, Token::Numeral(String::new())),
|
||||||
Rule::Terminal(65, Token::Or),
|
(32, Token::StringLiteral(String::new())),
|
||||||
Rule::NonTerminal(7, 22, 29),
|
(32, Token::DotDotDot),
|
||||||
Rule::NonTerminal(7, 68, 114),
|
(32, Token::Name(String::new())),
|
||||||
Rule::Terminal(7, Token::Return),
|
(2, Token::Nil),
|
||||||
Rule::NonTerminal(7, 40, 22),
|
(2, Token::False),
|
||||||
Rule::Terminal(7, Token::Semicolon),
|
(2, Token::True),
|
||||||
Rule::Terminal(7, Token::Break),
|
(2, Token::Numeral(String::new())),
|
||||||
Rule::NonTerminal(7, 82, 81),
|
(2, Token::StringLiteral(String::new())),
|
||||||
Rule::NonTerminal(7, 79, 44),
|
(2, Token::DotDotDot),
|
||||||
Rule::NonTerminal(7, 47, 25),
|
(2, Token::Name(String::new())),
|
||||||
Rule::NonTerminal(7, 63, 46),
|
(3, Token::Comma),
|
||||||
Rule::NonTerminal(7, 104, 59),
|
(3, Token::Semicolon),
|
||||||
Rule::NonTerminal(7, 85, 30),
|
(62, Token::Comma),
|
||||||
Rule::NonTerminal(7, 75, 89),
|
(62, Token::Semicolon),
|
||||||
Rule::NonTerminal(7, 33, 0),
|
(85, Token::For),
|
||||||
Rule::NonTerminal(7, 58, 93),
|
(13, Token::Name(String::new())),
|
||||||
Rule::NonTerminal(7, 85, 31),
|
(75, Token::Function),
|
||||||
Rule::NonTerminal(7, 33, 11),
|
(82, Token::Goto),
|
||||||
Rule::NonTerminal(7, 71, 87),
|
(58, Token::If),
|
||||||
Rule::NonTerminal(7, 71, 73),
|
(53, Token::In),
|
||||||
Rule::NonTerminal(7, 39, 51),
|
(33, Token::Local),
|
||||||
Rule::Terminal(47, Token::Do),
|
(36, Token::Name(String::new())),
|
||||||
Rule::Terminal(18, Token::Else),
|
(4, Token::Name(String::new())),
|
||||||
Rule::NonTerminal(50, 18, 7),
|
(5, Token::DotDotDot),
|
||||||
Rule::Terminal(50, Token::Else),
|
(5, Token::Name(String::new())),
|
||||||
Rule::Terminal(84, Token::Elseif),
|
(71, Token::Name(String::new())),
|
||||||
Rule::NonTerminal(74, 84, 61),
|
(104, Token::Repeat),
|
||||||
Rule::NonTerminal(64, 74, 64),
|
(29, Token::Return),
|
||||||
Rule::NonTerminal(64, 84, 61),
|
(114, Token::Semicolon),
|
||||||
Rule::NonTerminal(61, 55, 49),
|
(114, Token::Nil),
|
||||||
Rule::NonTerminal(49, 67, 7),
|
(114, Token::False),
|
||||||
Rule::Terminal(49, Token::Then),
|
(114, Token::True),
|
||||||
Rule::Terminal(94, Token::End),
|
(114, Token::Numeral(String::new())),
|
||||||
Rule::Terminal(55, Token::Nil),
|
(114, Token::StringLiteral(String::new())),
|
||||||
Rule::Terminal(55, Token::False),
|
(114, Token::DotDotDot),
|
||||||
Rule::Terminal(55, Token::True),
|
(114, Token::Name(String::new())),
|
||||||
Rule::Terminal(55, Token::Numeral(String::new())),
|
(68, Token::Return),
|
||||||
Rule::Terminal(55, Token::StringLiteral(String::new())),
|
(40, Token::Semicolon),
|
||||||
Rule::Terminal(55, Token::DotDotDot),
|
(40, Token::Break),
|
||||||
Rule::NonTerminal(55, 48, 55),
|
(22, Token::Semicolon),
|
||||||
Rule::NonTerminal(55, 55, 109),
|
(22, Token::Break),
|
||||||
Rule::NonTerminal(55, 75, 113),
|
(54, Token::End),
|
||||||
Rule::NonTerminal(55, 23, 78),
|
(70, Token::End),
|
||||||
Rule::NonTerminal(55, 57, 14),
|
(100, Token::End),
|
||||||
Rule::Terminal(55, Token::Name(String::new())),
|
(11, Token::Name(String::new())),
|
||||||
Rule::NonTerminal(55, 71, 77),
|
(25, Token::End),
|
||||||
Rule::NonTerminal(55, 71, 26),
|
(78, Token::CurlyClosed),
|
||||||
Rule::NonTerminal(55, 71, 87),
|
(67, Token::Then),
|
||||||
Rule::NonTerminal(55, 71, 73),
|
(48, Token::Minus),
|
||||||
Rule::NonTerminal(109, 65, 55),
|
(48, Token::Not),
|
||||||
Rule::NonTerminal(21, 55, 96),
|
(48, Token::Hash),
|
||||||
Rule::Terminal(21, Token::Nil),
|
(48, Token::Tilde),
|
||||||
Rule::Terminal(21, Token::False),
|
(90, Token::Until),
|
||||||
Rule::Terminal(21, Token::True),
|
(106, Token::Name(String::new())),
|
||||||
Rule::Terminal(21, Token::Numeral(String::new())),
|
(79, Token::Name(String::new())),
|
||||||
Rule::Terminal(21, Token::StringLiteral(String::new())),
|
(63, Token::While),
|
||||||
Rule::Terminal(21, Token::DotDotDot),
|
(23, Token::CurlyOpen),
|
||||||
Rule::NonTerminal(21, 48, 55),
|
(76, Token::CurlyClosed),
|
||||||
Rule::NonTerminal(21, 55, 109),
|
|
||||||
Rule::NonTerminal(21, 75, 113),
|
];
|
||||||
Rule::NonTerminal(21, 23, 78),
|
|
||||||
Rule::NonTerminal(21, 57, 14),
|
pub const NONTERMINAL_RULES: [(u8, u8, u8); 219] =
|
||||||
Rule::Terminal(21, Token::Name(String::new())),
|
[
|
||||||
Rule::NonTerminal(21, 71, 77),
|
(80, 22, 29),
|
||||||
Rule::NonTerminal(21, 71, 26),
|
(80, 68, 114),
|
||||||
Rule::NonTerminal(21, 71, 87),
|
(80, 40, 22),
|
||||||
Rule::NonTerminal(21, 71, 73),
|
(80, 82, 81),
|
||||||
Rule::NonTerminal(32, 43, 28),
|
(80, 79, 44),
|
||||||
Rule::NonTerminal(32, 81, 45),
|
(80, 47, 25),
|
||||||
Rule::Terminal(32, Token::Nil),
|
(80, 63, 46),
|
||||||
Rule::Terminal(32, Token::False),
|
(80, 104, 59),
|
||||||
Rule::Terminal(32, Token::True),
|
(80, 85, 30),
|
||||||
Rule::Terminal(32, Token::Numeral(String::new())),
|
(80, 75, 89),
|
||||||
Rule::Terminal(32, Token::StringLiteral(String::new())),
|
(80, 33, 0),
|
||||||
Rule::Terminal(32, Token::DotDotDot),
|
(80, 58, 93),
|
||||||
Rule::NonTerminal(32, 48, 55),
|
(80, 85, 31),
|
||||||
Rule::NonTerminal(32, 55, 109),
|
(80, 33, 11),
|
||||||
Rule::NonTerminal(32, 75, 113),
|
(80, 71, 87),
|
||||||
Rule::NonTerminal(32, 23, 78),
|
(80, 71, 73),
|
||||||
Rule::NonTerminal(32, 57, 14),
|
(80, 39, 51),
|
||||||
Rule::Terminal(32, Token::Name(String::new())),
|
(87, 57, 105),
|
||||||
Rule::NonTerminal(32, 71, 77),
|
(87, 23, 78),
|
||||||
Rule::NonTerminal(32, 71, 26),
|
(105, 21, 34),
|
||||||
Rule::NonTerminal(32, 71, 87),
|
(44, 98, 21),
|
||||||
Rule::NonTerminal(32, 71, 73),
|
(72, 81, 95),
|
||||||
Rule::NonTerminal(28, 55, 9),
|
(95, 103, 27),
|
||||||
Rule::NonTerminal(9, 42, 45),
|
(95, 20, 60),
|
||||||
Rule::NonTerminal(45, 98, 55),
|
(95, 112, 27),
|
||||||
Rule::NonTerminal(2, 32, 3),
|
(95, 8, 36),
|
||||||
Rule::NonTerminal(2, 43, 28),
|
(103, 20, 60),
|
||||||
Rule::NonTerminal(2, 81, 45),
|
(60, 81, 6),
|
||||||
Rule::Terminal(2, Token::Nil),
|
(7, 22, 29),
|
||||||
Rule::Terminal(2, Token::False),
|
(7, 68, 114),
|
||||||
Rule::Terminal(2, Token::True),
|
(7, 40, 22),
|
||||||
Rule::Terminal(2, Token::Numeral(String::new())),
|
(7, 82, 81),
|
||||||
Rule::Terminal(2, Token::StringLiteral(String::new())),
|
(7, 79, 44),
|
||||||
Rule::Terminal(2, Token::DotDotDot),
|
(7, 47, 25),
|
||||||
Rule::NonTerminal(2, 48, 55),
|
(7, 63, 46),
|
||||||
Rule::NonTerminal(2, 55, 109),
|
(7, 104, 59),
|
||||||
Rule::NonTerminal(2, 75, 113),
|
(7, 85, 30),
|
||||||
Rule::NonTerminal(2, 23, 78),
|
(7, 75, 89),
|
||||||
Rule::NonTerminal(2, 57, 14),
|
(7, 33, 0),
|
||||||
Rule::Terminal(2, Token::Name(String::new())),
|
(7, 58, 93),
|
||||||
Rule::NonTerminal(2, 71, 77),
|
(7, 85, 31),
|
||||||
Rule::NonTerminal(2, 71, 26),
|
(7, 33, 11),
|
||||||
Rule::NonTerminal(2, 71, 87),
|
(7, 71, 87),
|
||||||
Rule::NonTerminal(2, 71, 73),
|
(7, 71, 73),
|
||||||
Rule::NonTerminal(3, 41, 62),
|
(7, 39, 51),
|
||||||
Rule::Terminal(3, Token::Comma),
|
(50, 18, 7),
|
||||||
Rule::Terminal(3, Token::Semicolon),
|
(74, 84, 61),
|
||||||
Rule::NonTerminal(3, 111, 41),
|
(64, 74, 64),
|
||||||
Rule::NonTerminal(3, 62, 32),
|
(64, 84, 61),
|
||||||
Rule::Terminal(62, Token::Comma),
|
(61, 55, 49),
|
||||||
Rule::Terminal(62, Token::Semicolon),
|
(49, 67, 7),
|
||||||
Rule::Terminal(85, Token::For),
|
(55, 48, 55),
|
||||||
Rule::NonTerminal(38, 8, 55),
|
(55, 55, 109),
|
||||||
Rule::NonTerminal(113, 57, 1),
|
(55, 75, 113),
|
||||||
Rule::NonTerminal(1, 5, 52),
|
(55, 23, 78),
|
||||||
Rule::NonTerminal(1, 34, 25),
|
(55, 57, 14),
|
||||||
Rule::NonTerminal(52, 34, 25),
|
(55, 71, 77),
|
||||||
Rule::NonTerminal(13, 81, 102),
|
(55, 71, 26),
|
||||||
Rule::Terminal(13, Token::Name(String::new())),
|
(55, 71, 87),
|
||||||
Rule::NonTerminal(102, 24, 56),
|
(55, 71, 73),
|
||||||
Rule::NonTerminal(102, 26, 24),
|
(109, 65, 55),
|
||||||
Rule::NonTerminal(102, 69, 81),
|
(21, 55, 96),
|
||||||
Rule::NonTerminal(102, 99, 81),
|
(21, 48, 55),
|
||||||
Rule::NonTerminal(56, 99, 81),
|
(21, 55, 109),
|
||||||
Rule::NonTerminal(26, 69, 81),
|
(21, 75, 113),
|
||||||
Rule::NonTerminal(24, 26, 24),
|
(21, 23, 78),
|
||||||
Rule::NonTerminal(24, 69, 81),
|
(21, 57, 14),
|
||||||
Rule::Terminal(75, Token::Function),
|
(21, 71, 77),
|
||||||
Rule::NonTerminal(73, 99, 101),
|
(21, 71, 26),
|
||||||
Rule::NonTerminal(101, 81, 87),
|
(21, 71, 87),
|
||||||
Rule::Terminal(82, Token::Goto),
|
(21, 71, 73),
|
||||||
Rule::Terminal(58, Token::If),
|
(32, 43, 28),
|
||||||
Rule::Terminal(53, Token::In),
|
(32, 81, 45),
|
||||||
Rule::NonTerminal(51, 81, 39),
|
(32, 48, 55),
|
||||||
Rule::Terminal(33, Token::Local),
|
(32, 55, 109),
|
||||||
Rule::NonTerminal(112, 8, 36),
|
(32, 75, 113),
|
||||||
Rule::NonTerminal(27, 112, 27),
|
(32, 23, 78),
|
||||||
Rule::NonTerminal(27, 8, 36),
|
(32, 57, 14),
|
||||||
Rule::NonTerminal(36, 81, 103),
|
(32, 71, 77),
|
||||||
Rule::Terminal(36, Token::Name(String::new())),
|
(32, 71, 26),
|
||||||
Rule::NonTerminal(96, 38, 96),
|
(32, 71, 87),
|
||||||
Rule::NonTerminal(96, 8, 55),
|
(32, 71, 73),
|
||||||
Rule::NonTerminal(111, 62, 32),
|
(28, 55, 9),
|
||||||
Rule::NonTerminal(41, 111, 41),
|
(9, 42, 45),
|
||||||
Rule::NonTerminal(41, 62, 32),
|
(45, 98, 55),
|
||||||
Rule::NonTerminal(91, 37, 91),
|
(2, 32, 3),
|
||||||
Rule::NonTerminal(91, 8, 81),
|
(2, 43, 28),
|
||||||
Rule::NonTerminal(37, 8, 81),
|
(2, 81, 45),
|
||||||
Rule::NonTerminal(35, 37, 35),
|
(2, 48, 55),
|
||||||
Rule::NonTerminal(35, 8, 81),
|
(2, 55, 109),
|
||||||
Rule::NonTerminal(4, 81, 91),
|
(2, 75, 113),
|
||||||
Rule::Terminal(4, Token::Name(String::new())),
|
(2, 23, 78),
|
||||||
Rule::Terminal(5, Token::DotDotDot),
|
(2, 57, 14),
|
||||||
Rule::NonTerminal(5, 4, 83),
|
(2, 71, 77),
|
||||||
Rule::NonTerminal(5, 81, 91),
|
(2, 71, 26),
|
||||||
Rule::Terminal(5, Token::Name(String::new())),
|
(2, 71, 87),
|
||||||
Rule::NonTerminal(83, 8, 66),
|
(2, 71, 73),
|
||||||
Rule::NonTerminal(71, 57, 14),
|
(3, 41, 62),
|
||||||
Rule::Terminal(71, Token::Name(String::new())),
|
(3, 111, 41),
|
||||||
Rule::NonTerminal(71, 71, 77),
|
(3, 62, 32),
|
||||||
Rule::NonTerminal(71, 71, 26),
|
(38, 8, 55),
|
||||||
Rule::NonTerminal(71, 71, 87),
|
(113, 57, 1),
|
||||||
Rule::NonTerminal(71, 71, 73),
|
(1, 5, 52),
|
||||||
Rule::NonTerminal(14, 55, 34),
|
(1, 34, 25),
|
||||||
Rule::Terminal(104, Token::Repeat),
|
(52, 34, 25),
|
||||||
Rule::NonTerminal(29, 68, 114),
|
(13, 81, 102),
|
||||||
Rule::Terminal(29, Token::Return),
|
(102, 24, 56),
|
||||||
Rule::NonTerminal(114, 21, 15),
|
(102, 26, 24),
|
||||||
Rule::Terminal(114, Token::Semicolon),
|
(102, 69, 81),
|
||||||
Rule::NonTerminal(114, 55, 96),
|
(102, 99, 81),
|
||||||
Rule::Terminal(114, Token::Nil),
|
(56, 99, 81),
|
||||||
Rule::Terminal(114, Token::False),
|
(26, 69, 81),
|
||||||
Rule::Terminal(114, Token::True),
|
(24, 26, 24),
|
||||||
Rule::Terminal(114, Token::Numeral(String::new())),
|
(24, 69, 81),
|
||||||
Rule::Terminal(114, Token::StringLiteral(String::new())),
|
(73, 99, 101),
|
||||||
Rule::Terminal(114, Token::DotDotDot),
|
(101, 81, 87),
|
||||||
Rule::NonTerminal(114, 48, 55),
|
(51, 81, 39),
|
||||||
Rule::NonTerminal(114, 55, 109),
|
(112, 8, 36),
|
||||||
Rule::NonTerminal(114, 75, 113),
|
(27, 112, 27),
|
||||||
Rule::NonTerminal(114, 23, 78),
|
(27, 8, 36),
|
||||||
Rule::NonTerminal(114, 57, 14),
|
(36, 81, 103),
|
||||||
Rule::Terminal(114, Token::Name(String::new())),
|
(96, 38, 96),
|
||||||
Rule::NonTerminal(114, 71, 77),
|
(96, 8, 55),
|
||||||
Rule::NonTerminal(114, 71, 26),
|
(111, 62, 32),
|
||||||
Rule::NonTerminal(114, 71, 87),
|
(41, 111, 41),
|
||||||
Rule::NonTerminal(114, 71, 73),
|
(41, 62, 32),
|
||||||
Rule::Terminal(68, Token::Return),
|
(91, 37, 91),
|
||||||
Rule::Terminal(40, Token::Semicolon),
|
(91, 8, 81),
|
||||||
Rule::Terminal(40, Token::Break),
|
(37, 8, 81),
|
||||||
Rule::NonTerminal(40, 82, 81),
|
(35, 37, 35),
|
||||||
Rule::NonTerminal(40, 79, 44),
|
(35, 8, 81),
|
||||||
Rule::NonTerminal(40, 47, 25),
|
(4, 81, 91),
|
||||||
Rule::NonTerminal(40, 63, 46),
|
(5, 4, 83),
|
||||||
Rule::NonTerminal(40, 104, 59),
|
(5, 81, 91),
|
||||||
Rule::NonTerminal(40, 85, 30),
|
(83, 8, 66),
|
||||||
Rule::NonTerminal(40, 75, 89),
|
(71, 57, 14),
|
||||||
Rule::NonTerminal(40, 33, 0),
|
(71, 71, 77),
|
||||||
Rule::NonTerminal(40, 58, 93),
|
(71, 71, 26),
|
||||||
Rule::NonTerminal(40, 85, 31),
|
(71, 71, 87),
|
||||||
Rule::NonTerminal(40, 33, 11),
|
(71, 71, 73),
|
||||||
Rule::NonTerminal(40, 71, 87),
|
(14, 55, 34),
|
||||||
Rule::NonTerminal(40, 71, 73),
|
(29, 68, 114),
|
||||||
Rule::NonTerminal(40, 39, 51),
|
(114, 21, 15),
|
||||||
Rule::NonTerminal(22, 40, 22),
|
(114, 55, 96),
|
||||||
Rule::Terminal(22, Token::Semicolon),
|
(114, 48, 55),
|
||||||
Rule::Terminal(22, Token::Break),
|
(114, 55, 109),
|
||||||
Rule::NonTerminal(22, 82, 81),
|
(114, 75, 113),
|
||||||
Rule::NonTerminal(22, 79, 44),
|
(114, 23, 78),
|
||||||
Rule::NonTerminal(22, 47, 25),
|
(114, 57, 14),
|
||||||
Rule::NonTerminal(22, 63, 46),
|
(114, 71, 77),
|
||||||
Rule::NonTerminal(22, 104, 59),
|
(114, 71, 26),
|
||||||
Rule::NonTerminal(22, 85, 30),
|
(114, 71, 87),
|
||||||
Rule::NonTerminal(22, 75, 89),
|
(114, 71, 73),
|
||||||
Rule::NonTerminal(22, 33, 0),
|
(40, 82, 81),
|
||||||
Rule::NonTerminal(22, 58, 93),
|
(40, 79, 44),
|
||||||
Rule::NonTerminal(22, 85, 31),
|
(40, 47, 25),
|
||||||
Rule::NonTerminal(22, 33, 11),
|
(40, 63, 46),
|
||||||
Rule::NonTerminal(22, 71, 87),
|
(40, 104, 59),
|
||||||
Rule::NonTerminal(22, 71, 73),
|
(40, 85, 30),
|
||||||
Rule::NonTerminal(22, 39, 51),
|
(40, 75, 89),
|
||||||
Rule::NonTerminal(97, 53, 10),
|
(40, 33, 0),
|
||||||
Rule::NonTerminal(10, 21, 16),
|
(40, 58, 93),
|
||||||
Rule::NonTerminal(89, 13, 113),
|
(40, 85, 31),
|
||||||
Rule::NonTerminal(0, 75, 19),
|
(40, 33, 11),
|
||||||
Rule::NonTerminal(19, 81, 113),
|
(40, 71, 87),
|
||||||
Rule::NonTerminal(93, 55, 12),
|
(40, 71, 73),
|
||||||
Rule::NonTerminal(12, 67, 54),
|
(40, 39, 51),
|
||||||
Rule::NonTerminal(54, 7, 70),
|
(22, 40, 22),
|
||||||
Rule::NonTerminal(54, 64, 100),
|
(22, 82, 81),
|
||||||
Rule::NonTerminal(54, 50, 94),
|
(22, 79, 44),
|
||||||
Rule::Terminal(54, Token::End),
|
(22, 47, 25),
|
||||||
Rule::NonTerminal(70, 64, 100),
|
(22, 63, 46),
|
||||||
Rule::NonTerminal(70, 50, 94),
|
(22, 104, 59),
|
||||||
Rule::Terminal(70, Token::End),
|
(22, 85, 30),
|
||||||
Rule::NonTerminal(100, 50, 94),
|
(22, 75, 89),
|
||||||
Rule::Terminal(100, Token::End),
|
(22, 33, 0),
|
||||||
Rule::NonTerminal(31, 81, 17),
|
(22, 58, 93),
|
||||||
Rule::NonTerminal(17, 98, 92),
|
(22, 85, 31),
|
||||||
Rule::NonTerminal(92, 55, 108),
|
(22, 33, 11),
|
||||||
Rule::NonTerminal(46, 55, 16),
|
(22, 71, 87),
|
||||||
Rule::NonTerminal(108, 8, 88),
|
(22, 71, 73),
|
||||||
Rule::NonTerminal(88, 55, 110),
|
(22, 39, 51),
|
||||||
Rule::NonTerminal(110, 38, 16),
|
(97, 53, 10),
|
||||||
Rule::NonTerminal(110, 47, 25),
|
(10, 21, 16),
|
||||||
Rule::NonTerminal(11, 72, 44),
|
(89, 13, 113),
|
||||||
Rule::NonTerminal(11, 81, 95),
|
(0, 75, 19),
|
||||||
Rule::Terminal(11, Token::Name(String::new())),
|
(19, 81, 113),
|
||||||
Rule::NonTerminal(16, 47, 25),
|
(93, 55, 12),
|
||||||
Rule::NonTerminal(25, 7, 94),
|
(12, 67, 54),
|
||||||
Rule::Terminal(25, Token::End),
|
(54, 7, 70),
|
||||||
Rule::NonTerminal(59, 7, 86),
|
(54, 64, 100),
|
||||||
Rule::NonTerminal(59, 90, 55),
|
(54, 50, 94),
|
||||||
Rule::NonTerminal(86, 90, 55),
|
(70, 64, 100),
|
||||||
Rule::NonTerminal(30, 4, 97),
|
(70, 50, 94),
|
||||||
Rule::NonTerminal(78, 2, 76),
|
(100, 50, 94),
|
||||||
Rule::Terminal(78, Token::CurlyClosed),
|
(31, 81, 17),
|
||||||
Rule::Terminal(67, Token::Then),
|
(17, 98, 92),
|
||||||
Rule::Terminal(48, Token::Minus),
|
(92, 55, 108),
|
||||||
Rule::Terminal(48, Token::Not),
|
(46, 55, 16),
|
||||||
Rule::Terminal(48, Token::Hash),
|
(108, 8, 88),
|
||||||
Rule::Terminal(48, Token::Tilde),
|
(88, 55, 110),
|
||||||
Rule::Terminal(90, Token::Until),
|
(110, 38, 16),
|
||||||
Rule::Terminal(106, Token::Name(String::new())),
|
(110, 47, 25),
|
||||||
Rule::NonTerminal(106, 71, 77),
|
(11, 72, 44),
|
||||||
Rule::NonTerminal(106, 71, 26),
|
(11, 81, 95),
|
||||||
Rule::NonTerminal(77, 43, 107),
|
(16, 47, 25),
|
||||||
Rule::NonTerminal(107, 55, 42),
|
(25, 7, 94),
|
||||||
Rule::NonTerminal(79, 106, 35),
|
(59, 7, 86),
|
||||||
Rule::Terminal(79, Token::Name(String::new())),
|
(59, 90, 55),
|
||||||
Rule::NonTerminal(79, 71, 77),
|
(86, 90, 55),
|
||||||
Rule::NonTerminal(79, 71, 26),
|
(30, 4, 97),
|
||||||
Rule::Terminal(63, Token::While),
|
(78, 2, 76),
|
||||||
Rule::Terminal(23, Token::CurlyOpen),
|
(106, 71, 77),
|
||||||
Rule::Terminal(76, Token::CurlyClosed),
|
(106, 71, 26),
|
||||||
|
(77, 43, 107),
|
||||||
|
(107, 55, 42),
|
||||||
|
(79, 106, 35),
|
||||||
|
(79, 71, 77),
|
||||||
|
(79, 71, 26),
|
||||||
|
|
||||||
];
|
];
|
||||||
@ -1,5 +1,6 @@
|
|||||||
pub mod tokenizer;
|
pub mod tokenizer;
|
||||||
pub mod parser;
|
pub mod parser;
|
||||||
|
pub mod grammar;
|
||||||
|
|
||||||
use std::{env, fs};
|
use std::{env, fs};
|
||||||
|
|
||||||
|
|||||||
323
src/parser.rs
323
src/parser.rs
@ -1,290 +1,111 @@
|
|||||||
use std::slice::Iter;
|
use std::{slice::Iter, iter::{Peekable, Map}, collections::HashMap};
|
||||||
|
|
||||||
use crate::tokenizer::Token;
|
use crate::tokenizer::Token;
|
||||||
#[derive(Debug)]
|
use crate::grammar::{NONTERMINAL_NAMES, Rule, NONTERMINAL_RULES, TERMINAL_RULES};
|
||||||
pub struct BlockNode
|
|
||||||
{
|
|
||||||
stats: Vec<StatNode>,
|
|
||||||
retstat: Option<RetstatNode>
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub enum StatNode
|
|
||||||
{
|
|
||||||
Semicolon,
|
|
||||||
Assignment(AssignmentNode),
|
|
||||||
FunctionCall(FunctionCallNode),
|
|
||||||
Label(LabelNode),
|
|
||||||
Break,
|
|
||||||
Goto(GotoNode),
|
|
||||||
Block(BlockNode),
|
|
||||||
While(WhileNode),
|
|
||||||
Repeat(RepeatNode),
|
|
||||||
If(IfNode),
|
|
||||||
For(ForNode),
|
|
||||||
ForIn(ForInNode),
|
|
||||||
Function(FunctionNode),
|
|
||||||
LocalFunction(LocalFunctionNode),
|
|
||||||
Local(LocalNode),
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct AssignmentNode
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
#[derive(Debug, Clone, Copy)]
|
||||||
#[derive(Debug)]
|
pub struct Node
|
||||||
pub struct FunctionCallNode
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct RetstatNode
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct LabelNode
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct GotoNode
|
|
||||||
{
|
|
||||||
label: String
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct WhileNode
|
|
||||||
{
|
|
||||||
condition: ExpNode,
|
|
||||||
block: BlockNode,
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct RepeatNode
|
|
||||||
{
|
|
||||||
condition: ExpNode,
|
|
||||||
block: BlockNode,
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct IfNode
|
|
||||||
{
|
|
||||||
condition: ExpNode,
|
|
||||||
then_block: BlockNode,
|
|
||||||
elseifs: Option<Vec<ElseIfNode>>
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct ElseIfNode
|
|
||||||
{
|
|
||||||
condition: ExpNode,
|
|
||||||
block: BlockNode,
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct ForNode
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct ForInNode
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct FunctionNode
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct LocalFunctionNode
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct LocalNode
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct ExpNode
|
|
||||||
{
|
{
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn parse(tokens: Vec<Token>) -> Result<BlockNode, &'static str>
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub struct AmbiguousNode
|
||||||
{
|
{
|
||||||
let mut iter = tokens.iter();
|
|
||||||
return parse_block(&mut iter);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_block(tokens: &mut Iter<'_, Token>) -> Result<BlockNode, &'static str>
|
pub fn parse(tokens: Vec<Token>) -> Result<Node, &'static str>
|
||||||
{
|
{
|
||||||
let mut block_node = BlockNode { retstat: None, stats: Vec::new() };
|
return cyk(tokens);
|
||||||
while let Some(stat) = parse_stat(tokens)?
|
|
||||||
{
|
|
||||||
block_node.stats.push(stat);
|
|
||||||
}
|
|
||||||
block_node.retstat = parse_retstat(tokens)?;
|
|
||||||
return Ok(block_node);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_stat(tokens: &mut Iter<'_, Token>) -> Result<Option<StatNode>, &'static str>
|
pub fn cyk(tokens: Vec<Token>) -> Result<Node, &'static str>
|
||||||
{
|
{
|
||||||
match tokens.next()
|
let r = NONTERMINAL_NAMES.len();
|
||||||
|
let n = tokens.len();
|
||||||
|
macro_rules! index {
|
||||||
|
($x:expr, $y:expr, $z:expr) => {
|
||||||
|
($x + $y * n + ($z as usize) * n * n)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
let mut p = vec![false; n * n * r];
|
||||||
|
//let mut back: Vec<Vec<(usize, u8, u8)>> = vec![Vec::new(); n * n * r];
|
||||||
|
println!("{n}, {r}, {}", p.len());
|
||||||
|
for s in 0..n
|
||||||
{
|
{
|
||||||
Some(token) =>
|
for (index, token) in TERMINAL_RULES
|
||||||
{
|
{
|
||||||
match token
|
if let Token::Name(_) = tokens[s]
|
||||||
{
|
{
|
||||||
Token::Semicolon => return Ok(Some(StatNode::Semicolon)),
|
if let Token::Name(_) = token
|
||||||
Token::Break => return Ok(Some(StatNode::Break)),
|
|
||||||
Token::Goto =>
|
|
||||||
{
|
{
|
||||||
match tokens.next()
|
p[index!(0, s, index)] = true
|
||||||
{
|
|
||||||
Some(Token::Name(string)) => return Ok(Some(StatNode::Goto(GotoNode { label: string.clone() }))),
|
|
||||||
_ => return Err("Failed to parse goto statement")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Token::Do =>
|
else if let Token::StringLiteral(_) = tokens[s]
|
||||||
{
|
{
|
||||||
let block = parse_block(tokens)?;
|
if let Token::StringLiteral(_) = token
|
||||||
match tokens.next()
|
|
||||||
{
|
{
|
||||||
Some(Token::End) => return Ok(Some(StatNode::Block(block))),
|
p[index!(0, s, index)] = true
|
||||||
_ => return Err("Failed to parse block statement")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Token::While =>
|
else if let Token::Numeral(_) = tokens[s]
|
||||||
{
|
{
|
||||||
if let Some(condition) = parse_exp(tokens)?
|
if let Token::Numeral(_) = token
|
||||||
{
|
{
|
||||||
match tokens.next()
|
p[index!(0, s, index)] = true
|
||||||
{
|
|
||||||
Some(Token::Do) =>
|
|
||||||
{
|
|
||||||
let block = parse_block(tokens)?;
|
|
||||||
match tokens.next()
|
|
||||||
{
|
|
||||||
Some(Token::End) => return Ok(Some(StatNode::While(WhileNode { condition, block }))),
|
|
||||||
_ => Err("No end after while block")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ => Err("No do after while condition"),
|
else if token == tokens[s]
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
{
|
||||||
return Err("Failed to parse while condition");
|
p[index!(0, s, index)] = true
|
||||||
}
|
|
||||||
}
|
|
||||||
Token::Repeat =>
|
|
||||||
{
|
|
||||||
let block = parse_block(tokens)?;
|
|
||||||
match tokens.next()
|
|
||||||
{
|
|
||||||
Some(Token::Until) =>
|
|
||||||
{
|
|
||||||
match parse_exp(tokens)?
|
|
||||||
{
|
|
||||||
Some(exp) => return Ok(Some(StatNode::Repeat(RepeatNode { condition: exp, block }))),
|
|
||||||
None => Err("Failed to parse repeat condition")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => Err("Missing until after repeat block")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Token::If =>
|
|
||||||
{
|
|
||||||
if let Some(condition) = parse_exp(tokens)?
|
|
||||||
{
|
|
||||||
match tokens.next()
|
|
||||||
{
|
|
||||||
Some(Token::Then) =>
|
|
||||||
{
|
|
||||||
let block = parse_block(tokens)?;
|
|
||||||
let mut elseifs: Vec<ElseIfNode> = Vec::new();
|
|
||||||
loop
|
|
||||||
{
|
|
||||||
match tokens.next()
|
|
||||||
{
|
|
||||||
Some(Token::Elseif) =>
|
|
||||||
{
|
|
||||||
if let Some(elseif_condition) = parse_exp(tokens)?
|
|
||||||
{
|
|
||||||
match tokens.next()
|
|
||||||
{
|
|
||||||
Some(Token::Then) =>
|
|
||||||
{
|
|
||||||
elseifs.push(ElseIfNode { block: parse_block(tokens)?, condition: elseif_condition });
|
|
||||||
}
|
|
||||||
_ => return Err("Missing Then after elseif statement"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
return Err("Failed to parse elseif condition");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some(Token::Else) =>
|
|
||||||
{
|
|
||||||
match tokens.next()
|
|
||||||
{
|
|
||||||
Some(Token::End) => return Ok(Some(StatNode::If(IfNode { condition, then_block: block, elseifs: Some(elseifs) }))),
|
|
||||||
_ => return Err("Missing end after else")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some(Token::End) =>
|
|
||||||
{
|
|
||||||
return Ok(Some(StatNode::If(IfNode { condition, then_block: block, elseifs: None })));
|
|
||||||
}
|
|
||||||
_ => return Err("Missing end after if block")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ => return Err("Missing then after if condition")
|
println!("Done initializing");
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
return Err("Failed to parse if condition");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Token::For =>
|
|
||||||
{
|
|
||||||
match tokens.next()
|
|
||||||
{
|
|
||||||
Some(Token::Name(name)) =>
|
|
||||||
{
|
|
||||||
match tokens.next()
|
|
||||||
{
|
|
||||||
Some(Token::Equals) =>
|
|
||||||
{
|
|
||||||
todo!();
|
|
||||||
}
|
|
||||||
_ => todo!()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => Err("Missing name after for"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => todo!()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None => return Ok(None),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_exp(tokens: &mut Iter<'_, Token>) -> Result<Option<ExpNode>, &'static str>
|
for l in 2..=n
|
||||||
{
|
|
||||||
todo!();
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_retstat(tokens: &mut Iter<'_, Token>) -> Result<Option<RetstatNode>, &'static str>
|
|
||||||
{
|
|
||||||
match tokens.next()
|
|
||||||
{
|
{
|
||||||
Some(Token::Return) =>
|
for s in 1..=(n - l + 1)
|
||||||
{
|
{
|
||||||
|
for _p in 1..=(l-1)
|
||||||
|
{
|
||||||
|
for &(a, b, c) in &NONTERMINAL_RULES
|
||||||
|
{
|
||||||
|
if p[index!(_p - 1, s - 1, b)] && p[index!(l - _p - 1, s + _p - 1, c)]
|
||||||
|
{
|
||||||
|
let index = index!(l - 1, s - 1, a);
|
||||||
|
p[index] = true;
|
||||||
|
/* if !back[index].contains(&(_p, b, c))
|
||||||
|
{
|
||||||
|
back[index].push((_p, b, c));
|
||||||
|
}*/
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println!("{l}");
|
||||||
|
}
|
||||||
|
let start_index = NONTERMINAL_NAMES.iter().position(|x| x == &"S_0").expect("no start index found");
|
||||||
|
if p[index!(n - 1, 0, start_index)]
|
||||||
|
{
|
||||||
|
println!("Is part of the language");
|
||||||
todo!()
|
todo!()
|
||||||
|
//return Ok(disambiguate(traverse_back(back, tokens, n, 1, start_index)));
|
||||||
}
|
}
|
||||||
None => return Ok(None),
|
else
|
||||||
_ => return Err("Found wrong token at the beginning of retstat")
|
{
|
||||||
|
return Err("Input is not part of the language")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn traverse_back(back: Vec<Vec<(usize, u8, u8)>>, tokens: Vec<Token>, l: usize, s: usize, a: usize) -> AmbiguousNode
|
||||||
|
{
|
||||||
|
todo!()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn disambiguate(root: AmbiguousNode) -> Node
|
||||||
|
{
|
||||||
|
todo!()
|
||||||
|
}
|
||||||
@ -1,4 +1,4 @@
|
|||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
pub enum Token
|
pub enum Token
|
||||||
{
|
{
|
||||||
Name(String),
|
Name(String),
|
||||||
@ -11,8 +11,7 @@ pub enum Token
|
|||||||
EqualsEquals, TildeEquals, LtEquals, GtEquals, Lt, Gt, Equals,
|
EqualsEquals, TildeEquals, LtEquals, GtEquals, Lt, Gt, Equals,
|
||||||
RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, ColonColon,
|
RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, ColonColon,
|
||||||
Semicolon, Colon, Comma, Dot, DotDot, DotDotDot,
|
Semicolon, Colon, Comma, Dot, DotDot, DotDotDot,
|
||||||
IntLiteral(String),
|
Numeral(String),
|
||||||
HexLiteral(String),
|
|
||||||
StringLiteral(String),
|
StringLiteral(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -171,7 +170,7 @@ fn tokenize_char(state: &mut TokenizerState, ch: char, last_index: &mut i32, ind
|
|||||||
'/' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Slash), TokenizerState::Slash),
|
'/' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Slash), TokenizerState::Slash),
|
||||||
';' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Semicolon), TokenizerState::Semicolon),
|
';' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Semicolon), TokenizerState::Semicolon),
|
||||||
'^' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Caret), TokenizerState::Caret),
|
'^' => tokenize_terminal_no_str(last_index, *index, token, state, Some(Token::Caret), TokenizerState::Caret),
|
||||||
'0' => tokenize_terminal(last_index, *index, token, state, Some(Token::IntLiteral("0".to_string())), TokenizerState::Zero, token_str, ch),
|
'0' => tokenize_terminal(last_index, *index, token, state, Some(Token::Numeral("0".to_string())), TokenizerState::Zero, token_str, ch),
|
||||||
'"' =>
|
'"' =>
|
||||||
{
|
{
|
||||||
*token = None;
|
*token = None;
|
||||||
@ -191,7 +190,7 @@ fn tokenize_char(state: &mut TokenizerState, ch: char, last_index: &mut i32, ind
|
|||||||
}
|
}
|
||||||
else if ch.is_numeric() && ch.is_ascii()
|
else if ch.is_numeric() && ch.is_ascii()
|
||||||
{
|
{
|
||||||
tokenize_terminal(last_index, *index, token, state, Some(Token::IntLiteral(token_str.clone())), TokenizerState::Number, token_str, ch);
|
tokenize_terminal(last_index, *index, token, state, Some(Token::Numeral(token_str.clone())), TokenizerState::Number, token_str, ch);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -422,7 +421,7 @@ fn tokenize_char(state: &mut TokenizerState, ch: char, last_index: &mut i32, ind
|
|||||||
{
|
{
|
||||||
*last_index = *index as i32;
|
*last_index = *index as i32;
|
||||||
token_str.push(ch);
|
token_str.push(ch);
|
||||||
*token = Some(Token::IntLiteral(token_str.clone()));
|
*token = Some(Token::Numeral(token_str.clone()));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -441,7 +440,7 @@ fn tokenize_char(state: &mut TokenizerState, ch: char, last_index: &mut i32, ind
|
|||||||
{
|
{
|
||||||
*last_index = *index as i32;
|
*last_index = *index as i32;
|
||||||
token_str.push(ch);
|
token_str.push(ch);
|
||||||
*token = Some(Token::HexLiteral(token_str.clone()));
|
*token = Some(Token::Numeral(token_str.clone()));
|
||||||
*state = TokenizerState::HexNumber;
|
*state = TokenizerState::HexNumber;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@ -469,7 +468,7 @@ fn tokenize_char(state: &mut TokenizerState, ch: char, last_index: &mut i32, ind
|
|||||||
{
|
{
|
||||||
*last_index = *index as i32;
|
*last_index = *index as i32;
|
||||||
token_str.push(ch);
|
token_str.push(ch);
|
||||||
*token = Some(Token::HexLiteral(token_str.clone()));
|
*token = Some(Token::Numeral(token_str.clone()));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -494,7 +493,7 @@ fn tokenize_char(state: &mut TokenizerState, ch: char, last_index: &mut i32, ind
|
|||||||
{
|
{
|
||||||
*last_index = *index as i32;
|
*last_index = *index as i32;
|
||||||
token_str.push(ch);
|
token_str.push(ch);
|
||||||
*token = Some(Token::IntLiteral(token_str.clone()));
|
*token = Some(Token::Numeral(token_str.clone()));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user