0x4261756D/luaaaaah
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add float lexing

Browse Source
This commit is contained in:
0x4261756D 2024-01-29 18:24:08 +01:00
parent ec312b3132
commit 39521fbb19
2 changed files with 41 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
Tokenizer.cs
View File

@ -376,7 +376,7 @@ class Tokenizer
else if(char.IsDigit(ch)) else if(char.IsDigit(ch))
{ {
lastIndex = index; lastIndex = index;
state = State.Number; state = State.Integer;
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Numeral, data: new Token.NumeralData(new INumeral.Integer(ch - '0'))); currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Numeral, data: new Token.NumeralData(new INumeral.Integer(ch - '0')));
} }
else else
@ -702,7 +702,11 @@ class Tokenizer
} }
else if(ch == '.') else if(ch == '.')
{ {
throw new NotImplementedException(); state = State.Float;
currentToken!.type = null;
currentToken!.data = null;
AppendDataChar('0');
AppendDataChar('.');
} }
else if(char.IsAsciiDigit(ch)) else if(char.IsAsciiDigit(ch))
{ {
@ -724,6 +728,31 @@ class Tokenizer
} }
} }
break; break;
case State.Float:
{
if(char.IsAsciiDigit(ch))
{
lastIndex = index;
AppendDataChar(ch);
}
else
{
if(currentToken == null)
{
throw new Exception($"Lexer error at {currentLocation}");
}
currentLocation = new(currentToken.region.end);
currentToken.type = TokenType.Numeral;
currentToken.data = new Token.NumeralData(new INumeral.Float(float.Parse(((Token.StringData)currentToken.data!).data)));
tokens.Add(currentToken);
currentToken = null;
index = lastIndex!.Value;
lastIndex = null;
state = State.Start;
}
}
break;
case State.HexNumberX: case State.HexNumberX:
{ {
if(char.IsAsciiHexDigit(ch)) if(char.IsAsciiHexDigit(ch))
@ -776,7 +805,7 @@ class Tokenizer
} }
} }
break; break;
case State.Number: case State.Integer:
{ {
if(ch == 'e') if(ch == 'e')
{ {
@ -785,7 +814,9 @@ class Tokenizer
} }
else if(ch == '.') else if(ch == '.')
{ {
throw new NotImplementedException(); currentToken!.type = null;
currentToken.data = new Token.StringData($"{((INumeral.Integer)((Token.NumeralData)currentToken!.data!).numeral).value}.");
state = State.Float;
} }
else if(char.IsAsciiDigit(ch)) else if(char.IsAsciiDigit(ch))
{ {
@ -3610,7 +3641,7 @@ class Tokenizer
private enum State private enum State
{ {
Start, Start,
Quote, SingleQuote, Name, Number, Zero, Quote, SingleQuote, Name, Integer, Float, Zero,
A, B, D, E, F, G, I, L, N, O, R, T, U, W, A, B, D, E, F, G, I, L, N, O, R, T, U, W,
Plus, Minus, Star, Slash, Percent, Caret, Hash, Plus, Minus, Star, Slash, Percent, Caret, Hash,
Ampersand, Tilde, Pipe, Lt, Gt, Equals, RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, StringStartLongBracket, StringWithLongBracket, StringEndLongBracket, Ampersand, Tilde, Pipe, Lt, Gt, Equals, RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, StringStartLongBracket, StringWithLongBracket, StringEndLongBracket,

5
test/simpleFloat.lua Normal file
View File

@ -0,0 +1,5 @@
0.000001
0.1
0.12345
1234566788.21212