0x4261756D/luaaaaah
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add float lexing

Browse Source
This commit is contained in:
0x4261756D 2024-01-29 18:24:08 +01:00
parent ec312b3132
commit 39521fbb19
2 changed files with 41 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
Tokenizer.cs
View File

@ -376,7 +376,7 @@ class Tokenizer
else if(char.IsDigit(ch))
{
lastIndex = index;
state = State.Number;
state = State.Integer;
currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Numeral, data: new Token.NumeralData(new INumeral.Integer(ch - '0')));
}
else
@ -702,7 +702,11 @@ class Tokenizer
}
else if(ch == '.')
{
throw new NotImplementedException();
state = State.Float;
currentToken!.type = null;
currentToken!.data = null;
AppendDataChar('0');
AppendDataChar('.');
}
else if(char.IsAsciiDigit(ch))
{
@ -724,6 +728,31 @@ class Tokenizer
}
}
break;
case State.Float:
{
if(char.IsAsciiDigit(ch))
{
lastIndex = index;
AppendDataChar(ch);
}
else
{
if(currentToken == null)
{
throw new Exception($"Lexer error at {currentLocation}");
}
currentLocation = new(currentToken.region.end);
currentToken.type = TokenType.Numeral;
currentToken.data = new Token.NumeralData(new INumeral.Float(float.Parse(((Token.StringData)currentToken.data!).data)));
tokens.Add(currentToken);
currentToken = null;
index = lastIndex!.Value;
lastIndex = null;
state = State.Start;
}
}
break;
case State.HexNumberX:
{
if(char.IsAsciiHexDigit(ch))
@ -776,7 +805,7 @@ class Tokenizer
}
}
break;
case State.Number:
case State.Integer:
{
if(ch == 'e')
{
@ -785,7 +814,9 @@ class Tokenizer
}
else if(ch == '.')
{
throw new NotImplementedException();
currentToken!.type = null;
currentToken.data = new Token.StringData($"{((INumeral.Integer)((Token.NumeralData)currentToken!.data!).numeral).value}.");
state = State.Float;
}
else if(char.IsAsciiDigit(ch))
{
@ -3610,7 +3641,7 @@ class Tokenizer
private enum State
{
Start,
Quote, SingleQuote, Name, Number, Zero,
Quote, SingleQuote, Name, Integer, Float, Zero,
A, B, D, E, F, G, I, L, N, O, R, T, U, W,
Plus, Minus, Star, Slash, Percent, Caret, Hash,
Ampersand, Tilde, Pipe, Lt, Gt, Equals, RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, StringStartLongBracket, StringWithLongBracket, StringEndLongBracket,

5
test/simpleFloat.lua Normal file
View File

@ -0,0 +1,5 @@
0.000001
0.1
0.12345
1234566788.21212