From 39521fbb19f911f328e02070eda70f04b2c4b424 Mon Sep 17 00:00:00 2001
From: 0x4261756D <38735823+0x4261756D@users.noreply.github.com>
Date: Mon, 29 Jan 2024 18:24:08 +0100
Subject: [PATCH] Add float lexing

---
 Tokenizer.cs         | 41 ++++++++++++++++++++++++++++++++++++-----
 test/simpleFloat.lua |  5 +++++
 2 files changed, 41 insertions(+), 5 deletions(-)
 create mode 100644 test/simpleFloat.lua

diff --git a/Tokenizer.cs b/Tokenizer.cs
index 3189493..d37df2d 100644
--- a/Tokenizer.cs
+++ b/Tokenizer.cs
@@ -376,7 +376,7 @@ class Tokenizer
 						else if(char.IsDigit(ch))
 						{
 							lastIndex = index;
-							state = State.Number;
+							state = State.Integer;
 							currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.Numeral, data: new Token.NumeralData(new INumeral.Integer(ch - '0')));
 						}
 						else
@@ -702,7 +702,11 @@ class Tokenizer
 				}
 				else if(ch == '.')
 				{
-					throw new NotImplementedException();
+					state = State.Float;
+					currentToken!.type = null;
+					currentToken!.data = null;
+					AppendDataChar('0');
+					AppendDataChar('.');
 				}
 				else if(char.IsAsciiDigit(ch))
 				{
@@ -724,6 +728,31 @@ class Tokenizer
 				}
 			}
 			break;
+			case State.Float:
+			{
+				if(char.IsAsciiDigit(ch))
+				{
+					lastIndex = index;
+					AppendDataChar(ch);
+				}
+				else
+				{
+					if(currentToken == null)
+					{
+						throw new Exception($"Lexer error at {currentLocation}");
+					}
+					currentLocation = new(currentToken.region.end);
+					currentToken.type = TokenType.Numeral;
+					currentToken.data = new Token.NumeralData(new INumeral.Float(float.Parse(((Token.StringData)currentToken.data!).data)));
+					tokens.Add(currentToken);
+					currentToken = null;
+					index = lastIndex!.Value;
+					lastIndex = null;
+					state = State.Start;
+
+				}
+			}
+			break;
 			case State.HexNumberX:
 			{
 				if(char.IsAsciiHexDigit(ch))
@@ -776,7 +805,7 @@ class Tokenizer
 				}
 			}
 			break;
-			case State.Number:
+			case State.Integer:
 			{
 				if(ch == 'e')
 				{
@@ -785,7 +814,9 @@ class Tokenizer
 				}
 				else if(ch == '.')
 				{
-					throw new NotImplementedException();
+					currentToken!.type = null;
+					currentToken.data = new Token.StringData($"{((INumeral.Integer)((Token.NumeralData)currentToken!.data!).numeral).value}.");
+					state = State.Float;
 				}
 				else if(char.IsAsciiDigit(ch))
 				{
@@ -3610,7 +3641,7 @@ class Tokenizer
 	private enum State
 	{
 		Start,
-		Quote, SingleQuote, Name, Number, Zero,
+		Quote, SingleQuote, Name, Integer, Float, Zero,
 		A, B, D, E, F, G, I, L, N, O, R, T, U, W,
 		Plus, Minus, Star, Slash, Percent, Caret, Hash,
 		Ampersand, Tilde, Pipe, Lt, Gt, Equals, RoundOpen, RoundClosed, CurlyOpen, CurlyClosed, SquareOpen, SquareClosed, StringStartLongBracket, StringWithLongBracket, StringEndLongBracket,
diff --git a/test/simpleFloat.lua b/test/simpleFloat.lua
new file mode 100644
index 0000000..09cb15e
--- /dev/null
+++ b/test/simpleFloat.lua
@@ -0,0 +1,5 @@
+0.000001
+0.1
+0.12345
+1234566788.21212
+