Remove throwing part of tokenizerString test

.vscode/launch.json

@@ -11,7 +11,7 @@
 			"preLaunchTask": "build",
 			// If you have changed target frameworks, make sure to update the program path.
 			"program": "${workspaceFolder}/bin/Debug/net8.0/luaaaaah.dll",
-			"args": ["test/simpleString.lua"],
+			"args": ["run", "test/stringDataNotSet.lua"],
 			"cwd": "${workspaceFolder}",
 			// For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
 			"console": "internalConsole",
@@ -23,4 +23,4 @@
 			"request": "attach"
 		}
 	]
-}
+}

Tokenizer.cs

@@ -265,14 +265,12 @@ class Tokenizer
 					break;
 					case '"':
 					{
-						currentToken = null;
 						state = State.Quote;
 						currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral);
 					}
 					break;
 					case '\'':
 					{
-						currentToken = null;
 						state = State.SingleQuote;
 						currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral);
 					}
@@ -319,6 +317,7 @@ class Tokenizer
 					{
 						currentToken.type = TokenType.StringLiteral;
 						currentToken.region.end = new(currentLocation);
+						currentToken.data ??= new Token.StringData("");
 					}
 				}
 				else
@@ -400,7 +399,7 @@ class Tokenizer
 					case 'x':
 					{
 						state = State.QuoteBackslashX;
-						throw new NotImplementedException($"\\u escape sequences are broken right now");
+						throw new NotImplementedException($"\\x escape sequences are broken right now");
 					}
 					case 'u':
 					{
@@ -515,7 +514,7 @@ class Tokenizer
 					{
 						currentToken.type = TokenType.StringLiteral;
 						currentToken.region.end = new(currentLocation);
-						currentToken.data = new Token.StringData("");
+						currentToken.data ??= new Token.StringData("");
 					}
 				}
 				else
@@ -823,6 +822,10 @@ class Tokenizer
 					AppendDataIntHex(ch);
 					state = State.HexNumber;
 				}
+				else if(ch == '.')
+				{
+					throw new NotImplementedException($"{currentLocation}: Hex floats at are not implemented");
+				}
 				else
 				{
 					BacktrackNoTypeChange();
@@ -842,6 +845,10 @@ class Tokenizer
 					currentToken!.type = TokenType.Numeral;
 					AppendDataIntHex(ch);
 				}
+				else if(ch == '.')
+				{
+					throw new NotImplementedException($"{currentLocation}: Hex floats at are not implemented");
+				}
 				else
 				{
 					BacktrackNoTypeChange();

test/stringDataNotSet.lua

@@ -0,0 +1 @@
+local smt = getmetatable("")

test/tokenizerString.lua

@@ -8,6 +8,3 @@
 
 
 abc" "123" "sdlfkgj<3" "asldkfj" zzz "" "" "" "" "" "fasd!" "afd" "" "as" zzzz
-
-"\xf7\xAff\x43"
-"\u{fa4}\u{1234}\u{12000}\u{123}"