.vscode/launch.json

@@ -11,7 +11,7 @@
 			"preLaunchTask": "build",
 			// If you have changed target frameworks, make sure to update the program path.
 			"program": "${workspaceFolder}/bin/Debug/net8.0/luaaaaah.dll",
-			"args": ["run", "test/stringDataNotSet.lua"],
+			"args": ["test/simpleString.lua"],
 			"cwd": "${workspaceFolder}",
 			// For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
 			"console": "internalConsole",
@@ -23,4 +23,4 @@
 			"request": "attach"
 		}
 	]
-}
+}

Tokenizer.cs

@@ -265,12 +265,14 @@ class Tokenizer
 					break;
 					case '"':
 					{
+						currentToken = null;
 						state = State.Quote;
 						currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral);
 					}
 					break;
 					case '\'':
 					{
+						currentToken = null;
 						state = State.SingleQuote;
 						currentToken = new(region: new(start: new(currentLocation), end: new(currentLocation)), type: TokenType.StringLiteral);
 					}
@@ -317,7 +319,6 @@ class Tokenizer
 					{
 						currentToken.type = TokenType.StringLiteral;
 						currentToken.region.end = new(currentLocation);
-						currentToken.data ??= new Token.StringData("");
 					}
 				}
 				else
@@ -399,7 +400,7 @@ class Tokenizer
 					case 'x':
 					{
 						state = State.QuoteBackslashX;
-						throw new NotImplementedException($"\\x escape sequences are broken right now");
+						throw new NotImplementedException($"\\u escape sequences are broken right now");
 					}
 					case 'u':
 					{
@@ -514,7 +515,7 @@ class Tokenizer
 					{
 						currentToken.type = TokenType.StringLiteral;
 						currentToken.region.end = new(currentLocation);
-						currentToken.data ??= new Token.StringData("");
+						currentToken.data = new Token.StringData("");
 					}
 				}
 				else
@@ -822,10 +823,6 @@ class Tokenizer
 					AppendDataIntHex(ch);
 					state = State.HexNumber;
 				}
-				else if(ch == '.')
-				{
-					throw new NotImplementedException($"{currentLocation}: Hex floats at are not implemented");
-				}
 				else
 				{
 					BacktrackNoTypeChange();
@@ -845,10 +842,6 @@ class Tokenizer
 					currentToken!.type = TokenType.Numeral;
 					AppendDataIntHex(ch);
 				}
-				else if(ch == '.')
-				{
-					throw new NotImplementedException($"{currentLocation}: Hex floats at are not implemented");
-				}
 				else
 				{
 					BacktrackNoTypeChange();

test/stringDataNotSet.lua

@@ -1 +0,0 @@
-local smt = getmetatable("")

test/tokenizerString.lua

@@ -8,3 +8,6 @@
 
 
 abc" "123" "sdlfkgj<3" "asldkfj" zzz "" "" "" "" "" "fasd!" "afd" "" "as" zzzz
+
+"\xf7\xAff\x43"
+"\u{fa4}\u{1234}\u{12000}\u{123}"