TFileLexer: Lexing String

AdventureT · Feb 3, 2024 · 863c041 · 863c041
1 parent 3ead918
commit 863c041
Show file tree

Hide file tree

Showing 2 changed files with 77 additions and 5 deletions.
diff --git a/Tools/UnitTests/Source/TKernel/TFileLexer_Tests.cpp b/Tools/UnitTests/Source/TKernel/TFileLexer_Tests.cpp
@@ -18,11 +18,29 @@ TEST_CASE("Single Line Comment", "[TFileLexer]")
 TEST_CASE("Multi Line Comment", "[TFileLexer]")
 {
 	TFile* f = TFile::Create("multiLineComment.ini", TMODE_CREATE);
-	TPCCHAR data = "/* Test Comment\n \tABCD\n*/";
+	TPCCHAR data = "/* Test Comment\n \tABCD\n*//**/";
 	f->Write(data, strlen(data));
 	f->Destroy();
 	f = TFile::Create("multiLineComment.ini", TMODE_READONLY);
 	TFileLexer lexer = TFileLexer(f, 2);
 	REQUIRE(lexer.Expect(TFileLexer::TOKEN_COMMENT));
+	REQUIRE(lexer.Expect(TFileLexer::TOKEN_COMMENT));
+	TFileLexer::Token token = lexer.GetLastToken();
+	REQUIRE(token.GetLine() == 3);
+	f->Destroy();
+}
+
+TEST_CASE("String", "[TFileLexer]")
+{
+	TFile* f = TFile::Create("quotes.ini", TMODE_CREATE);
+	TPCCHAR data = "\"Test\"\n";
+	f->Write(data, strlen(data));
+	f->Destroy();
+	f = TFile::Create("quotes.ini", TMODE_READONLY);
+	TFileLexer lexer = TFileLexer(f, 2);
+	REQUIRE(lexer.Expect(TFileLexer::TOKEN_STRING));
+	TFileLexer::Token token = lexer.GetLastToken();
+	REQUIRE(token.GetString().GetCString() == "Test");
+	REQUIRE(token.GetLine() == 1);
 	f->Destroy();
 }
diff --git a/Toshi/Source/TKernel/TFileLexer.cpp b/Toshi/Source/TKernel/TFileLexer.cpp
@@ -197,18 +197,19 @@ TFileLexer::Token TFileLexer::get_next_token()
 		return Token(TOKEN_EOF, m_iLine);
 	}
 
+	static TCHAR s_Buffer[WORDBUF_SIZE];
+
 	if (m_bOutputComments) {
 		if (peek(0) == '/' && peek(1) == '/') {
 			advance(2);
 			TINT len = 0;
-			char buffer[WORDBUF_SIZE];
 			for (TINT i = peek(); i != '\n' && i != '\r'; i = peek()) {
-				buffer[len++] = i;
+				s_Buffer[len++] = i;
 				advance();
 				TASSERT(len < WORDBUF_SIZE);
 			}
-			buffer[len] = '\0';
-			return Token(TOKEN_COMMENT, m_iLine, TSystem::GetCStringPool()->Get(buffer));
+			s_Buffer[len] = '\0';
+			return Token(TOKEN_COMMENT, m_iLine, TSystem::GetCStringPool()->Get(s_Buffer));
 		}
 		if (peek(0) == '/' && peek(1) == '*') {
 			advance(2);
@@ -234,6 +235,59 @@ TFileLexer::Token TFileLexer::get_next_token()
 		}
 	}
 
+	if (iswalpha(peek()) || peek() == '_') {
+		TINT len = 0;
+		do
+		{
+			s_Buffer[len++] = peek();
+			advance();
+			TASSERT(len < WORDBUF_SIZE);
+		} while (iswalnum(peek()) || peek() == '_');
+		s_Buffer[len] = '\0';
+		return Token(TOKEN_IDENT, m_iLine, TSystem::GetCStringPool()->Get(s_Buffer));
+	}
+
+	if (iswdigit(peek()) != 0 || peek() != '-') {
+		if (peek() == '.' && iswdigit(peek(1))) {
+			// Do Number handling
+		}
+		else if (peek() == '"') {
+			TINT len = 0;
+			TINT prev = peek();
+			advance();
+			do
+			{
+				if (peek() == '"' && prev != '\\') {
+					advance();
+					s_Buffer[len] = '\0';
+					return Token(TOKEN_STRING, m_iLine, TSystem::GetCStringPool()->Get(s_Buffer));
+				}
+				if (peek() == '\\') {
+					advance();
+					TINT val = peek();
+					if (val == '\n') {
+						m_iLine++;
+					}
+					else if (val == 'n') {
+						val = '\n';
+					}
+					TASSERT(len < WORDBUF_SIZE);
+					s_Buffer[len++] = val;
+					advance();
+					continue;
+				}
+				if (peek() == '\n') {
+					m_iLine++;
+				}
+				TASSERT(len < WORDBUF_SIZE);
+				s_Buffer[len++] = peek();
+				advance();
+			} while (true);
+			s_Buffer[len] = '\0';
+			return Token(TOKEN_IDENT, m_iLine, TSystem::GetCStringPool()->Get(s_Buffer));
+		}
+	}
+
 	return TFileLexer::Token();
 }