TFileLexer work

Fix TMin/TMax (building error)
Fix TNativeFile::Read bug

Tools/UnitTests/Source/TKernel/TFileLexer_Tests.cpp

@@ -0,0 +1,10 @@
+#include <catch2/catch_test_macros.hpp>
+#include "TKernel/TFileLexer.h"
+
+TOSHI_NAMESPACE_USING
+
+TEST_CASE("Lexme", "[TFileLexer]")
+{
+	TFileLexer lexer = TFileLexer(TFile::Create("F:\\Jurassic Park Operation Genesis\\^JPOG^\\Data\\Credits.ini", TMODE_READONLY), 2);
+	TFileLexer::Token token = lexer.GetNextToken();
+}

Toshi/Include/TKernel/TArray.h

@@ -144,6 +144,12 @@ class TArray
 	};
 
 public:
+
+	TArray() : TArray(10, 0)
+	{
+
+	}
+
 	TArray(TINT a_iGrowSize, TINT a_iSize)
 	{
 		m_iGrowSize = a_iGrowSize;

Toshi/Include/TKernel/TCString.h

@@ -6,9 +6,13 @@
 
 TOSHI_NAMESPACE_BEGIN
 
+class TPCString;
+
 class TKERNELINTERFACE_EXPORTS TCString 
 {
 public:
+
+	friend TCString TOSHI_API operator+(TPCCHAR a_pLHS, const TCString& a_rRHS);
 
 	TCString()
 	{
@@ -98,6 +102,19 @@ class TKERNELINTERFACE_EXPORTS TCString
 		return m_pBuffer;
 	}
 
+	TCString operator+(TPCCHAR a_pRHS) const
+	{
+		TCString str(*this);
+		TASSERT(a_pRHS);
+		return str.Concat(a_pRHS);
+	}
+
+	TCString operator+(const TCString& a_rRHS) const
+	{
+		TCString str(*this);
+		return str.Concat(a_rRHS);
+	}
+
 	TCString& operator+=(TPCCHAR a_pcString)
 	{
 		return *this;
@@ -132,7 +149,6 @@ class TKERNELINTERFACE_EXPORTS TCString
 	TINT Length() const { return m_iStrLen; }
 
 private:
-
 	TBOOL AllocBuffer(TINT a_iLength, TBOOL a_bClear = TTRUE);
 
 	void FreeBuffer()
@@ -161,4 +177,7 @@ class TKERNELINTERFACE_EXPORTS TCString
 	TINT m_iStrLen    : 24 = 0; // 0x5
 };
 
+TCString TOSHI_API operator+(TPCCHAR a_pLHS, const TCString& a_rRHS);
+TCString TOSHI_API operator+(const TCString& a_rLHS, const TPCString& a_rRHS);
+
 TOSHI_NAMESPACE_END

Toshi/Include/TKernel/TFileLexer.h

@@ -2,6 +2,18 @@
 
 #include "TDebug.h"
 #include "TPCString.h"
+#include "TFile.h"
+#include "TMemory.h"
+#include <new>
+
+#if defined(TLEXERUTF8)
+#define TLEXERGETCHAR m_pFile->GetCChar
+#elif defined(TLEXERUTF16)
+#define TLEXERGETCHAR m_pFile->GetWChar
+#else
+#define TLEXERGETCHAR m_pFile->GetCChar
+#endif
+#define TGETLOOKAHEADSIZE(size) 1 << (((size * 2 - 1) >> 0x17) + 0x81U & 0x1f)
 
 TOSHI_NAMESPACE_BEGIN
 
@@ -35,8 +47,10 @@ class TKERNELINTERFACE_EXPORTS TFileLexer
 
 public:
 
+	TFileLexer();
+	TFileLexer(TFile* a_pFile, TINT a_iTokenLookaheadSize);
 
-	class Token 
+	class TKERNELINTERFACE_EXPORTS Token
 	{
 	public:
 		void assign(const Token& a_rToken);
@@ -131,9 +145,9 @@ class TKERNELINTERFACE_EXPORTS TFileLexer
 			m_iLine = a_iLine;
 			TASSERT(m_type == TFileLexer::TOKEN_IDENT   ||
 				    m_type == TFileLexer::TOKEN_STRING  ||
-				    m_type == TFileLexer::TOKEN_COMMENT ||
-				    m_type == TFileLexer::TOKEN_FLOAT   || 
-				    m_type == TFileLexer::TOKEN_COMMENT);
+				    m_type == TFileLexer::TOKEN_INTEGER ||
+			        m_type == TFileLexer::TOKEN_FLOAT   || 
+		            m_type == TFileLexer::TOKEN_COMMENT);
 		}
 
 		Token(const TFileLexer::Token& a_rToken)
@@ -161,17 +175,93 @@ class TKERNELINTERFACE_EXPORTS TFileLexer
 
 	private:
 		TFileLexer::TokenType m_type; // 0x0
-		union {						  // |
-			TPCString m_szValue;	  // |
-			TINT m_iValue;			  //  -> 0x4
-			TUINT m_uiValue;		  // |
-			TFLOAT m_fValue;		  // |
+		union {                       // |
+			TPCString m_szValue;      // |
+			TINT m_iValue;            //  -> 0x4
+			TUINT m_uiValue;          // |
+			TFLOAT m_fValue;          // |
 		};                            // |
 		TINT m_iLine;                 // 0x8        
 	};
 
+	struct LookaheadTokens
+	{
+	public:
+		Token* GetTokens()
+		{
+			return (Token*)(this + 1);
+		}
+
+		// That's crazy but they probably did that too
+		static LookaheadTokens* Allocate(int a_iCount = 1)
+		{
+			LookaheadTokens* ltokens = (LookaheadTokens*)::operator new (sizeof(LookaheadTokens) + sizeof(Token) * a_iCount);
+			ltokens->m_iCount = a_iCount;
+			for (TINT i = 0; i < a_iCount; i++) {
+				new (&ltokens->GetTokens()[i]) Token();
+			}
+			return ltokens;
+		}
+
+		static void Free(Token* a_pHeadToken)
+		{
+			delete FromToken(a_pHeadToken);
+		}
+
+		static LookaheadTokens* FromToken(Token* a_pHeadToken)
+		{
+			return (LookaheadTokens*)((TUINT*)a_pHeadToken - sizeof(LookaheadTokens));
+		}
+
+	public:
+		int m_iCount;
+	};
+
+	TBOOL Expect(TFileLexer::TokenType a_type);
+	TBOOL Expect(TFileLexer::TokenType a_type, TFileLexer::Token& a_rToken);
+
+	TFileLexer::Token GetLastToken();
+	TFileLexer::Token GetNextToken();
+
+	TFileLexer::Token PeekNextToken(TINT a_iDistance);
+
+	void SetCharacterLookaheadSize(TINT a_iCharLookaheadSize);
+	void SetInputStream(TFile* a_pFile);
+
 	static TPCCHAR TOSHI_API tostring(TokenType a_type);
 
+protected:
+	TFileLexer::Token get_next_token();
+	void skipWhiteSpace();
+
+	void advance();
+	void advance(TINT a_dist);
+
+	void fillLookAhead();
+
+	TINT peek();
+	TINT peek(TINT a_dist);
+
+private:
+	TFile* m_pFile;                        // 0x4
+	TBOOL m_bOutputComments;               // 0x8
+	TINT m_iCharLookaheadSize;             // 0xC
+	TINT m_iCharLookaheadMask;             // 0x10
+	TINT* m_piCharLookahead;               // 0x14
+	TINT m_iCharLookaheadBack;             // 0x18
+	TINT m_iCharLookaheadFront;            // 0x1C
+
+	TINT m_iLine;                          // 0x20
+	TINT m_iTokenLookaheadSize;            // 0x24
+	TINT m_iTokenLookaheadMask;            // 0x28
+	TFileLexer::Token m_oToken;            // 0x2C
+	TFileLexer::Token* m_pLookaheadTokens; // 0x38
+	TINT m_iTokenLookaheadBuffered;        // 0x3C
+	TINT m_iTokenLookaheadFront;           // 0x40
+	TINT m_iTokenLookaheadBack;            // 0x44
+
+	TBOOL m_bEOF;                          // 0x6D
+	TArray<TPCString> m_aDefines;          // 0x70;
 };
 
 TOSHI_NAMESPACE_END

Toshi/Include/TKernel/TMath.h

@@ -1,4 +1,4 @@
 #pragma once
 
-#define TMIN(x,y) (((a)<(b))?(a):(b))
-#define TMAX(x,y) (((a)>(b))?(a):(b))
+#define TMIN(a,b) (((a)<(b))?(a):(b))
+#define TMAX(a,b) (((a)>(b))?(a):(b))

Toshi/Include/TKernel/TPCString.h

@@ -9,50 +9,83 @@
 
 TOSHI_NAMESPACE_BEGIN
 
-class TKERNELINTERFACE_EXPORTS TPCString
+class TCStringPool;
+
+class TKERNELINTERFACE_EXPORTS TPooledCString : public TRefCounted
 {
+	DECLARE_FREELIST(TPooledCString)
 
-};
+	friend class TCStringPool;
+	friend class TPCString;
 
-class TPooledCString;
+private:
 
-class TKERNELINTERFACE_EXPORTS TCStringPool
-{
-	friend class TPooledCString;
-public:
-	TPCString Get(TPCCHAR a_szString);
-protected:
-	void Remove(TPooledCString& a_rPooledCString);
+	void Delete();
+
+	~TPooledCString();
 
 private:
-	TArray<TPooledCString> m_oPooledCStrings;
+	// 0x0 base (m_iRefCount)
+	TCString m_oString;          // 0x4
+	TCStringPool* m_pStringPool; // 0xC
 };
 
-class TKERNELINTERFACE_EXPORTS TPooledCString : public TRefCounted
+class TKERNELINTERFACE_EXPORTS TPCString
 {
-	DECLARE_FREELIST(TPooledCString)
-
-	friend class TCStringPool;
+public:
 
-private:
+	static inline const TCString ms_sEmpty;
 
-	void Delete()
+	TPCString()
 	{
-		GetFreeList().Delete(this);
-		delete this;
+		m_pPCS = TNULL;
 	}
 
-	~TPooledCString()
+	TPCString(const TPCString& a_rPCS) : TPCString()
 	{
-		if (m_pStringPool) {
-			m_pStringPool->Remove(*this);
+		if (a_rPCS.m_pPCS) {
+			a_rPCS.m_pPCS->IncRefCount();
 		}
 	}
 
+	~TPCString()
+	{
+		if (m_pPCS) {
+			if (m_pPCS->DecRefCount() == 0) {
+				m_pPCS->Delete();
+			}
+			m_pPCS = TNULL;
+		}
+	}
+
+	operator const TCString* () const
+	{
+		return m_pPCS ? &m_pPCS->m_oString : &ms_sEmpty;
+	}
+
 private:
-	// 0x0 base (m_iRefCount)
-	TCString m_oString;          // 0x4
-	TCStringPool* m_pStringPool; // 0xC
+
+	TPCString(TPooledCString* a_pPCS)
+	{
+		m_pPCS = a_pPCS;
+		if (a_pPCS) {
+			a_pPCS->IncRefCount();
+		}
+	}
+
+	TPooledCString* m_pPCS;
+};
+
+class TKERNELINTERFACE_EXPORTS TCStringPool
+{
+	friend class TPooledCString;
+public:
+	TPCString Get(TPCCHAR a_szString);
+protected:
+	void Remove(TPooledCString& a_rPooledCString);
+
+private:
+	TArray<TPooledCString> m_oPooledCStrings;
 };
 
 TOSHI_NAMESPACE_END

Toshi/Include/TKernel/TRefCounted.h

@@ -7,7 +7,7 @@ class TKERNELINTERFACE_EXPORTS TRefCounted
 {
 public:
 
-	TINT DecRefCount(TPVOID)
+	TINT DecRefCount(TPVOID a_pData = TNULL)
 	{
 		return --m_iRefCount;
 	}
@@ -17,7 +17,7 @@ class TKERNELINTERFACE_EXPORTS TRefCounted
 		return m_iRefCount;
 	}
 
-	TINT IncRefCount(TPVOID)
+	TINT IncRefCount(TPVOID a_pData = TNULL)
 	{
 		return ++m_iRefCount;
 	}

Toshi/Source/TKernel/TCString.cpp

@@ -2,8 +2,9 @@
 #include "TSystemTools.h"
 #include <string.h>
 #include <TKernel/TMemory.h>
+#include "TPCString.h"
 
-TOSHI_NAMESPACE_USING
+TOSHI_NAMESPACE_BEGIN
 
 TBOOL TCString::AllocBuffer(TINT a_iLength, TBOOL a_bClear)
 {
@@ -149,3 +150,17 @@ void TCString::Truncate(TINT a_iLength)
 		tfree(pBuffer);
 	}
 }
+
+TCString TOSHI_API operator+(TPCCHAR a_pLHS, const TCString& a_rRHS)
+{
+	TCString str(a_pLHS);
+	TASSERT(a_pLHS);
+	return str.Concat(a_rRHS);
+}
+
+TCString TOSHI_API operator+(const TCString& a_rLHS, const TPCString& a_rRHS)
+{
+	return "";
+}
+
+TOSHI_NAMESPACE_END