Extend API of Lexer with a new method PeekNextToken

bnoazx005 · bnoazx005 · commit c9452176420a · 2023-07-11T22:00:23.000+04:00
diff --git a/source/tcppLibrary.hpp b/source/tcppLibrary.hpp
@@ -195,6 +195,14 @@ namespace tcpp
 
 			TToken GetNextToken() TCPP_NOEXCEPT;
 
+			/*!
+				\brief The method allows to peek token without changing current pointer
+
+				\param[in] offset Distance of overlooking, passing 0 is equivalent to invoking GetNextToken()
+			*/
+
+			TToken PeekNextToken(size_t offset = 1);
+
 			bool HasNextToken() const TCPP_NOEXCEPT;
 
 			void AppendFront(const std::vector<TToken>& tokens) TCPP_NOEXCEPT;
@@ -205,6 +213,8 @@ namespace tcpp
 			size_t GetCurrLineIndex() const TCPP_NOEXCEPT;
 			size_t GetCurrPos() const TCPP_NOEXCEPT;
 		private:
+			TToken _getNextTokenInternal(bool ignoreQueue) TCPP_NOEXCEPT;
+
 			TToken _scanTokens(std::string& inputLine) TCPP_NOEXCEPT;
 
 			std::string _requestSourceLine() TCPP_NOEXCEPT;
@@ -450,24 +460,27 @@ namespace tcpp
 
 	TToken Lexer::GetNextToken() TCPP_NOEXCEPT
 	{
-		if (!mTokensQueue.empty())
+		return _getNextTokenInternal(false);
+	}
+
+	TToken Lexer::PeekNextToken(size_t offset)
+	{
+		if (!mTokensQueue.empty() && offset < mTokensQueue.size())
 		{
-			auto currToken = mTokensQueue.front();
-			mTokensQueue.pop_front();
+			return *std::next(mTokensQueue.begin(), offset);
+		}
 
-			return currToken;
+		for (size_t i = 0; i < (offset - mTokensQueue.size()); i++)
+		{
+			mTokensQueue.push_back(_getNextTokenInternal(true));
 		}
 
-		if (mCurrLine.empty())
+		if (mTokensQueue.empty())
 		{
-			// \note if it's still empty then we've reached the end of the source
-			if ((mCurrLine = _requestSourceLine()).empty())
-			{
-				return mEOFToken;
-			}
+			return GetNextToken();
 		}
 
-		return _scanTokens(mCurrLine);
+		return mTokensQueue.back();
 	}
 
 	bool Lexer::HasNextToken() const TCPP_NOEXCEPT
@@ -585,6 +598,28 @@ namespace tcpp
 	}
 
 
+	TToken Lexer::_getNextTokenInternal(bool ignoreQueue) TCPP_NOEXCEPT
+	{
+		if (!ignoreQueue && !mTokensQueue.empty())
+		{
+			auto currToken = mTokensQueue.front();
+			mTokensQueue.pop_front();
+
+			return currToken;
+		}
+
+		if (mCurrLine.empty())
+		{
+			// \note if it's still empty then we've reached the end of the source
+			if ((mCurrLine = _requestSourceLine()).empty())
+			{
+				return mEOFToken;
+			}
+		}
+
+		return _scanTokens(mCurrLine);
+	}
+
 	TToken Lexer::_scanTokens(std::string& inputLine) TCPP_NOEXCEPT
 	{
 		char ch = '\0';
@@ -983,10 +1018,19 @@ namespace tcpp
 	}
 
 
+	static const std::vector<std::string> BuiltInDefines
+	{
+		"__LINE__",
+	};
+
+
 	Preprocessor::Preprocessor(Lexer& lexer, const TOnErrorCallback& onErrorCallback, const TOnIncludeCallback& onIncludeCallback) TCPP_NOEXCEPT:
 		mpLexer(&lexer), mOnErrorCallback(onErrorCallback), mOnIncludeCallback(onIncludeCallback)
 	{
-		mSymTable.push_back({ "__LINE__" });
+		for (auto&& currSystemDefine : BuiltInDefines)
+		{
+			mSymTable.push_back({ currSystemDefine });
+		}
 	}
 
 	bool Preprocessor::AddCustomDirectiveHandler(const std::string& directive, const TDirectiveHandler& handler) TCPP_NOEXCEPT
@@ -1253,7 +1297,7 @@ namespace tcpp
 		{
 			static const std::unordered_map<std::string, std::function<TToken()>> systemMacrosTable
 			{
-				{ "__LINE__", [&idToken]() { return TToken { E_TOKEN_TYPE::BLOB, std::to_string(idToken.mLineId) }; } }
+				{ BuiltInDefines[0], [&idToken]() { return TToken {E_TOKEN_TYPE::BLOB, std::to_string(idToken.mLineId)}; }} // __LINE__
 			};
 
 			auto iter = systemMacrosTable.find(macroDesc.mName);
diff --git a/tests/lexerTests.cpp b/tests/lexerTests.cpp
@@ -329,4 +329,19 @@ TEST_CASE("Lexer Tests")
 		REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::COMMENTARY);
 		REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END);
 	}
+
+	SECTION("TestPeekNextToken_IterateOverSequenceUsingOffset_CorrectlyProcessStreamAndReturnsTokens")
+	{
+		Lexer lexer(std::make_unique<MockInputStream>(std::vector<std::string> { "(2, 3)" }));
+
+		REQUIRE(lexer.PeekNextToken(0).mType == E_TOKEN_TYPE::OPEN_BRACKET); // PeekNextToken(0) equals to GetNextToken()
+		REQUIRE(lexer.PeekNextToken(1).mType == E_TOKEN_TYPE::NUMBER);
+		REQUIRE(lexer.PeekNextToken(2).mType == E_TOKEN_TYPE::COMMA);
+		REQUIRE(lexer.PeekNextToken(3).mType == E_TOKEN_TYPE::SPACE);
+		REQUIRE(lexer.PeekNextToken(4).mType == E_TOKEN_TYPE::NUMBER);
+		REQUIRE(lexer.PeekNextToken(5).mType == E_TOKEN_TYPE::CLOSE_BRACKET);
+		REQUIRE(lexer.PeekNextToken(6).mType == E_TOKEN_TYPE::END);
+
+		REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NUMBER);
+	}
 }
