Commit 4e0880bd authored by Erik Verbruggen's avatar Erik Verbruggen

C++: Small fixes to the token buffer.

- Fixed blocking macro check
- Enabled buffer "compression": when tokens are generated and no new
macro is being blocked, then prepend the tokens to the previous buffer.
This happens a lot when undo-ing look-ahead.
- Added documentation

Change-Id: I6fa816d94ce4696e473bdbc4f3bf477d77e4dd51
Reviewed-by: default avatarNikolai Kosjar <>
parent eb09c87d
......@@ -106,6 +106,14 @@ typedef ScopedSwap<unsigned> ScopedUnsignedSwap;
namespace CPlusPlus {
namespace Internal {
/// Buffers tokens for the Preprocessor::lex() to read next. Do not use this
/// class directly, but use Preprocessor::State::pushTokenBuffer .
/// New tokens are added when undoing look-ahead, or after expanding a macro.
/// When macro expansion happened, the macro is passed in, and blocked until
/// all tokens generated by it (and by subsequent expansion of those generated
/// tokens) are read from the buffer. See Preprocessor::lex() for details on
/// exactly when the buffer (and subsequently a blocking macro) is removed.
struct TokenBuffer
std::deque<PPToken> tokens;
......@@ -121,8 +129,9 @@ struct TokenBuffer
return false;
for (const TokenBuffer *it = this; it; it = it->next)
if (it->macro == macro && it->macro->name() == macro->name())
return true;
if (it->macro)
if (it->macro == macro || (it->macro->name() == macro->name()))
return true;
return false;
......@@ -555,22 +564,21 @@ Preprocessor::State::State()
void Preprocessor::State::pushTokenBuffer(const PPToken *start, const PPToken *end, const Macro *macro)
if (m_tokenBufferDepth <= MAX_TOKEN_BUFFER_DEPTH) {
// This does not work correctly for boost's preprocessor library, or that library exposes a bug in the code.
if (macro || !m_tokenBuffer) {
// If there is a new blocking macro (or no token buffer yet), create
// one.
m_tokenBuffer = new TokenBuffer(start, end, macro, m_tokenBuffer);
} else {
// No new blocking macro is passed in, so tokens can be prepended to
// the existing buffer.
m_tokenBuffer->tokens.insert(m_tokenBuffer->tokens.begin(), start, end);
unsigned tkCount = 0;
for (TokenBuffer *it = m_tokenBuffer; it; it = m_tokenBuffer->next)
tkCount += it->tokens.size();
qDebug()<<"New depth:" << m_tokenBufferDepth << "with total token count:" << tkCount;
m_tokenBuffer = new TokenBuffer(start, end, macro, m_tokenBuffer);
......@@ -693,13 +701,22 @@ void Preprocessor::lex(PPToken *tk)
if (m_state.m_tokenBuffer) {
// There is a token buffer, so read from there.
if (m_state.m_tokenBuffer->tokens.empty()) {
// The token buffer is empty, so pop it, and start over.
goto _Lagain;
*tk = m_state.m_tokenBuffer->tokens.front();
// The token buffer might now be empty. We leave it in, because the
// token we just read might expand into new tokens, or might be a call
// to the macro that generated this token. In either case, the macro
// that generated the token still needs to be blocked (!), which is
// recorded in the token buffer. Removing the blocked macro and the
// empty token buffer happens the next time that this method is called.
} else {
// No token buffer, so have the lexer scan the next token.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment