[C++] Rewrite of the preprocessor.

This rewrite fixes a couple of issues with the pre-processor. It now
supports:
- macros in macro bodies
- stringification of parameters [cpp.stringize]
- the concatenation operator [cpp.concat]
- #include MACRO_HERE
- defined() inside macro bodies used in pp-conditions.

Change-Id: Ifdb78041fb6afadf44f939a4bd66ce2832b8601f
Reviewed-by: Roberto Raggi <roberto.raggi@nokia.com>

src/libs/cplusplus/CppDocument.cpp

@@ -198,6 +198,8 @@ protected:
 };
 
 
+#define DO_NOT_DUMP_ALL_PARSER_ERRORS
+
 class DocumentDiagnosticClient : public DiagnosticClient
 {
     enum { MAX_MESSAGE_COUNT = 10 };
@@ -217,8 +219,10 @@ public:
         if (level == Error) {
             ++errorCount;
 
+#ifdef DO_NOT_DUMP_ALL_PARSER_ERRORS
             if (errorCount >= MAX_MESSAGE_COUNT)
                 return; // ignore the error
+#endif // DO_NOT_DUMP_ALL_PARSER_ERRORS
         }
 
         const QString fileName = QString::fromUtf8(fileId->chars(), fileId->size());
@@ -229,6 +233,16 @@ public:
         QString message;
         message.vsprintf(format, ap);
 
+#ifndef DO_NOT_DUMP_ALL_PARSER_ERRORS
+        {
+            const char *levelStr = "Unknown level";
+            if (level == Document::DiagnosticMessage::Warning) levelStr = "Warning";
+            if (level == Document::DiagnosticMessage::Error) levelStr = "Error";
+            if (level == Document::DiagnosticMessage::Fatal) levelStr = "Fatal";
+            qDebug("%s:%u:%u: %s: %s", fileId->chars(), line, column, levelStr, message.toUtf8().constData());
+        }
+#endif // DO_NOT_DUMP_ALL_PARSER_ERRORS
+
         Document::DiagnosticMessage m(convertLevel(level), doc->fileName(),
                                       line, column, message);
         messages->append(m);
@@ -341,10 +355,9 @@ void Document::appendMacro(const Macro &macro)
 
 void Document::addMacroUse(const Macro &macro, unsigned offset, unsigned length,
                            unsigned beginLine,
-                           const QVector<MacroArgumentReference> &actuals, bool inCondition)
+                           const QVector<MacroArgumentReference> &actuals)
 {
     MacroUse use(macro, offset, offset + length, beginLine);
-    use.setInCondition(inCondition);
 
     foreach (const MacroArgumentReference &actual, actuals) {
         const Block arg(actual.position(), actual.position() + actual.length());

src/libs/cplusplus/CppDocument.h

@@ -77,8 +77,7 @@ public:
 
     void appendMacro(const Macro &macro);
     void addMacroUse(const Macro &macro, unsigned offset, unsigned length,
-                     unsigned beginLine, const QVector<MacroArgumentReference> &range,
-                     bool inCondition);
+                     unsigned beginLine, const QVector<MacroArgumentReference> &range);
     void addUndefinedMacroUse(const QByteArray &name, unsigned offset);
 
     Control *control() const;
@@ -247,7 +246,6 @@ public:
     class MacroUse: public Block {
         Macro _macro;
         QVector<Block> _arguments;
-        bool _inCondition;
         unsigned _beginLine;
 
     public:
@@ -255,7 +253,6 @@ public:
                         unsigned begin, unsigned end, unsigned beginLine)
             : Block(begin, end),
               _macro(macro),
-              _inCondition(false),
               _beginLine(beginLine)
         { }
 
@@ -268,9 +265,6 @@ public:
         QVector<Block> arguments() const
         { return _arguments; }
 
-        bool isInCondition() const
-        { return _inCondition; }
-
         unsigned beginLine() const
         { return _beginLine; }
 
@@ -281,9 +275,6 @@ public:
         void addArgument(const Block &block)
         { _arguments.append(block); }
 
-        void setInCondition(bool set)
-        { _inCondition = set; }
-
         friend class Document;
     };
 

src/libs/cplusplus/FastPreprocessor.h

@@ -69,7 +69,6 @@ public:
     virtual void startExpandingMacro(unsigned,
                                      const Macro &,
                                      const QByteArray &,
-                                     bool,
                                      const QVector<MacroArgumentReference> &) {}
 
     virtual void stopExpandingMacro(unsigned, const Macro &) {}

src/libs/cplusplus/Macro.cpp

@@ -92,19 +92,11 @@ QString Macro::decoratedName() const
 QString Macro::toString() const
 {
     QString text = decoratedName();
-    text.append(QString::fromUtf8(_definition.constData(), _definition.size()));
+    text.append(QString::fromUtf8(_definitionText.constData(), _definitionText.size()));
     return text;
 }
 
 QString Macro::toStringWithLineBreaks() const
 {
-    if (_lineBreaks.isEmpty())
-        return toString();
-
-    QString text = decoratedName();
-    QString definitionWithBreaks = QString::fromUtf8(_definition.constData(), _definition.size());
-    foreach (unsigned pos, _lineBreaks)
-        definitionWithBreaks[pos] = '\n';
-    text.append(definitionWithBreaks);
-    return text;
+    return toString();
 }

src/libs/cplusplus/Macro.h

@@ -52,6 +52,8 @@
 #ifndef CPLUSPLUS_PP_MACRO_H
 #define CPLUSPLUS_PP_MACRO_H
 
+#include "PPToken.h"
+
 #include <CPlusPlusForwardDeclarations.h>
 
 #include <QByteArray>
@@ -60,8 +62,12 @@
 
 namespace CPlusPlus {
 
+class Environment;
+
 class CPLUSPLUS_EXPORT Macro
 {
+    typedef Internal::PPToken PPToken;
+
 public:
     Macro();
 
@@ -71,13 +77,16 @@ public:
     void setName(const QByteArray &name)
     { _name = name; }
 
-    QByteArray definition() const
-    { return _definition; }
+    const QByteArray definitionText() const
+    { return _definitionText; }
+
+    const QVector<PPToken> &definitionTokens() const
+    { return _definitionTokens; }
 
-    void setDefinition(const QByteArray &definition)
-    { _definition = definition; }
+    void setDefinition(const QByteArray &definitionText, const QVector<PPToken> &definitionTokens)
+    { _definitionText = definitionText; _definitionTokens = definitionTokens; }
 
-    QVector<QByteArray> formals() const
+    const QVector<QByteArray> &formals() const
     { return _formals; }
 
     void addFormal(const QByteArray &formal)
@@ -125,16 +134,11 @@ public:
     void setVariadic(bool isVariadic)
     { f._variadic = isVariadic; }
 
-    void setLineBreaks(const QList<unsigned> &breaks)
-    { _lineBreaks = breaks; }
-
-    const QList<unsigned> &lineBreaks() const
-    { return _lineBreaks; }
-
     QString toString() const;
     QString toStringWithLineBreaks() const;
 
-// ### private
+private:
+    friend class Environment;
     Macro *_next;
     unsigned _hashcode;
 
@@ -149,10 +153,10 @@ private:
     };
 
     QByteArray _name;
-    QByteArray _definition;
+    QByteArray _definitionText;
+    QVector<PPToken> _definitionTokens;
     QVector<QByteArray> _formals;
     QString _fileName;
-    QList<unsigned> _lineBreaks;
     unsigned _line;
     unsigned _offset;
     unsigned _length;

src/libs/cplusplus/PPToken.cpp

+#include "PPToken.h"
+
+#include <cstring>
+
+using namespace CPlusPlus::Internal;
+
+ByteArrayRef::ByteArrayRef()
+    : m_ref(0)
+    , m_offset(0)
+    , m_length(0)
+{}
+
+bool ByteArrayRef::startsWith(const char *s) const
+{
+    int l = std::strlen(s);
+    if (l > m_length)
+        return false;
+    return !qstrncmp(start(), s, l);
+}
+
+int ByteArrayRef::count(char ch) const
+{
+    if (!m_ref)
+        return 0;
+
+    int num = 0;
+    const char *b = start();
+    const char *i = b + m_length;
+    while (i != b)
+        if (*--i == ch)
+            ++num;
+    return num;
+}
+
+PPToken::PPToken()
+{}
+
+void PPToken::squeeze()
+{
+    if (isValid()) {
+        m_src = m_src.mid(offset, length());
+        m_src.squeeze();
+        offset = 0;
+    }
+}

src/libs/cplusplus/PPToken.h

+#ifndef CPLUSPLUS_INTERNAL_PPTOKEN_H
+#define CPLUSPLUS_INTERNAL_PPTOKEN_H
+
+#include <CPlusPlus.h>
+#include <Token.h>
+
+#include <QByteArray>
+
+namespace CPlusPlus {
+namespace Internal {
+
+class CPLUSPLUS_EXPORT ByteArrayRef
+{
+public:
+    ByteArrayRef();
+
+    ByteArrayRef(const QByteArray *ref)
+        : m_ref(ref)
+        , m_offset(0)
+        , m_length(ref->length())
+    {}
+
+    ByteArrayRef(const QByteArray *ref, int offset, int length)
+        : m_ref(ref)
+        , m_offset(offset)
+        , m_length(length)
+    {
+        Q_ASSERT(ref);
+        Q_ASSERT(offset >= 0);
+        Q_ASSERT(length >= 0);
+        Q_ASSERT(offset + length <= ref->size());
+    }
+
+    inline const char *start() const
+    { return m_ref ? m_ref->constData() + m_offset : 0; }
+
+    inline int length() const
+    { return m_length; }
+
+    inline int size() const
+    { return length(); }
+
+    inline char at(int pos) const
+    { return m_ref && pos >= 0 && pos < m_length ? m_ref->at(m_offset + pos) : '\0'; }
+
+    inline char operator[](int pos) const
+    { return at(pos); }
+
+    QByteArray toByteArray() const
+    { return m_ref ? QByteArray(m_ref->constData() + m_offset, m_length) : QByteArray(); }
+
+    bool operator==(const QByteArray &other) const
+    { return m_ref ? (m_length == other.length() && !qstrncmp(m_ref->constData() + m_offset, other.constData(), m_length)) : false; }
+    bool operator!=(const QByteArray &other) const
+    { return !this->operator==(other); }
+
+    bool startsWith(const char *ch) const;
+
+    int count(char c) const;
+
+private:
+    const QByteArray *m_ref;
+    int m_offset;
+    int m_length;
+};
+
+inline bool operator==(const QByteArray &other, const ByteArrayRef &ref)
+{ return ref == other; }
+
+inline bool operator!=(const QByteArray &other, const ByteArrayRef &ref)
+{ return ref != other; }
+
+class CPLUSPLUS_EXPORT PPToken: public Token
+{
+public:
+    PPToken();
+
+    PPToken(const QByteArray &src)
+        : m_src(src)
+    {}
+
+    void setSource(const QByteArray &src)
+    { m_src = src; }
+
+    const QByteArray &source() const
+    { return m_src; }
+
+    const char *start() const
+    { return m_src.constData() + offset; }
+
+    ByteArrayRef asByteArrayRef() const
+    { return ByteArrayRef(&m_src, offset, length()); }
+
+    bool isValid() const
+    { return !m_src.isEmpty(); }
+
+    void squeeze();
+
+private:
+    QByteArray m_src;
+};
+
+} // namespace Internal
+} // namespace CPlusPlus
+
+#endif // CPLUSPLUS_INTERNAL_PPTOKEN_H

src/libs/cplusplus/PreprocessorClient.h

@@ -75,7 +75,7 @@ public:
 
 public:
   Client();
-  virtual ~Client();
+  virtual ~Client() = 0;
 
   virtual void macroAdded(const Macro &macro) = 0;
 
@@ -85,13 +85,13 @@ public:
   virtual void startExpandingMacro(unsigned offset,
                                    const Macro &macro,
                                    const QByteArray &originalText,
-                                   bool inCondition = false,
                                    const QVector<MacroArgumentReference> &actuals
                                             = QVector<MacroArgumentReference>()) = 0;
 
   virtual void stopExpandingMacro(unsigned offset,
                                   const Macro &macro) = 0;
 
+  /// Start skipping from the given offset.
   virtual void startSkippingBlocks(unsigned offset) = 0;
   virtual void stopSkippingBlocks(unsigned offset) = 0;
 

src/libs/cplusplus/PreprocessorEnvironment.cpp

@@ -150,7 +150,7 @@ void Environment::reset()
     _hash_count = 401;
 }
 
-bool Environment::isBuiltinMacro(const QByteArray &s)
+bool Environment::isBuiltinMacro(const Internal::ByteArrayRef &s)
 {
     if (s.length() != 8)
         return false;
@@ -236,6 +236,22 @@ Macro *Environment::resolve(const QByteArray &name) const
     return it;
 }
 
+Macro *Environment::resolve(const Internal::ByteArrayRef &name) const
+{
+    if (! _macros)
+        return 0;
+
+    Macro *it = _hash[hashCode(name) % _hash_count];
+    for (; it; it = it->_next) {
+        if (it->name() != name)
+            continue;
+        else if (it->isHidden())
+            return 0;
+        else break;
+    }
+    return it;
+}
+
 unsigned Environment::hashCode(const QByteArray &s)
 {
     unsigned hash_value = 0;
@@ -246,6 +262,16 @@ unsigned Environment::hashCode(const QByteArray &s)
     return hash_value;
 }
 
+unsigned Environment::hashCode(const Internal::ByteArrayRef &s)
+{
+    unsigned hash_value = 0;
+
+    for (int i = 0; i < s.length(); ++i)
+        hash_value = (hash_value << 5) - hash_value + s.at(i);
+
+    return hash_value;
+}
+
 void Environment::rehash()
 {
     if (_hash) {

src/libs/cplusplus/PreprocessorEnvironment.h

@@ -53,6 +53,7 @@
 #define CPLUSPLUS_PP_ENVIRONMENT_H
 
 #include "CPlusPlusForwardDeclarations.h"
+#include "PPToken.h"
 
 #include <QList>
 #include <QByteArray>
@@ -78,6 +79,7 @@ public:
     Macro *remove(const QByteArray &name);
 
     Macro *resolve(const QByteArray &name) const;
+    Macro *resolve(const Internal::ByteArrayRef &name) const;
 
     iterator firstMacro() const;
     iterator lastMacro() const;
@@ -85,10 +87,11 @@ public:
     void reset();
     void addMacros(const QList<Macro> &macros);
 
-    static bool isBuiltinMacro(const QByteArray &name);
+    static bool isBuiltinMacro(const Internal::ByteArrayRef &name);
 
 private:
     static unsigned hashCode(const QByteArray &s);
+    static unsigned hashCode(const Internal::ByteArrayRef &s);
     void rehash();
 
 public:

src/libs/cplusplus/cplusplus-lib.pri

@@ -51,9 +51,9 @@ HEADERS += \
     $$PWD/pp.h \
     $$PWD/pp-cctype.h \
     $$PWD/pp-engine.h \
-    $$PWD/pp-macro-expander.h \
     $$PWD/pp-scanner.h \
-    $$PWD/findcdbbreakpoint.h
+    $$PWD/findcdbbreakpoint.h \
+    $$PWD/PPToken.h
 
 SOURCES += \
     $$PWD/SimpleLexer.cpp \
@@ -78,8 +78,8 @@ SOURCES += \
     $$PWD/FastPreprocessor.cpp \
     $$PWD/Macro.cpp \
     $$PWD/pp-engine.cpp \
-    $$PWD/pp-macro-expander.cpp \
     $$PWD/pp-scanner.cpp \
-    $$PWD/findcdbbreakpoint.cpp
+    $$PWD/findcdbbreakpoint.cpp \
+    $$PWD/PPToken.cpp
 
 RESOURCES += $$PWD/cplusplus.qrc

src/libs/cplusplus/cplusplus.pro

 TEMPLATE = lib
 TARGET = CPlusPlus
 
-DEFINES += NDEBUG
-unix:QMAKE_CXXFLAGS_DEBUG += -O2
-
+#DEFINES += NDEBUG
+#unix:QMAKE_CXXFLAGS_DEBUG += -O2
+QMAKE_CXXFLAGS_DEBUG += -ggdb
 include(../../qtcreatorlibrary.pri)
 include(cplusplus-lib.pri)
 include(../languageutils/languageutils.pri)

src/libs/cplusplus/cplusplus.qbs

@@ -149,8 +149,8 @@ DynamicLibrary {
         "pp-cctype.h",
         "pp-engine.cpp",
         "pp-engine.h",
-        "pp-macro-expander.cpp",
-        "pp-macro-expander.h",
+        "PPToken.cpp",
+        "PPToken.h",
         "pp-scanner.cpp",
         "pp-scanner.h",
         "pp.h",

src/libs/cplusplus/pp-engine.h

@@ -52,29 +52,39 @@
 #ifndef CPLUSPLUS_PP_ENGINE_H
 #define CPLUSPLUS_PP_ENGINE_H
 
+#include "PPToken.h"
 #include "PreprocessorClient.h"
-#include "pp-macro-expander.h"
 
+#include <Lexer.h>
 #include <Token.h>
 #include <QVector>
 #include <QBitArray>
+#include <QByteArray>
 
 namespace CPlusPlus {
 
-struct Value;
 class Environment;
 
+namespace Internal {
+class PPToken;
+struct TokenBuffer;
+struct Value;
+}
+
 class CPLUSPLUS_EXPORT Preprocessor
 {
+    typedef Internal::PPToken PPToken;
+    typedef Internal::Value Value;
+
 public:
     Preprocessor(Client *client, Environment *env);
 
     QByteArray operator()(const QString &filename, const QString &source);
-    QByteArray operator()(const QString &filename, const QByteArray &source);
+    QByteArray operator()(const QString &filename, const QByteArray &source, bool noLines = false, bool markGeneratedTokens = true);
 
     void preprocess(const QString &filename,
                     const QByteArray &source,
-                    QByteArray *result);
+                    QByteArray *result, bool noLines, bool markGeneratedTokens, bool inCondition);
 
     bool expandMacros() const;
     void setExpandMacros(bool expandMacros);
@@ -85,126 +95,101 @@ public:
 private:
     enum { MAX_LEVEL = 512 };
 
-    enum PP_DIRECTIVE_TYPE
-    {
-        PP_UNKNOWN_DIRECTIVE,
-        PP_DEFINE,
-        PP_IMPORT,
-        PP_INCLUDE,
-        PP_INCLUDE_NEXT,
-        PP_ELIF,
-        PP_ELSE,
-        PP_ENDIF,
-        PP_IF,
-        PP_IFDEF,
-        PP_IFNDEF,
-        PP_UNDEF
-    };
-
-    typedef const CPlusPlus::Token *TokenIterator;
-
     struct State {
-        QByteArray source;
-        QVector<CPlusPlus::Token> tokens;
-        TokenIterator dot;
-    };
+        State();
 
-    bool markGeneratedTokens(bool markGeneratedTokens, TokenIterator dot = 0);
-    bool markGeneratedTokens(bool markGeneratedTokens, int position, int extraLines=0, bool newline=false);
+        QString m_currentFileName;
 
-    QByteArray expand(const QByteArray &source);
-    void expand(const QByteArray &source, QByteArray *result);
-    void expand(const char *first, const char *last, QByteArray *result);
-    void expandBuiltinMacro(TokenIterator identifierToken,
-                            const QByteArray &spell);
-    void expandObjectLikeMacro(TokenIterator identifierToken,
-                               const QByteArray &spell,
-                               Macro *m, QByteArray *result);
-    void expandFunctionLikeMacro(TokenIterator identifierToken, Macro *m,
-                                 const QVector<MacroArgumentReference> &actuals);
+        QByteArray m_source;
+        Lexer *m_lexer;
+        QBitArray m_skipping;
+        QBitArray m_trueTest;
+        int m_ifLevel;
+        Internal::TokenBuffer *m_tokenBuffer;
+        bool m_inPreprocessorDirective;
 
-    void resetIfLevel();
-    bool testIfLevel();
-    int skipping() const;
+        QByteArray *m_result;
+        bool m_markGeneratedTokens;
 
-    PP_DIRECTIVE_TYPE classifyDirective(const QByteArray &directive) const;
+        bool m_noLines;
+        bool m_inCondition;
+        bool m_inDefine;
+    };
+
+    void handleDefined(PPToken *tk);