Commit 60db5736 authored by Erik Verbruggen's avatar Erik Verbruggen

[C++] Rewrite of the preprocessor.

This rewrite fixes a couple of issues with the pre-processor. It now
supports:
- macros in macro bodies
- stringification of parameters [cpp.stringize]
- the concatenation operator [cpp.concat]
- #include MACRO_HERE
- defined() inside macro bodies used in pp-conditions.

Change-Id: Ifdb78041fb6afadf44f939a4bd66ce2832b8601f
Reviewed-by: default avatarRoberto Raggi <roberto.raggi@nokia.com>
parent 159058d9
......@@ -198,6 +198,8 @@ protected:
};
#define DO_NOT_DUMP_ALL_PARSER_ERRORS
class DocumentDiagnosticClient : public DiagnosticClient
{
enum { MAX_MESSAGE_COUNT = 10 };
......@@ -217,8 +219,10 @@ public:
if (level == Error) {
++errorCount;
#ifdef DO_NOT_DUMP_ALL_PARSER_ERRORS
if (errorCount >= MAX_MESSAGE_COUNT)
return; // ignore the error
#endif // DO_NOT_DUMP_ALL_PARSER_ERRORS
}
const QString fileName = QString::fromUtf8(fileId->chars(), fileId->size());
......@@ -229,6 +233,16 @@ public:
QString message;
message.vsprintf(format, ap);
#ifndef DO_NOT_DUMP_ALL_PARSER_ERRORS
{
const char *levelStr = "Unknown level";
if (level == Document::DiagnosticMessage::Warning) levelStr = "Warning";
if (level == Document::DiagnosticMessage::Error) levelStr = "Error";
if (level == Document::DiagnosticMessage::Fatal) levelStr = "Fatal";
qDebug("%s:%u:%u: %s: %s", fileId->chars(), line, column, levelStr, message.toUtf8().constData());
}
#endif // DO_NOT_DUMP_ALL_PARSER_ERRORS
Document::DiagnosticMessage m(convertLevel(level), doc->fileName(),
line, column, message);
messages->append(m);
......@@ -341,10 +355,9 @@ void Document::appendMacro(const Macro &macro)
void Document::addMacroUse(const Macro &macro, unsigned offset, unsigned length,
unsigned beginLine,
const QVector<MacroArgumentReference> &actuals, bool inCondition)
const QVector<MacroArgumentReference> &actuals)
{
MacroUse use(macro, offset, offset + length, beginLine);
use.setInCondition(inCondition);
foreach (const MacroArgumentReference &actual, actuals) {
const Block arg(actual.position(), actual.position() + actual.length());
......
......@@ -77,8 +77,7 @@ public:
void appendMacro(const Macro &macro);
void addMacroUse(const Macro &macro, unsigned offset, unsigned length,
unsigned beginLine, const QVector<MacroArgumentReference> &range,
bool inCondition);
unsigned beginLine, const QVector<MacroArgumentReference> &range);
void addUndefinedMacroUse(const QByteArray &name, unsigned offset);
Control *control() const;
......@@ -247,7 +246,6 @@ public:
class MacroUse: public Block {
Macro _macro;
QVector<Block> _arguments;
bool _inCondition;
unsigned _beginLine;
public:
......@@ -255,7 +253,6 @@ public:
unsigned begin, unsigned end, unsigned beginLine)
: Block(begin, end),
_macro(macro),
_inCondition(false),
_beginLine(beginLine)
{ }
......@@ -268,9 +265,6 @@ public:
QVector<Block> arguments() const
{ return _arguments; }
bool isInCondition() const
{ return _inCondition; }
unsigned beginLine() const
{ return _beginLine; }
......@@ -281,9 +275,6 @@ public:
void addArgument(const Block &block)
{ _arguments.append(block); }
void setInCondition(bool set)
{ _inCondition = set; }
friend class Document;
};
......
......@@ -69,7 +69,6 @@ public:
virtual void startExpandingMacro(unsigned,
const Macro &,
const QByteArray &,
bool,
const QVector<MacroArgumentReference> &) {}
virtual void stopExpandingMacro(unsigned, const Macro &) {}
......
......@@ -92,19 +92,11 @@ QString Macro::decoratedName() const
QString Macro::toString() const
{
QString text = decoratedName();
text.append(QString::fromUtf8(_definition.constData(), _definition.size()));
text.append(QString::fromUtf8(_definitionText.constData(), _definitionText.size()));
return text;
}
QString Macro::toStringWithLineBreaks() const
{
if (_lineBreaks.isEmpty())
return toString();
QString text = decoratedName();
QString definitionWithBreaks = QString::fromUtf8(_definition.constData(), _definition.size());
foreach (unsigned pos, _lineBreaks)
definitionWithBreaks[pos] = '\n';
text.append(definitionWithBreaks);
return text;
return toString();
}
......@@ -52,6 +52,8 @@
#ifndef CPLUSPLUS_PP_MACRO_H
#define CPLUSPLUS_PP_MACRO_H
#include "PPToken.h"
#include <CPlusPlusForwardDeclarations.h>
#include <QByteArray>
......@@ -60,8 +62,12 @@
namespace CPlusPlus {
class Environment;
class CPLUSPLUS_EXPORT Macro
{
typedef Internal::PPToken PPToken;
public:
Macro();
......@@ -71,13 +77,16 @@ public:
void setName(const QByteArray &name)
{ _name = name; }
QByteArray definition() const
{ return _definition; }
const QByteArray definitionText() const
{ return _definitionText; }
const QVector<PPToken> &definitionTokens() const
{ return _definitionTokens; }
void setDefinition(const QByteArray &definition)
{ _definition = definition; }
void setDefinition(const QByteArray &definitionText, const QVector<PPToken> &definitionTokens)
{ _definitionText = definitionText; _definitionTokens = definitionTokens; }
QVector<QByteArray> formals() const
const QVector<QByteArray> &formals() const
{ return _formals; }
void addFormal(const QByteArray &formal)
......@@ -125,16 +134,11 @@ public:
void setVariadic(bool isVariadic)
{ f._variadic = isVariadic; }
void setLineBreaks(const QList<unsigned> &breaks)
{ _lineBreaks = breaks; }
const QList<unsigned> &lineBreaks() const
{ return _lineBreaks; }
QString toString() const;
QString toStringWithLineBreaks() const;
// ### private
private:
friend class Environment;
Macro *_next;
unsigned _hashcode;
......@@ -149,10 +153,10 @@ private:
};
QByteArray _name;
QByteArray _definition;
QByteArray _definitionText;
QVector<PPToken> _definitionTokens;
QVector<QByteArray> _formals;
QString _fileName;
QList<unsigned> _lineBreaks;
unsigned _line;
unsigned _offset;
unsigned _length;
......
#include "PPToken.h"
#include <cstring>
using namespace CPlusPlus::Internal;
ByteArrayRef::ByteArrayRef()
: m_ref(0)
, m_offset(0)
, m_length(0)
{}
bool ByteArrayRef::startsWith(const char *s) const
{
int l = std::strlen(s);
if (l > m_length)
return false;
return !qstrncmp(start(), s, l);
}
int ByteArrayRef::count(char ch) const
{
if (!m_ref)
return 0;
int num = 0;
const char *b = start();
const char *i = b + m_length;
while (i != b)
if (*--i == ch)
++num;
return num;
}
PPToken::PPToken()
{}
void PPToken::squeeze()
{
if (isValid()) {
m_src = m_src.mid(offset, length());
m_src.squeeze();
offset = 0;
}
}
#ifndef CPLUSPLUS_INTERNAL_PPTOKEN_H
#define CPLUSPLUS_INTERNAL_PPTOKEN_H
#include <CPlusPlus.h>
#include <Token.h>
#include <QByteArray>
namespace CPlusPlus {
namespace Internal {
class CPLUSPLUS_EXPORT ByteArrayRef
{
public:
ByteArrayRef();
ByteArrayRef(const QByteArray *ref)
: m_ref(ref)
, m_offset(0)
, m_length(ref->length())
{}
ByteArrayRef(const QByteArray *ref, int offset, int length)
: m_ref(ref)
, m_offset(offset)
, m_length(length)
{
Q_ASSERT(ref);
Q_ASSERT(offset >= 0);
Q_ASSERT(length >= 0);
Q_ASSERT(offset + length <= ref->size());
}
inline const char *start() const
{ return m_ref ? m_ref->constData() + m_offset : 0; }
inline int length() const
{ return m_length; }
inline int size() const
{ return length(); }
inline char at(int pos) const
{ return m_ref && pos >= 0 && pos < m_length ? m_ref->at(m_offset + pos) : '\0'; }
inline char operator[](int pos) const
{ return at(pos); }
QByteArray toByteArray() const
{ return m_ref ? QByteArray(m_ref->constData() + m_offset, m_length) : QByteArray(); }
bool operator==(const QByteArray &other) const
{ return m_ref ? (m_length == other.length() && !qstrncmp(m_ref->constData() + m_offset, other.constData(), m_length)) : false; }
bool operator!=(const QByteArray &other) const
{ return !this->operator==(other); }
bool startsWith(const char *ch) const;
int count(char c) const;
private:
const QByteArray *m_ref;
int m_offset;
int m_length;
};
inline bool operator==(const QByteArray &other, const ByteArrayRef &ref)
{ return ref == other; }
inline bool operator!=(const QByteArray &other, const ByteArrayRef &ref)
{ return ref != other; }
class CPLUSPLUS_EXPORT PPToken: public Token
{
public:
PPToken();
PPToken(const QByteArray &src)
: m_src(src)
{}
void setSource(const QByteArray &src)
{ m_src = src; }
const QByteArray &source() const
{ return m_src; }
const char *start() const
{ return m_src.constData() + offset; }
ByteArrayRef asByteArrayRef() const
{ return ByteArrayRef(&m_src, offset, length()); }
bool isValid() const
{ return !m_src.isEmpty(); }
void squeeze();
private:
QByteArray m_src;
};
} // namespace Internal
} // namespace CPlusPlus
#endif // CPLUSPLUS_INTERNAL_PPTOKEN_H
......@@ -75,7 +75,7 @@ public:
public:
Client();
virtual ~Client();
virtual ~Client() = 0;
virtual void macroAdded(const Macro &macro) = 0;
......@@ -85,13 +85,13 @@ public:
virtual void startExpandingMacro(unsigned offset,
const Macro &macro,
const QByteArray &originalText,
bool inCondition = false,
const QVector<MacroArgumentReference> &actuals
= QVector<MacroArgumentReference>()) = 0;
virtual void stopExpandingMacro(unsigned offset,
const Macro &macro) = 0;
/// Start skipping from the given offset.
virtual void startSkippingBlocks(unsigned offset) = 0;
virtual void stopSkippingBlocks(unsigned offset) = 0;
......
......@@ -150,7 +150,7 @@ void Environment::reset()
_hash_count = 401;
}
bool Environment::isBuiltinMacro(const QByteArray &s)
bool Environment::isBuiltinMacro(const Internal::ByteArrayRef &s)
{
if (s.length() != 8)
return false;
......@@ -236,6 +236,22 @@ Macro *Environment::resolve(const QByteArray &name) const
return it;
}
Macro *Environment::resolve(const Internal::ByteArrayRef &name) const
{
if (! _macros)
return 0;
Macro *it = _hash[hashCode(name) % _hash_count];
for (; it; it = it->_next) {
if (it->name() != name)
continue;
else if (it->isHidden())
return 0;
else break;
}
return it;
}
unsigned Environment::hashCode(const QByteArray &s)
{
unsigned hash_value = 0;
......@@ -246,6 +262,16 @@ unsigned Environment::hashCode(const QByteArray &s)
return hash_value;
}
unsigned Environment::hashCode(const Internal::ByteArrayRef &s)
{
unsigned hash_value = 0;
for (int i = 0; i < s.length(); ++i)
hash_value = (hash_value << 5) - hash_value + s.at(i);
return hash_value;
}
void Environment::rehash()
{
if (_hash) {
......
......@@ -53,6 +53,7 @@
#define CPLUSPLUS_PP_ENVIRONMENT_H
#include "CPlusPlusForwardDeclarations.h"
#include "PPToken.h"
#include <QList>
#include <QByteArray>
......@@ -78,6 +79,7 @@ public:
Macro *remove(const QByteArray &name);
Macro *resolve(const QByteArray &name) const;
Macro *resolve(const Internal::ByteArrayRef &name) const;
iterator firstMacro() const;
iterator lastMacro() const;
......@@ -85,10 +87,11 @@ public:
void reset();
void addMacros(const QList<Macro> &macros);
static bool isBuiltinMacro(const QByteArray &name);
static bool isBuiltinMacro(const Internal::ByteArrayRef &name);
private:
static unsigned hashCode(const QByteArray &s);
static unsigned hashCode(const Internal::ByteArrayRef &s);
void rehash();
public:
......
......@@ -51,9 +51,9 @@ HEADERS += \
$$PWD/pp.h \
$$PWD/pp-cctype.h \
$$PWD/pp-engine.h \
$$PWD/pp-macro-expander.h \
$$PWD/pp-scanner.h \
$$PWD/findcdbbreakpoint.h
$$PWD/findcdbbreakpoint.h \
$$PWD/PPToken.h
SOURCES += \
$$PWD/SimpleLexer.cpp \
......@@ -78,8 +78,8 @@ SOURCES += \
$$PWD/FastPreprocessor.cpp \
$$PWD/Macro.cpp \
$$PWD/pp-engine.cpp \
$$PWD/pp-macro-expander.cpp \
$$PWD/pp-scanner.cpp \
$$PWD/findcdbbreakpoint.cpp
$$PWD/findcdbbreakpoint.cpp \
$$PWD/PPToken.cpp
RESOURCES += $$PWD/cplusplus.qrc
TEMPLATE = lib
TARGET = CPlusPlus
DEFINES += NDEBUG
unix:QMAKE_CXXFLAGS_DEBUG += -O2
#DEFINES += NDEBUG
#unix:QMAKE_CXXFLAGS_DEBUG += -O2
QMAKE_CXXFLAGS_DEBUG += -ggdb
include(../../qtcreatorlibrary.pri)
include(cplusplus-lib.pri)
include(../languageutils/languageutils.pri)
......@@ -149,8 +149,8 @@ DynamicLibrary {
"pp-cctype.h",
"pp-engine.cpp",
"pp-engine.h",
"pp-macro-expander.cpp",
"pp-macro-expander.h",
"PPToken.cpp",
"PPToken.h",
"pp-scanner.cpp",
"pp-scanner.h",
"pp.h",
......
This diff is collapsed.
......@@ -52,29 +52,39 @@
#ifndef CPLUSPLUS_PP_ENGINE_H
#define CPLUSPLUS_PP_ENGINE_H
#include "PPToken.h"
#include "PreprocessorClient.h"
#include "pp-macro-expander.h"
#include <Lexer.h>
#include <Token.h>
#include <QVector>
#include <QBitArray>
#include <QByteArray>
namespace CPlusPlus {
struct Value;
class Environment;
namespace Internal {
class PPToken;
struct TokenBuffer;
struct Value;
}
class CPLUSPLUS_EXPORT Preprocessor
{
typedef Internal::PPToken PPToken;
typedef Internal::Value Value;
public:
Preprocessor(Client *client, Environment *env);
QByteArray operator()(const QString &filename, const QString &source);
QByteArray operator()(const QString &filename, const QByteArray &source);
QByteArray operator()(const QString &filename, const QByteArray &source, bool noLines = false, bool markGeneratedTokens = true);
void preprocess(const QString &filename,
const QByteArray &source,
QByteArray *result);
QByteArray *result, bool noLines, bool markGeneratedTokens, bool inCondition);
bool expandMacros() const;
void setExpandMacros(bool expandMacros);
......@@ -85,126 +95,101 @@ public:
private:
enum { MAX_LEVEL = 512 };
enum PP_DIRECTIVE_TYPE
{
PP_UNKNOWN_DIRECTIVE,
PP_DEFINE,
PP_IMPORT,
PP_INCLUDE,
PP_INCLUDE_NEXT,
PP_ELIF,
PP_ELSE,
PP_ENDIF,
PP_IF,
PP_IFDEF,
PP_IFNDEF,
PP_UNDEF
};
typedef const CPlusPlus::Token *TokenIterator;
struct State {
QByteArray source;
QVector<CPlusPlus::Token> tokens;
TokenIterator dot;
};
State();
bool markGeneratedTokens(bool markGeneratedTokens, TokenIterator dot = 0);
bool markGeneratedTokens(bool markGeneratedTokens, int position, int extraLines=0, bool newline=false);
QString m_currentFileName;
QByteArray expand(const QByteArray &source);
void expand(const QByteArray &source, QByteArray *result);
void expand(const char *first, const char *last, QByteArray *result);
void expandBuiltinMacro(TokenIterator identifierToken,
const QByteArray &spell);
void expandObjectLikeMacro(TokenIterator identifierToken,
const QByteArray &spell,
Macro *m, QByteArray *result);
void expandFunctionLikeMacro(TokenIterator identifierToken, Macro *m,
const QVector<MacroArgumentReference> &actuals);
QByteArray m_source;
Lexer *m_lexer;
QBitArray m_skipping;
QBitArray m_trueTest;
int m_ifLevel;
Internal::TokenBuffer *m_tokenBuffer;
bool m_inPreprocessorDirective;
void resetIfLevel();