Some work on the new preprocessor.

tests/manual/plain-cplusplus/Preprocessor.cpp

+/**************************************************************************
+**
+** This file is part of Qt Creator
+**
+** Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
+**
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** Commercial Usage
+**
+** Licensees holding valid Qt Commercial licenses may use this file in
+** accordance with the Qt Commercial License Agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Nokia.
+**
+** GNU Lesser General Public License Usage
+**
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** If you are unsure which license is appropriate for your use, please
+** contact the sales department at http://qt.nokia.com/contact.
+**
+**************************************************************************/
+
+#include "Preprocessor.h"
+#include "Lexer.h"
+#include <list>
+#include <iostream>
+#include <cassert>
+
+using namespace CPlusPlus;
+
+std::ostream &operator << (std::ostream &out, const StringRef &s)
+{
+    out.write(s.text(), s.size());
+    return out;
+}
+
+struct Preprocessor::TokenBuffer
+{
+    std::list<Token> tokens;
+    const Macro *macro;
+    TokenBuffer *next;
+
+    template <typename _Iterator>
+    TokenBuffer(_Iterator firstToken, _Iterator lastToken, const Macro *macro, TokenBuffer *next)
+        : tokens(firstToken, lastToken), macro(macro), next(next) {}
+};
+
+Lexer *Preprocessor::switchLexer(Lexer *lex)
+{
+    Lexer *previousLexer = _lexer;
+    _lexer = lex;
+    return previousLexer;
+}
+
+StringRef Preprocessor::switchSource(const StringRef &source)
+{
+    StringRef previousSource = _source;
+    _source = source;
+    return previousSource;
+}
+
+const Preprocessor::Macro *Preprocessor::resolveMacro(const StringRef &name) const
+{
+    std::map<StringRef, Macro>::const_iterator it = macros.find(name);
+    if (it != macros.end()) {
+        const Macro *m = &it->second;
+        for (TokenBuffer *r = _tokenBuffer; r; r = r->next) {
+            if (r->macro == m)
+                return 0;
+        }
+        return m;
+    }
+
+    return 0;
+}
+
+void Preprocessor::collectActualArguments(Token *tk, std::vector<std::vector<Token> > *actuals)
+{
+    lex(tk);
+
+    assert(tk->is(T_LPAREN));
+
+    lex(tk);
+
+    std::vector<Token> tokens;
+    scanActualArgument(tk, &tokens);
+
+    actuals->push_back(tokens);
+
+    while (tk->is(T_COMMA)) {
+        lex(tk);
+
+        std::vector<Token> tokens;
+        scanActualArgument(tk, &tokens);
+        actuals->push_back(tokens);
+    }
+
+    assert(tk->is(T_RPAREN));
+    lex(tk);
+}
+
+void Preprocessor::scanActualArgument(Token *tk, std::vector<Token> *tokens)
+{
+    int count = 0;
+
+    while (tk->isNot(T_EOF_SYMBOL)) {
+        if (tk->is(T_LPAREN))
+            ++count;
+
+        else if (tk->is(T_RPAREN)) {
+            if (! count)
+                break;
+
+            --count;
+        }
+
+        else if (! count && tk->is(T_COMMA))
+            break;
+
+        tokens->push_back(*tk);
+        lex(tk);
+    }
+}
+
+void Preprocessor::lex(Token *tk)
+{
+_Lagain:
+    if (_tokenBuffer) {
+        if (_tokenBuffer->tokens.empty()) {
+            TokenBuffer *r = _tokenBuffer;
+            _tokenBuffer = _tokenBuffer->next;
+            delete r;
+            goto _Lagain;
+        }
+        *tk = _tokenBuffer->tokens.front();
+        _tokenBuffer->tokens.pop_front();
+    } else {
+        _lexer->scan(tk);
+    }
+
+_Lclassify:
+    if (! inPreprocessorDirective) {
+        if (tk->newline() && tk->is(T_POUND)) {
+            handlePreprocessorDirective(tk);
+            goto _Lclassify;
+
+        } else if (tk->is(T_IDENTIFIER)) {
+            const StringRef id = asStringRef(*tk);
+
+            if (const Macro *macro = resolveMacro(id)) {
+                std::vector<Token> body = macro->body;
+
+                if (macro->isFunctionLike) {
+                    std::vector<std::vector<Token> > actuals;
+                    collectActualArguments(tk, &actuals);
+
+                    std::vector<Token> expanded;
+                    for (size_t i = 0; i < body.size(); ++i) {
+                        const Token &token = body[i];
+
+                        if (token.isNot(T_IDENTIFIER))
+                            expanded.push_back(token);
+                        else {
+                            const StringRef id = asStringRef(token);
+                            size_t j = 0;
+                            for (; j < macro->formals.size(); ++j) {
+                                if (macro->formals[j] == id) {
+                                    expanded.insert(expanded.end(), actuals[j].begin(), actuals[j].end());
+                                    break;
+                                }
+                            }
+
+                            if (j == macro->formals.size())
+                                expanded.push_back(token);
+                        }
+                    }
+
+                    const Token currentTokenBuffer[] = { *tk };
+                    _tokenBuffer = new TokenBuffer(currentTokenBuffer, currentTokenBuffer + 1,
+                                                   /*macro */ 0, _tokenBuffer);
+
+                    body = expanded;
+                }
+
+                _tokenBuffer = new TokenBuffer(body.begin(), body.end(),
+                                               macro, _tokenBuffer);
+                goto _Lagain;
+            }
+        }
+    }
+}
+
+void Preprocessor::handlePreprocessorDirective(Token *tk)
+{
+    inPreprocessorDirective = true;
+
+    lex(tk); // scan the directive
+
+    if (tk->newline() && ! tk->joined())
+        return; // nothing to do.
+
+    const StringRef ppDefine("define", 6);
+
+    if (tk->is(T_IDENTIFIER)) {
+        const StringRef directive = asStringRef(*tk);
+
+        if (directive == ppDefine)
+            handleDefineDirective(tk);
+        else
+            skipPreprocesorDirective(tk);
+    }
+
+    inPreprocessorDirective = false;
+}
+
+bool Preprocessor::isValidToken(const Token &tk) const
+{
+    if (tk.isNot(T_EOF_SYMBOL) && (! tk.newline() || tk.joined()))
+        return true;
+
+    return false;
+}
+
+void Preprocessor::handleDefineDirective(Token *tk)
+{
+    lex(tk);
+
+    if (tk->is(T_IDENTIFIER)) {
+        const StringRef macroName = asStringRef(*tk);
+        Macro macro;
+
+        lex(tk);
+
+        if (isValidToken(*tk) && tk->is(T_LPAREN) && ! tk->whitespace()) {
+            macro.isFunctionLike = true;
+
+            lex(tk); // skip `('
+
+            if (isValidToken(*tk) && tk->is(T_IDENTIFIER)) {
+                macro.formals.push_back(asStringRef(*tk));
+
+                lex(tk);
+
+                while (isValidToken(*tk) && tk->is(T_COMMA)) {
+                    lex(tk);
+
+                    if (isValidToken(*tk) && tk->is(T_IDENTIFIER)) {
+                        macro.formals.push_back(asStringRef(*tk));
+                        lex(tk);
+                    }
+                }
+            }
+
+            if (isValidToken(*tk) && tk->is(T_RPAREN))
+                lex(tk); // skip `)'
+        }
+
+        while (isValidToken(*tk)) {
+            macro.body.push_back(*tk);
+            lex(tk);
+        }
+
+        macros.insert(std::make_pair(macroName, macro));
+    } else {
+        skipPreprocesorDirective(tk);
+    }
+}
+
+void Preprocessor::skipPreprocesorDirective(Token *tk)
+{
+    do {
+        lex(tk);
+    } while (isValidToken(*tk));
+}
+
+StringRef Preprocessor::asStringRef(const Token &tk) const
+{ return StringRef(_source.begin() + tk.begin(), tk.length()); }
+
+Preprocessor::Preprocessor(std::ostream &out)
+    : out(out), _lexer(0), inPreprocessorDirective(false)
+{ }
+
+void Preprocessor::operator()(const char *source, unsigned size, const StringRef &currentFileName)
+{
+    _currentFileName = currentFileName;
+    run(source, size);
+}
+
+void Preprocessor::run(const char *source, unsigned size)
+{
+    _tokenBuffer = 0;
+
+    const StringRef previousSource = switchSource(StringRef(source, size));
+    Lexer thisLexer(source, source + size);
+    thisLexer.setScanKeywords(false);
+    Lexer *previousLexer = switchLexer(&thisLexer);
+    inPreprocessorDirective = false;
+
+    Token tk;
+    unsigned lineno = 0;
+    do {
+        lex(&tk);
+
+        if (lineno != tk.lineno) {
+            if (lineno > tk.lineno || tk.lineno - lineno > 3)
+                out << std::endl << "#line " << tk.lineno << " \"" << _currentFileName << "\"" << std::endl;
+
+            else {
+                for (unsigned i = lineno; i < tk.lineno; ++i)
+                    out << std::endl;
+            }
+
+            lineno = tk.lineno;
+
+        } else {
+            if (tk.newline())
+                out << std::endl;
+
+            if (tk.whitespace())
+                out << ' ';
+        }
+
+        out << asStringRef(tk);
+        lineno = tk.lineno;
+    } while (tk.isNot(T_EOF_SYMBOL));
+
+    out << std::endl;
+
+    switchLexer(previousLexer);
+    switchSource(previousSource);
+}
+
+
+
+

tests/manual/plain-cplusplus/Preprocessor.h

+/**************************************************************************
+**
+** This file is part of Qt Creator
+**
+** Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
+**
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** Commercial Usage
+**
+** Licensees holding valid Qt Commercial licenses may use this file in
+** accordance with the Qt Commercial License Agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Nokia.
+**
+** GNU Lesser General Public License Usage
+**
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** If you are unsure which license is appropriate for your use, please
+** contact the sales department at http://qt.nokia.com/contact.
+**
+**************************************************************************/
+#ifndef CPLUSPLUS_PREPROCESSOR_H
+#define CPLUSPLUS_PREPROCESSOR_H
+
+#include <CPlusPlusForwardDeclarations.h>
+#include <iosfwd>
+#include <vector>
+#include <map>
+
+namespace CPlusPlus {
+
+class Lexer;
+class Token;
+
+class CPLUSPLUS_EXPORT StringRef
+{
+    const char *_text;
+    unsigned _size;
+
+public:
+    typedef const char *iterator;
+    typedef const char *const_iterator;
+
+    StringRef()
+        : _text(0), _size(0) {}
+
+    StringRef(const char *text, unsigned size)
+        : _text(text), _size(size) {}
+
+    StringRef(const char *text)
+        : _text(text), _size(strlen(text)) {}
+
+    inline const char *text() const { return _text; }
+    inline unsigned size() const { return _size; }
+
+    inline const_iterator begin() const { return _text; }
+    inline const_iterator end() const { return _text + _size; }
+
+    bool operator == (const StringRef &other) const
+    {
+        if (_size == other._size)
+            return _text == other._text || ! strncmp(_text, other._text, _size);
+
+        return false;
+    }
+
+    bool operator != (const StringRef &other) const
+    { return ! operator == (other); }
+
+    bool operator < (const StringRef &other) const
+    { return std::lexicographical_compare(begin(), end(), other.begin(), other.end()); }
+};
+
+class CPLUSPLUS_EXPORT Preprocessor
+{
+public:
+    Preprocessor(std::ostream &out);
+
+    void operator()(const char *source, unsigned size, const StringRef &currentFileName);
+
+private:
+    struct Macro
+    {
+        Macro(): isFunctionLike(false), isVariadic(false) {}
+
+        std::vector<StringRef> formals;
+        std::vector<Token> body;
+        bool isFunctionLike: 1;
+        bool isVariadic: 1;
+    };
+
+    void run(const char *source, unsigned size);
+
+    Lexer *switchLexer(Lexer *lex);
+    StringRef switchSource(const StringRef &source);
+
+    const Macro *resolveMacro(const StringRef &name) const;
+
+    StringRef asStringRef(const Token &tk) const;
+    void lex(Token *tk);
+    bool isValidToken(const Token &tk) const;
+
+    void handlePreprocessorDirective(Token *tk);
+    void handleDefineDirective(Token *tk);
+    void skipPreprocesorDirective(Token *tk);
+
+    void collectActualArguments(Token *tk, std::vector<std::vector<Token> > *actuals);
+    void scanActualArgument(Token *tk, std::vector<Token> *tokens);
+
+private:
+    struct TokenBuffer;
+
+    std::ostream &out;
+    StringRef _currentFileName;
+    Lexer *_lexer;
+    StringRef _source;
+    TokenBuffer *_tokenBuffer;
+    bool inPreprocessorDirective: 1;
+    std::map<StringRef, Macro> macros;
+};
+
+} // end of namespace CPlusPlus
+
+CPLUSPLUS_EXPORT std::ostream &operator << (std::ostream &out, const CPlusPlus::StringRef &s);
+
+#endif // CPLUSPLUS_PREPROCESSOR_H
+

tests/manual/plain-cplusplus/plain-cplusplus.pro

@@ -9,6 +9,8 @@ macx {
 include(../../../src/shared/cplusplus/cplusplus.pri)
 
 # Input
+HEADERS += Preprocessor.h
+SOURCES += Preprocessor.cpp
 SOURCES += main.cpp
 
 unix {