Commit c09a8e4f authored by Roberto Raggi's avatar Roberto Raggi
Browse files

Some work on the new preprocessor.

parent 7518164a
/**************************************************************************
**
** This file is part of Qt Creator
**
** Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
**
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** Commercial Usage
**
** Licensees holding valid Qt Commercial licenses may use this file in
** accordance with the Qt Commercial License Agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Nokia.
**
** GNU Lesser General Public License Usage
**
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** If you are unsure which license is appropriate for your use, please
** contact the sales department at http://qt.nokia.com/contact.
**
**************************************************************************/
#include "Preprocessor.h"
#include "Lexer.h"
#include <list>
#include <iostream>
#include <cassert>
using namespace CPlusPlus;
std::ostream &operator << (std::ostream &out, const StringRef &s)
{
out.write(s.text(), s.size());
return out;
}
struct Preprocessor::TokenBuffer
{
std::list<Token> tokens;
const Macro *macro;
TokenBuffer *next;
template <typename _Iterator>
TokenBuffer(_Iterator firstToken, _Iterator lastToken, const Macro *macro, TokenBuffer *next)
: tokens(firstToken, lastToken), macro(macro), next(next) {}
};
Lexer *Preprocessor::switchLexer(Lexer *lex)
{
Lexer *previousLexer = _lexer;
_lexer = lex;
return previousLexer;
}
StringRef Preprocessor::switchSource(const StringRef &source)
{
StringRef previousSource = _source;
_source = source;
return previousSource;
}
const Preprocessor::Macro *Preprocessor::resolveMacro(const StringRef &name) const
{
std::map<StringRef, Macro>::const_iterator it = macros.find(name);
if (it != macros.end()) {
const Macro *m = &it->second;
for (TokenBuffer *r = _tokenBuffer; r; r = r->next) {
if (r->macro == m)
return 0;
}
return m;
}
return 0;
}
void Preprocessor::collectActualArguments(Token *tk, std::vector<std::vector<Token> > *actuals)
{
lex(tk);
assert(tk->is(T_LPAREN));
lex(tk);
std::vector<Token> tokens;
scanActualArgument(tk, &tokens);
actuals->push_back(tokens);
while (tk->is(T_COMMA)) {
lex(tk);
std::vector<Token> tokens;
scanActualArgument(tk, &tokens);
actuals->push_back(tokens);
}
assert(tk->is(T_RPAREN));
lex(tk);
}
void Preprocessor::scanActualArgument(Token *tk, std::vector<Token> *tokens)
{
int count = 0;
while (tk->isNot(T_EOF_SYMBOL)) {
if (tk->is(T_LPAREN))
++count;
else if (tk->is(T_RPAREN)) {
if (! count)
break;
--count;
}
else if (! count && tk->is(T_COMMA))
break;
tokens->push_back(*tk);
lex(tk);
}
}
void Preprocessor::lex(Token *tk)
{
_Lagain:
if (_tokenBuffer) {
if (_tokenBuffer->tokens.empty()) {
TokenBuffer *r = _tokenBuffer;
_tokenBuffer = _tokenBuffer->next;
delete r;
goto _Lagain;
}
*tk = _tokenBuffer->tokens.front();
_tokenBuffer->tokens.pop_front();
} else {
_lexer->scan(tk);
}
_Lclassify:
if (! inPreprocessorDirective) {
if (tk->newline() && tk->is(T_POUND)) {
handlePreprocessorDirective(tk);
goto _Lclassify;
} else if (tk->is(T_IDENTIFIER)) {
const StringRef id = asStringRef(*tk);
if (const Macro *macro = resolveMacro(id)) {
std::vector<Token> body = macro->body;
if (macro->isFunctionLike) {
std::vector<std::vector<Token> > actuals;
collectActualArguments(tk, &actuals);
std::vector<Token> expanded;
for (size_t i = 0; i < body.size(); ++i) {
const Token &token = body[i];
if (token.isNot(T_IDENTIFIER))
expanded.push_back(token);
else {
const StringRef id = asStringRef(token);
size_t j = 0;
for (; j < macro->formals.size(); ++j) {
if (macro->formals[j] == id) {
expanded.insert(expanded.end(), actuals[j].begin(), actuals[j].end());
break;
}
}
if (j == macro->formals.size())
expanded.push_back(token);
}
}
const Token currentTokenBuffer[] = { *tk };
_tokenBuffer = new TokenBuffer(currentTokenBuffer, currentTokenBuffer + 1,
/*macro */ 0, _tokenBuffer);
body = expanded;
}
_tokenBuffer = new TokenBuffer(body.begin(), body.end(),
macro, _tokenBuffer);
goto _Lagain;
}
}
}
}
void Preprocessor::handlePreprocessorDirective(Token *tk)
{
inPreprocessorDirective = true;
lex(tk); // scan the directive
if (tk->newline() && ! tk->joined())
return; // nothing to do.
const StringRef ppDefine("define", 6);
if (tk->is(T_IDENTIFIER)) {
const StringRef directive = asStringRef(*tk);
if (directive == ppDefine)
handleDefineDirective(tk);
else
skipPreprocesorDirective(tk);
}
inPreprocessorDirective = false;
}
bool Preprocessor::isValidToken(const Token &tk) const
{
if (tk.isNot(T_EOF_SYMBOL) && (! tk.newline() || tk.joined()))
return true;
return false;
}
void Preprocessor::handleDefineDirective(Token *tk)
{
lex(tk);
if (tk->is(T_IDENTIFIER)) {
const StringRef macroName = asStringRef(*tk);
Macro macro;
lex(tk);
if (isValidToken(*tk) && tk->is(T_LPAREN) && ! tk->whitespace()) {
macro.isFunctionLike = true;
lex(tk); // skip `('
if (isValidToken(*tk) && tk->is(T_IDENTIFIER)) {
macro.formals.push_back(asStringRef(*tk));
lex(tk);
while (isValidToken(*tk) && tk->is(T_COMMA)) {
lex(tk);
if (isValidToken(*tk) && tk->is(T_IDENTIFIER)) {
macro.formals.push_back(asStringRef(*tk));
lex(tk);
}
}
}
if (isValidToken(*tk) && tk->is(T_RPAREN))
lex(tk); // skip `)'
}
while (isValidToken(*tk)) {
macro.body.push_back(*tk);
lex(tk);
}
macros.insert(std::make_pair(macroName, macro));
} else {
skipPreprocesorDirective(tk);
}
}
void Preprocessor::skipPreprocesorDirective(Token *tk)
{
do {
lex(tk);
} while (isValidToken(*tk));
}
StringRef Preprocessor::asStringRef(const Token &tk) const
{ return StringRef(_source.begin() + tk.begin(), tk.length()); }
Preprocessor::Preprocessor(std::ostream &out)
: out(out), _lexer(0), inPreprocessorDirective(false)
{ }
void Preprocessor::operator()(const char *source, unsigned size, const StringRef &currentFileName)
{
_currentFileName = currentFileName;
run(source, size);
}
void Preprocessor::run(const char *source, unsigned size)
{
_tokenBuffer = 0;
const StringRef previousSource = switchSource(StringRef(source, size));
Lexer thisLexer(source, source + size);
thisLexer.setScanKeywords(false);
Lexer *previousLexer = switchLexer(&thisLexer);
inPreprocessorDirective = false;
Token tk;
unsigned lineno = 0;
do {
lex(&tk);
if (lineno != tk.lineno) {
if (lineno > tk.lineno || tk.lineno - lineno > 3)
out << std::endl << "#line " << tk.lineno << " \"" << _currentFileName << "\"" << std::endl;
else {
for (unsigned i = lineno; i < tk.lineno; ++i)
out << std::endl;
}
lineno = tk.lineno;
} else {
if (tk.newline())
out << std::endl;
if (tk.whitespace())
out << ' ';
}
out << asStringRef(tk);
lineno = tk.lineno;
} while (tk.isNot(T_EOF_SYMBOL));
out << std::endl;
switchLexer(previousLexer);
switchSource(previousSource);
}
/**************************************************************************
**
** This file is part of Qt Creator
**
** Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
**
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** Commercial Usage
**
** Licensees holding valid Qt Commercial licenses may use this file in
** accordance with the Qt Commercial License Agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Nokia.
**
** GNU Lesser General Public License Usage
**
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** If you are unsure which license is appropriate for your use, please
** contact the sales department at http://qt.nokia.com/contact.
**
**************************************************************************/
#ifndef CPLUSPLUS_PREPROCESSOR_H
#define CPLUSPLUS_PREPROCESSOR_H
#include <CPlusPlusForwardDeclarations.h>
#include <iosfwd>
#include <vector>
#include <map>
namespace CPlusPlus {
class Lexer;
class Token;
class CPLUSPLUS_EXPORT StringRef
{
const char *_text;
unsigned _size;
public:
typedef const char *iterator;
typedef const char *const_iterator;
StringRef()
: _text(0), _size(0) {}
StringRef(const char *text, unsigned size)
: _text(text), _size(size) {}
StringRef(const char *text)
: _text(text), _size(strlen(text)) {}
inline const char *text() const { return _text; }
inline unsigned size() const { return _size; }
inline const_iterator begin() const { return _text; }
inline const_iterator end() const { return _text + _size; }
bool operator == (const StringRef &other) const
{
if (_size == other._size)
return _text == other._text || ! strncmp(_text, other._text, _size);
return false;
}
bool operator != (const StringRef &other) const
{ return ! operator == (other); }
bool operator < (const StringRef &other) const
{ return std::lexicographical_compare(begin(), end(), other.begin(), other.end()); }
};
class CPLUSPLUS_EXPORT Preprocessor
{
public:
Preprocessor(std::ostream &out);
void operator()(const char *source, unsigned size, const StringRef &currentFileName);
private:
struct Macro
{
Macro(): isFunctionLike(false), isVariadic(false) {}
std::vector<StringRef> formals;
std::vector<Token> body;
bool isFunctionLike: 1;
bool isVariadic: 1;
};
void run(const char *source, unsigned size);
Lexer *switchLexer(Lexer *lex);
StringRef switchSource(const StringRef &source);
const Macro *resolveMacro(const StringRef &name) const;
StringRef asStringRef(const Token &tk) const;
void lex(Token *tk);
bool isValidToken(const Token &tk) const;
void handlePreprocessorDirective(Token *tk);
void handleDefineDirective(Token *tk);
void skipPreprocesorDirective(Token *tk);
void collectActualArguments(Token *tk, std::vector<std::vector<Token> > *actuals);
void scanActualArgument(Token *tk, std::vector<Token> *tokens);
private:
struct TokenBuffer;
std::ostream &out;
StringRef _currentFileName;
Lexer *_lexer;
StringRef _source;
TokenBuffer *_tokenBuffer;
bool inPreprocessorDirective: 1;
std::map<StringRef, Macro> macros;
};
} // end of namespace CPlusPlus
CPLUSPLUS_EXPORT std::ostream &operator << (std::ostream &out, const CPlusPlus::StringRef &s);
#endif // CPLUSPLUS_PREPROCESSOR_H
......@@ -9,6 +9,8 @@ macx {
include(../../../src/shared/cplusplus/cplusplus.pri)
# Input
HEADERS += Preprocessor.h
SOURCES += Preprocessor.cpp
SOURCES += main.cpp
unix {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment