Commit e42ca194 authored by Erik Verbruggen's avatar Erik Verbruggen

Introduced token caching to prevent repetetive tokenizing.

Also removed TokenUnderCursor as it's functionality is in the token cache.

Reviewed-by: ckamm
parent 2e100162
......@@ -27,33 +27,52 @@
**
**************************************************************************/
#include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h>
#include <QtGui/QTextCursor>
#include <QTextDocument>
using namespace CPlusPlus;
BackwardsScanner::BackwardsScanner(const QTextCursor &cursor, const QString &suffix, int maxBlockCount)
: _offset(0)
BackwardsScanner::BackwardsScanner(TokenCache *tokenCache, const QTextCursor &cursor, int maxBlockCount, const QString &suffix)
: _tokenCache(tokenCache)
, _offset(0)
, _blocksTokenized(0)
, _block(cursor.block())
, _maxBlockCount(maxBlockCount)
{
_tokenize.setQtMocRunEnabled(true);
_tokenize.setSkipComments(true);
_tokenize.setObjCEnabled(true);
_text = _block.text().left(cursor.position() - cursor.block().position());
int pos = cursor.position() - cursor.block().position();
_text = _block.text().left(pos);
_text += suffix;
if (! suffix.isEmpty())
_text += suffix;
_tokens.append(tokenCache->tokensForBlock(_block));
_tokens.append(_tokenize(_text, previousBlockState(_block)));
for (int i = _tokens.size() - 1; i >= 0; --i) {
const int tokenEnd = _tokens.at(i).end();
if ((tokenEnd < pos) ||
(tokenEnd == pos && suffix.isEmpty())) {
break;
} else {
_tokens.removeAt(i);
}
}
QString remainingText;
if (!_tokens.isEmpty())
remainingText = _text.mid(_tokens.last().end());
if (!remainingText.isEmpty()) {
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(true);
tokenize.setSkipComments(true);
tokenize.setObjCEnabled(true);
_tokens.append(tokenize(remainingText, TokenCache::previousBlockState(_block)));
}
_startToken = _tokens.size();
}
int BackwardsScanner::state() const
{ return _tokenize.state(); }
SimpleToken BackwardsScanner::LA(int index) const
{ return const_cast<BackwardsScanner *>(this)->fetchToken(_startToken - index); }
......@@ -71,6 +90,7 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
} else {
++_blocksTokenized;
QList<SimpleToken> newTokens = _tokenCache->tokensForBlock(_block);
QString blockText = _block.text();
_text.prepend(QLatin1Char('\n'));
_text.prepend(blockText);
......@@ -79,12 +99,11 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
for (int i = 0; i < _tokens.size(); ++i) {
SimpleToken t = _tokens.at(i);
t.setPosition(t.position() + blockText.length() + 1);
t.setText(_text.midRef(t.position(), t.length()));
adaptedTokens.append(t);
}
_tokens = _tokenize(blockText, previousBlockState(_block));
_offset += _tokens.size();
_tokens = newTokens;
_offset += newTokens.size();
_tokens += adaptedTokens;
}
}
......@@ -119,20 +138,6 @@ QStringRef BackwardsScanner::textRef(int index) const
return _text.midRef(firstToken.begin(), firstToken.length());
}
int BackwardsScanner::previousBlockState(const QTextBlock &block)
{
const QTextBlock prevBlock = block.previous();
if (prevBlock.isValid()) {
int state = prevBlock.userState();
if (state != -1)
return state;
}
return 0;
}
int BackwardsScanner::size() const
{
return _tokens.size();
......
......@@ -36,16 +36,18 @@
namespace CPlusPlus {
class TokenCache;
class CPLUSPLUS_EXPORT BackwardsScanner
{
enum { MAX_BLOCK_COUNT = 10 };
public:
BackwardsScanner(const QTextCursor &cursor,
const QString &suffix = QString(),
int maxBlockCount = MAX_BLOCK_COUNT);
BackwardsScanner(TokenCache *cache,
const QTextCursor &cursor,
int maxBlockCount = MAX_BLOCK_COUNT,
const QString &suffix = QString());
int state() const;
int startToken() const;
int startPosition() const;
......@@ -67,20 +69,18 @@ public:
int startOfMatchingBrace(int index) const;
int startOfBlock(int index) const;
static int previousBlockState(const QTextBlock &block);
int size() const;
private:
const SimpleToken &fetchToken(int tokenIndex);
private:
TokenCache *_tokenCache;
QList<SimpleToken> _tokens;
int _offset;
int _blocksTokenized;
QTextBlock _block;
QString _text;
SimpleLexer _tokenize;
int _maxBlockCount;
int _startToken;
};
......
......@@ -30,6 +30,7 @@
#include "ExpressionUnderCursor.h"
#include "SimpleLexer.h"
#include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h>
#include <QTextCursor>
......@@ -37,8 +38,8 @@
using namespace CPlusPlus;
ExpressionUnderCursor::ExpressionUnderCursor()
: _jumpedComma(false)
ExpressionUnderCursor::ExpressionUnderCursor(TokenCache *tokenCache)
: _tokenCache(tokenCache), _jumpedComma(false)
{ }
ExpressionUnderCursor::~ExpressionUnderCursor()
......@@ -218,7 +219,7 @@ bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk)
QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
{
BackwardsScanner scanner(cursor);
BackwardsScanner scanner(_tokenCache, cursor);
_jumpedComma = false;
......@@ -232,7 +233,7 @@ QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
{
BackwardsScanner scanner(cursor);
BackwardsScanner scanner(_tokenCache, cursor);
int index = scanner.startToken();
......
......@@ -43,11 +43,12 @@ namespace CPlusPlus {
class BackwardsScanner;
class SimpleToken;
class TokenCache;
class CPLUSPLUS_EXPORT ExpressionUnderCursor
{
public:
ExpressionUnderCursor();
ExpressionUnderCursor(TokenCache *tokenCache);
~ExpressionUnderCursor();
QString operator()(const QTextCursor &cursor);
......@@ -59,6 +60,7 @@ private:
bool isAccessToken(const SimpleToken &tk);
private:
TokenCache *_tokenCache;
bool _jumpedComma;
};
......
......@@ -28,6 +28,7 @@
**************************************************************************/
#include "MatchingText.h"
#include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h>
......@@ -75,7 +76,8 @@ static bool isCompleteCharLiteral(const BackwardsScanner &tk, int index)
return false;
}
MatchingText::MatchingText()
MatchingText::MatchingText(TokenCache *tokenCache)
: _tokenCache(tokenCache)
{ }
bool MatchingText::shouldInsertMatchingText(const QTextCursor &tc)
......@@ -151,7 +153,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri
if (text.isEmpty() || !shouldInsertMatchingText(la))
return QString();
BackwardsScanner tk(tc, textToProcess.left(*skippedChars), MAX_NUM_LINES);
BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES, textToProcess.left(*skippedChars));
const int startToken = tk.startToken();
int index = startToken;
......@@ -211,7 +213,7 @@ bool MatchingText::shouldInsertNewline(const QTextCursor &tc) const
QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
{
BackwardsScanner tk(tc, QString(), MAX_NUM_LINES);
BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES);
int index = tk.startToken();
if (tk[index - 1].isNot(T_LBRACE))
......
......@@ -35,11 +35,12 @@
namespace CPlusPlus {
class BackwardsScanner;
class TokenCache;
class CPLUSPLUS_EXPORT MatchingText
{
public:
MatchingText();
MatchingText(TokenCache *tokenCache);
static bool shouldInsertMatchingText(const QTextCursor &tc);
static bool shouldInsertMatchingText(const QChar &lookAhead);
......@@ -50,6 +51,8 @@ public:
private:
bool shouldInsertNewline(const QTextCursor &tc) const;
TokenCache *_tokenCache;
};
} // end of namespace CPlusPlus
......
......@@ -37,12 +37,11 @@
using namespace CPlusPlus;
SimpleToken::SimpleToken(const Token &token, const QStringRef &text)
SimpleToken::SimpleToken(const Token &token)
: _kind(token.f.kind)
, _flags(0)
, _position(token.begin())
, _length(token.f.length)
, _text(text)
{
f._whitespace = token.f.whitespace;
f._newline = token.f.newline;
......@@ -148,17 +147,17 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
break;
QStringRef spell = text.midRef(lex.tokenOffset(), lex.tokenLength());
SimpleToken simpleTk(tk, spell);
SimpleToken simpleTk(tk);
lex.setScanAngleStringLiteralTokens(false);
if (tk.f.newline && tk.is(T_POUND))
inPreproc = true;
else if (inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
simpleTk.text() == QLatin1String("include"))
spell == QLatin1String("include"))
lex.setScanAngleStringLiteralTokens(true);
else if (_objCEnabled
&& inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
simpleTk.text() == QLatin1String("import"))
spell == QLatin1String("import"))
lex.setScanAngleStringLiteralTokens(true);
if (_objCEnabled && tk.is(T_IDENTIFIER))
......
......@@ -42,7 +42,7 @@ class Token;
class CPLUSPLUS_EXPORT SimpleToken
{
public:
SimpleToken(const Token &token, const QStringRef &text);
SimpleToken(const Token &token);
SimpleToken()
: _kind(0)
......@@ -66,9 +66,6 @@ public:
inline int end() const
{ return _position + _length; }
inline QStringRef text() const
{ return _text; }
inline bool followsNewline() const
{ return f._newline; }
......@@ -91,10 +88,6 @@ public:
inline void setPosition(int position)
{ _position = position; }
// internal
inline void setText(const QStringRef &text)
{ _text = text; }
public:
short _kind;
union {
......@@ -109,7 +102,6 @@ public:
int _position;
int _length;
QStringRef _text;
friend class SimpleLexer;
};
......
#include "SimpleLexer.h"
#include "TokenCache.h"
using namespace CPlusPlus;
TokenCache::TokenCache()
: m_doc(0)
, m_revision(-1)
{}
void TokenCache::setDocument(QTextDocument *doc)
{
m_doc = doc;
m_revision = -1;
}
QList<SimpleToken> TokenCache::tokensForBlock(const QTextBlock &block) const
{
Q_ASSERT(m_doc);
Q_ASSERT(m_doc == block.document());
const int documentRevision = m_doc->revision();
if (documentRevision != m_revision) {
m_tokensByBlock.clear();
m_revision = documentRevision;
}
const int blockNr = block.blockNumber();
if (m_tokensByBlock.contains(blockNr)) {
return m_tokensByBlock.value(blockNr);
} else {
SimpleLexer tokenize;
tokenize.setObjCEnabled(true);
tokenize.setQtMocRunEnabled(true);
tokenize.setSkipComments(false);
const int prevState = previousBlockState(block);
QList<SimpleToken> tokens = tokenize(block.text(), prevState);
m_tokensByBlock.insert(blockNr, tokens);
return tokens;
}
}
SimpleToken TokenCache::tokenUnderCursor(const QTextCursor &cursor) const
{
const QTextBlock block = cursor.block();
const QList<SimpleToken> tokens = tokensForBlock(block);
const int column = cursor.position() - block.position();
for (int index = tokens.size() - 1; index >= 0; --index) {
const SimpleToken &tk = tokens.at(index);
if (tk.position() < column)
return tk;
}
return SimpleToken();
}
QString TokenCache::text(const QTextBlock &block, int tokenIndex) const
{
SimpleToken tk = tokensForBlock(block).at(tokenIndex);
return block.text().mid(tk.position(), tk.length());
}
int TokenCache::previousBlockState(const QTextBlock &block)
{
const QTextBlock prevBlock = block.previous();
if (prevBlock.isValid()) {
int state = prevBlock.userState();
if (state != -1)
return state;
}
return 0;
}
#ifndef TOKENCACHE_H
#define TOKENCACHE_H
#include <CPlusPlusForwardDeclarations.h>
#include <cplusplus/SimpleLexer.h>
#include <QtCore/QHash>
#include <QtCore/QList>
#include <QtGui/QTextBlock>
#include <QtGui/QTextCursor>
#include <QtGui/QTextDocument>
namespace CPlusPlus {
class CPLUSPLUS_EXPORT TokenCache
{
public:
TokenCache();
void setDocument(QTextDocument *doc);
QList<CPlusPlus::SimpleToken> tokensForBlock(const QTextBlock &block) const;
CPlusPlus::SimpleToken tokenUnderCursor(const QTextCursor &cursor) const;
QString text(const QTextBlock &block, int tokenIndex) const;
static int previousBlockState(const QTextBlock &block);
private:
QTextDocument *m_doc;
mutable int m_revision;
mutable QHash<int, QList<CPlusPlus::SimpleToken> > m_tokensByBlock;
};
} // namespace CPlusPlus
#endif // TOKENCACHE_H
/**************************************************************************
**
** This file is part of Qt Creator
**
** Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
**
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** Commercial Usage
**
** Licensees holding valid Qt Commercial licenses may use this file in
** accordance with the Qt Commercial License Agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Nokia.
**
** GNU Lesser General Public License Usage
**
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** If you are unsure which license is appropriate for your use, please
** contact the sales department at http://qt.nokia.com/contact.
**
**************************************************************************/
#include "TokenUnderCursor.h"
#include "BackwardsScanner.h"
#include <Token.h>
#include <QTextCursor>
#include <QTextBlock>
#include <climits>
using namespace CPlusPlus;
TokenUnderCursor::TokenUnderCursor()
{ }
TokenUnderCursor::~TokenUnderCursor()
{ }
SimpleToken TokenUnderCursor::operator()(const QTextCursor &cursor, QTextBlock *b)
{
SimpleLexer tokenize;
tokenize.setObjCEnabled(true);
tokenize.setSkipComments(false);
QTextBlock block = cursor.block();
int column = cursor.position() - cursor.block().position();
_text = block.text();
_tokens = tokenize(_text, BackwardsScanner::previousBlockState(block));
for (int index = _tokens.size() - 1; index != -1; --index) {
const SimpleToken &tk = _tokens.at(index);
if (tk.position() < column) {
if (b)
*b = block;
return tk;
}
}
return SimpleToken();
}
/**************************************************************************
**
** This file is part of Qt Creator
**
** Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
**
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** Commercial Usage
**
** Licensees holding valid Qt Commercial licenses may use this file in
** accordance with the Qt Commercial License Agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Nokia.
**
** GNU Lesser General Public License Usage
**
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** If you are unsure which license is appropriate for your use, please
** contact the sales department at http://qt.nokia.com/contact.
**
**************************************************************************/
#ifndef CPLUSPLUS_TOKENUNDERCURSOR_H
#define CPLUSPLUS_TOKENUNDERCURSOR_H
#include "SimpleLexer.h"
#include <QList>
QT_BEGIN_NAMESPACE
class QString;
class QTextCursor;
class QTextBlock;
QT_END_NAMESPACE
namespace CPlusPlus {
class CPLUSPLUS_EXPORT TokenUnderCursor
{
public:
TokenUnderCursor();
~TokenUnderCursor();
SimpleToken operator()(const QTextCursor &cursor, QTextBlock *block = 0);
const QList<SimpleToken> &tokens() const
{ return _tokens; }
private:
QList<SimpleToken> _tokens;
QString _text;
};
} // end of namespace CPlusPlus
#endif // CPLUSPLUS_TOKENUNDERCURSOR_H
......@@ -12,18 +12,18 @@ contains(QT, gui) {
HEADERS += \
$$PWD/Icons.h \
$$PWD/ExpressionUnderCursor.h \
$$PWD/TokenUnderCursor.h \
$$PWD/BackwardsScanner.h \
$$PWD/MatchingText.h \
$$PWD/OverviewModel.h
$$PWD/OverviewModel.h \
$$PWD/TokenCache.h
SOURCES += \
$$PWD/Icons.cpp \
$$PWD/ExpressionUnderCursor.cpp \
$$PWD/TokenUnderCursor.cpp \
$$PWD/BackwardsScanner.cpp \
$$PWD/MatchingText.cpp \
$$PWD/OverviewModel.cpp
$$PWD/OverviewModel.cpp \
$$PWD/TokenCache.cpp
}
HEADERS += \
......
......@@ -51,11 +51,11 @@
#include <cplusplus/Overview.h>
#include <cplusplus/OverviewModel.h>
#include <cplusplus/SimpleLexer.h>
#include <cplusplus/TokenUnderCursor.h>
#include <cplusplus/MatchingText.h>
#include <cplusplus/BackwardsScanner.h>
#include <cplusplus/FastPreprocessor.h>
#include <cplusplus/CheckUndefinedSymbols.h>
#include <cplusplus/TokenCache.h>
#include <cpptools/cppmodelmanagerinterface.h>
......@@ -534,7 +534,7 @@ struct FindCanonicalSymbol
SemanticInfo info;
FindCanonicalSymbol(CPPEditor *editor, const SemanticInfo &info)
: editor(editor), info(info)
: editor(editor), expressionUnderCursor(editor->tokenCache()), info(info)
{
typeOfExpression.init(info.doc, info.snapshot);