Commit e42ca194 authored by Erik Verbruggen's avatar Erik Verbruggen

Introduced token caching to prevent repetetive tokenizing.

Also removed TokenUnderCursor as it's functionality is in the token cache.

Reviewed-by: ckamm
parent 2e100162
...@@ -27,33 +27,52 @@ ...@@ -27,33 +27,52 @@
** **
**************************************************************************/ **************************************************************************/
#include "BackwardsScanner.h" #include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h> #include <Token.h>
#include <QtGui/QTextCursor> #include <QtGui/QTextCursor>
#include <QTextDocument>
using namespace CPlusPlus; using namespace CPlusPlus;
BackwardsScanner::BackwardsScanner(const QTextCursor &cursor, const QString &suffix, int maxBlockCount) BackwardsScanner::BackwardsScanner(TokenCache *tokenCache, const QTextCursor &cursor, int maxBlockCount, const QString &suffix)
: _offset(0) : _tokenCache(tokenCache)
, _offset(0)
, _blocksTokenized(0) , _blocksTokenized(0)
, _block(cursor.block()) , _block(cursor.block())
, _maxBlockCount(maxBlockCount) , _maxBlockCount(maxBlockCount)
{ {
_tokenize.setQtMocRunEnabled(true); int pos = cursor.position() - cursor.block().position();
_tokenize.setSkipComments(true); _text = _block.text().left(pos);
_tokenize.setObjCEnabled(true); _text += suffix;
_text = _block.text().left(cursor.position() - cursor.block().position());
if (! suffix.isEmpty()) _tokens.append(tokenCache->tokensForBlock(_block));
_text += suffix;
_tokens.append(_tokenize(_text, previousBlockState(_block))); for (int i = _tokens.size() - 1; i >= 0; --i) {
const int tokenEnd = _tokens.at(i).end();
if ((tokenEnd < pos) ||
(tokenEnd == pos && suffix.isEmpty())) {
break;
} else {
_tokens.removeAt(i);
}
}
QString remainingText;
if (!_tokens.isEmpty())
remainingText = _text.mid(_tokens.last().end());
if (!remainingText.isEmpty()) {
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(true);
tokenize.setSkipComments(true);
tokenize.setObjCEnabled(true);
_tokens.append(tokenize(remainingText, TokenCache::previousBlockState(_block)));
}
_startToken = _tokens.size(); _startToken = _tokens.size();
} }
int BackwardsScanner::state() const
{ return _tokenize.state(); }
SimpleToken BackwardsScanner::LA(int index) const SimpleToken BackwardsScanner::LA(int index) const
{ return const_cast<BackwardsScanner *>(this)->fetchToken(_startToken - index); } { return const_cast<BackwardsScanner *>(this)->fetchToken(_startToken - index); }
...@@ -71,6 +90,7 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex) ...@@ -71,6 +90,7 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
} else { } else {
++_blocksTokenized; ++_blocksTokenized;
QList<SimpleToken> newTokens = _tokenCache->tokensForBlock(_block);
QString blockText = _block.text(); QString blockText = _block.text();
_text.prepend(QLatin1Char('\n')); _text.prepend(QLatin1Char('\n'));
_text.prepend(blockText); _text.prepend(blockText);
...@@ -79,12 +99,11 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex) ...@@ -79,12 +99,11 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
for (int i = 0; i < _tokens.size(); ++i) { for (int i = 0; i < _tokens.size(); ++i) {
SimpleToken t = _tokens.at(i); SimpleToken t = _tokens.at(i);
t.setPosition(t.position() + blockText.length() + 1); t.setPosition(t.position() + blockText.length() + 1);
t.setText(_text.midRef(t.position(), t.length()));
adaptedTokens.append(t); adaptedTokens.append(t);
} }
_tokens = _tokenize(blockText, previousBlockState(_block)); _tokens = newTokens;
_offset += _tokens.size(); _offset += newTokens.size();
_tokens += adaptedTokens; _tokens += adaptedTokens;
} }
} }
...@@ -119,20 +138,6 @@ QStringRef BackwardsScanner::textRef(int index) const ...@@ -119,20 +138,6 @@ QStringRef BackwardsScanner::textRef(int index) const
return _text.midRef(firstToken.begin(), firstToken.length()); return _text.midRef(firstToken.begin(), firstToken.length());
} }
int BackwardsScanner::previousBlockState(const QTextBlock &block)
{
const QTextBlock prevBlock = block.previous();
if (prevBlock.isValid()) {
int state = prevBlock.userState();
if (state != -1)
return state;
}
return 0;
}
int BackwardsScanner::size() const int BackwardsScanner::size() const
{ {
return _tokens.size(); return _tokens.size();
......
...@@ -36,16 +36,18 @@ ...@@ -36,16 +36,18 @@
namespace CPlusPlus { namespace CPlusPlus {
class TokenCache;
class CPLUSPLUS_EXPORT BackwardsScanner class CPLUSPLUS_EXPORT BackwardsScanner
{ {
enum { MAX_BLOCK_COUNT = 10 }; enum { MAX_BLOCK_COUNT = 10 };
public: public:
BackwardsScanner(const QTextCursor &cursor, BackwardsScanner(TokenCache *cache,
const QString &suffix = QString(), const QTextCursor &cursor,
int maxBlockCount = MAX_BLOCK_COUNT); int maxBlockCount = MAX_BLOCK_COUNT,
const QString &suffix = QString());
int state() const;
int startToken() const; int startToken() const;
int startPosition() const; int startPosition() const;
...@@ -67,20 +69,18 @@ public: ...@@ -67,20 +69,18 @@ public:
int startOfMatchingBrace(int index) const; int startOfMatchingBrace(int index) const;
int startOfBlock(int index) const; int startOfBlock(int index) const;
static int previousBlockState(const QTextBlock &block);
int size() const; int size() const;
private: private:
const SimpleToken &fetchToken(int tokenIndex); const SimpleToken &fetchToken(int tokenIndex);
private: private:
TokenCache *_tokenCache;
QList<SimpleToken> _tokens; QList<SimpleToken> _tokens;
int _offset; int _offset;
int _blocksTokenized; int _blocksTokenized;
QTextBlock _block; QTextBlock _block;
QString _text; QString _text;
SimpleLexer _tokenize;
int _maxBlockCount; int _maxBlockCount;
int _startToken; int _startToken;
}; };
......
...@@ -30,6 +30,7 @@ ...@@ -30,6 +30,7 @@
#include "ExpressionUnderCursor.h" #include "ExpressionUnderCursor.h"
#include "SimpleLexer.h" #include "SimpleLexer.h"
#include "BackwardsScanner.h" #include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h> #include <Token.h>
#include <QTextCursor> #include <QTextCursor>
...@@ -37,8 +38,8 @@ ...@@ -37,8 +38,8 @@
using namespace CPlusPlus; using namespace CPlusPlus;
ExpressionUnderCursor::ExpressionUnderCursor() ExpressionUnderCursor::ExpressionUnderCursor(TokenCache *tokenCache)
: _jumpedComma(false) : _tokenCache(tokenCache), _jumpedComma(false)
{ } { }
ExpressionUnderCursor::~ExpressionUnderCursor() ExpressionUnderCursor::~ExpressionUnderCursor()
...@@ -218,7 +219,7 @@ bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk) ...@@ -218,7 +219,7 @@ bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk)
QString ExpressionUnderCursor::operator()(const QTextCursor &cursor) QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
{ {
BackwardsScanner scanner(cursor); BackwardsScanner scanner(_tokenCache, cursor);
_jumpedComma = false; _jumpedComma = false;
...@@ -232,7 +233,7 @@ QString ExpressionUnderCursor::operator()(const QTextCursor &cursor) ...@@ -232,7 +233,7 @@ QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
{ {
BackwardsScanner scanner(cursor); BackwardsScanner scanner(_tokenCache, cursor);
int index = scanner.startToken(); int index = scanner.startToken();
......
...@@ -43,11 +43,12 @@ namespace CPlusPlus { ...@@ -43,11 +43,12 @@ namespace CPlusPlus {
class BackwardsScanner; class BackwardsScanner;
class SimpleToken; class SimpleToken;
class TokenCache;
class CPLUSPLUS_EXPORT ExpressionUnderCursor class CPLUSPLUS_EXPORT ExpressionUnderCursor
{ {
public: public:
ExpressionUnderCursor(); ExpressionUnderCursor(TokenCache *tokenCache);
~ExpressionUnderCursor(); ~ExpressionUnderCursor();
QString operator()(const QTextCursor &cursor); QString operator()(const QTextCursor &cursor);
...@@ -59,6 +60,7 @@ private: ...@@ -59,6 +60,7 @@ private:
bool isAccessToken(const SimpleToken &tk); bool isAccessToken(const SimpleToken &tk);
private: private:
TokenCache *_tokenCache;
bool _jumpedComma; bool _jumpedComma;
}; };
......
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
**************************************************************************/ **************************************************************************/
#include "MatchingText.h" #include "MatchingText.h"
#include "BackwardsScanner.h" #include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h> #include <Token.h>
...@@ -75,7 +76,8 @@ static bool isCompleteCharLiteral(const BackwardsScanner &tk, int index) ...@@ -75,7 +76,8 @@ static bool isCompleteCharLiteral(const BackwardsScanner &tk, int index)
return false; return false;
} }
MatchingText::MatchingText() MatchingText::MatchingText(TokenCache *tokenCache)
: _tokenCache(tokenCache)
{ } { }
bool MatchingText::shouldInsertMatchingText(const QTextCursor &tc) bool MatchingText::shouldInsertMatchingText(const QTextCursor &tc)
...@@ -151,7 +153,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri ...@@ -151,7 +153,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri
if (text.isEmpty() || !shouldInsertMatchingText(la)) if (text.isEmpty() || !shouldInsertMatchingText(la))
return QString(); return QString();
BackwardsScanner tk(tc, textToProcess.left(*skippedChars), MAX_NUM_LINES); BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES, textToProcess.left(*skippedChars));
const int startToken = tk.startToken(); const int startToken = tk.startToken();
int index = startToken; int index = startToken;
...@@ -211,7 +213,7 @@ bool MatchingText::shouldInsertNewline(const QTextCursor &tc) const ...@@ -211,7 +213,7 @@ bool MatchingText::shouldInsertNewline(const QTextCursor &tc) const
QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
{ {
BackwardsScanner tk(tc, QString(), MAX_NUM_LINES); BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES);
int index = tk.startToken(); int index = tk.startToken();
if (tk[index - 1].isNot(T_LBRACE)) if (tk[index - 1].isNot(T_LBRACE))
......
...@@ -35,11 +35,12 @@ ...@@ -35,11 +35,12 @@
namespace CPlusPlus { namespace CPlusPlus {
class BackwardsScanner; class BackwardsScanner;
class TokenCache;
class CPLUSPLUS_EXPORT MatchingText class CPLUSPLUS_EXPORT MatchingText
{ {
public: public:
MatchingText(); MatchingText(TokenCache *tokenCache);
static bool shouldInsertMatchingText(const QTextCursor &tc); static bool shouldInsertMatchingText(const QTextCursor &tc);
static bool shouldInsertMatchingText(const QChar &lookAhead); static bool shouldInsertMatchingText(const QChar &lookAhead);
...@@ -50,6 +51,8 @@ public: ...@@ -50,6 +51,8 @@ public:
private: private:
bool shouldInsertNewline(const QTextCursor &tc) const; bool shouldInsertNewline(const QTextCursor &tc) const;
TokenCache *_tokenCache;
}; };
} // end of namespace CPlusPlus } // end of namespace CPlusPlus
......
...@@ -37,12 +37,11 @@ ...@@ -37,12 +37,11 @@
using namespace CPlusPlus; using namespace CPlusPlus;
SimpleToken::SimpleToken(const Token &token, const QStringRef &text) SimpleToken::SimpleToken(const Token &token)
: _kind(token.f.kind) : _kind(token.f.kind)
, _flags(0) , _flags(0)
, _position(token.begin()) , _position(token.begin())
, _length(token.f.length) , _length(token.f.length)
, _text(text)
{ {
f._whitespace = token.f.whitespace; f._whitespace = token.f.whitespace;
f._newline = token.f.newline; f._newline = token.f.newline;
...@@ -148,17 +147,17 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state) ...@@ -148,17 +147,17 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
break; break;
QStringRef spell = text.midRef(lex.tokenOffset(), lex.tokenLength()); QStringRef spell = text.midRef(lex.tokenOffset(), lex.tokenLength());
SimpleToken simpleTk(tk, spell); SimpleToken simpleTk(tk);
lex.setScanAngleStringLiteralTokens(false); lex.setScanAngleStringLiteralTokens(false);
if (tk.f.newline && tk.is(T_POUND)) if (tk.f.newline && tk.is(T_POUND))
inPreproc = true; inPreproc = true;
else if (inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) && else if (inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
simpleTk.text() == QLatin1String("include")) spell == QLatin1String("include"))
lex.setScanAngleStringLiteralTokens(true); lex.setScanAngleStringLiteralTokens(true);
else if (_objCEnabled else if (_objCEnabled
&& inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) && && inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
simpleTk.text() == QLatin1String("import")) spell == QLatin1String("import"))
lex.setScanAngleStringLiteralTokens(true); lex.setScanAngleStringLiteralTokens(true);
if (_objCEnabled && tk.is(T_IDENTIFIER)) if (_objCEnabled && tk.is(T_IDENTIFIER))
......
...@@ -42,7 +42,7 @@ class Token; ...@@ -42,7 +42,7 @@ class Token;
class CPLUSPLUS_EXPORT SimpleToken class CPLUSPLUS_EXPORT SimpleToken
{ {
public: public:
SimpleToken(const Token &token, const QStringRef &text); SimpleToken(const Token &token);
SimpleToken() SimpleToken()
: _kind(0) : _kind(0)
...@@ -66,9 +66,6 @@ public: ...@@ -66,9 +66,6 @@ public:
inline int end() const inline int end() const
{ return _position + _length; } { return _position + _length; }
inline QStringRef text() const
{ return _text; }
inline bool followsNewline() const inline bool followsNewline() const
{ return f._newline; } { return f._newline; }
...@@ -91,10 +88,6 @@ public: ...@@ -91,10 +88,6 @@ public:
inline void setPosition(int position) inline void setPosition(int position)
{ _position = position; } { _position = position; }
// internal
inline void setText(const QStringRef &text)
{ _text = text; }
public: public:
short _kind; short _kind;
union { union {
...@@ -109,7 +102,6 @@ public: ...@@ -109,7 +102,6 @@ public:
int _position; int _position;
int _length; int _length;
QStringRef _text;
friend class SimpleLexer; friend class SimpleLexer;
}; };
......
#include "SimpleLexer.h"
#include "TokenCache.h"
using namespace CPlusPlus;
TokenCache::TokenCache()
: m_doc(0)
, m_revision(-1)
{}
void TokenCache::setDocument(QTextDocument *doc)
{
m_doc = doc;
m_revision = -1;
}
QList<SimpleToken> TokenCache::tokensForBlock(const QTextBlock &block) const
{
Q_ASSERT(m_doc);
Q_ASSERT(m_doc == block.document());
const int documentRevision = m_doc->revision();
if (documentRevision != m_revision) {
m_tokensByBlock.clear();
m_revision = documentRevision;
}
const int blockNr = block.blockNumber();
if (m_tokensByBlock.contains(blockNr)) {
return m_tokensByBlock.value(blockNr);
} else {
SimpleLexer tokenize;
tokenize.setObjCEnabled(true);
tokenize.setQtMocRunEnabled(true);
tokenize.setSkipComments(false);
const int prevState = previousBlockState(block);
QList<SimpleToken> tokens = tokenize(block.text(), prevState);
m_tokensByBlock.insert(blockNr, tokens);
return tokens;
}
}
SimpleToken TokenCache::tokenUnderCursor(const QTextCursor &cursor) const
{
const QTextBlock block = cursor.block();
const QList<SimpleToken> tokens = tokensForBlock(block);
const int column = cursor.position() - block.position();
for (int index = tokens.size() - 1; index >= 0; --index) {
const SimpleToken &tk = tokens.at(index);
if (tk.position() < column)
return tk;
}
return SimpleToken();
}
QString TokenCache::text(const QTextBlock &block, int tokenIndex) const
{
SimpleToken tk = tokensForBlock(block).at(tokenIndex);
return block.text().mid(tk.position(), tk.length());
}
int TokenCache::previousBlockState(const QTextBlock &block)
{
const QTextBlock prevBlock = block.previous();
if (prevBlock.isValid()) {
int state = prevBlock.userState();
if (state != -1)
return state;
}
return 0;
}
#ifndef TOKENCACHE_H
#define TOKENCACHE_H
#include <CPlusPlusForwardDeclarations.h>
#include <cplusplus/SimpleLexer.h>
#include <QtCore/QHash>
#include <QtCore/QList>
#include <QtGui/QTextBlock>
#include <QtGui/QTextCursor>
#include <QtGui/QTextDocument>
namespace CPlusPlus {
class CPLUSPLUS_EXPORT TokenCache
{
public:
TokenCache();
void setDocument(QTextDocument *doc);
QList<CPlusPlus::SimpleToken> tokensForBlock(const QTextBlock &block) const;
CPlusPlus::SimpleToken tokenUnderCursor(const QTextCursor &cursor) const;