Commit c2393df0 authored by Erik Verbruggen's avatar Erik Verbruggen

Introduced a token cache for the C++ editor.

This should speed things up a bit, because before, the line was tokenized at
least 3 times.
parent 414d9fe3
......@@ -27,33 +27,48 @@
**
**************************************************************************/
#include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h>
#include <QtGui/QTextCursor>
#include <QTextDocument>
using namespace CPlusPlus;
BackwardsScanner::BackwardsScanner(const QTextCursor &cursor, const QString &suffix, int maxBlockCount)
: _offset(0)
BackwardsScanner::BackwardsScanner(TokenCache *tokenCache, const QTextCursor &cursor, int maxBlockCount, const QString &suffix)
: _tokenCache(tokenCache)
, _offset(0)
, _blocksTokenized(0)
, _block(cursor.block())
, _maxBlockCount(maxBlockCount)
{
_tokenize.setQtMocRunEnabled(true);
_tokenize.setSkipComments(true);
_tokenize.setObjCEnabled(true);
_text = _block.text().left(cursor.position() - cursor.block().position());
int pos = cursor.position() - cursor.block().position();
_text = _block.text().left(pos);
if (suffix.isEmpty()) {
_tokens.append(tokenCache->tokensForBlock(_block));
int last = -1;
for (int i = _tokens.size() - 1; i >= 0; --i) {
if (_tokens.at(i).begin() < pos) {
last = i;
break;
}
}
for (int i = _tokens.size() - 1; i > last && i >= 0; --i)
_tokens.removeAt(i);
} else {
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(true);
tokenize.setSkipComments(true);
tokenize.setObjCEnabled(true);
if (! suffix.isEmpty())
_text += suffix;
_tokens.append(_tokenize(_text, previousBlockState(_block)));
_tokens.append(tokenize(_text, TokenCache::previousBlockState(_block)));
}
_startToken = _tokens.size();
}
int BackwardsScanner::state() const
{ return _tokenize.state(); }
SimpleToken BackwardsScanner::LA(int index) const
{ return const_cast<BackwardsScanner *>(this)->fetchToken(_startToken - index); }
......@@ -83,7 +98,7 @@ const SimpleToken &BackwardsScanner::fetchToken(int i)
adaptedTokens.append(t);
}
_tokens = _tokenize(blockText, previousBlockState(_block));
_tokens = _tokenCache->tokensForBlock(_block);
_offset += _tokens.size();
_tokens += adaptedTokens;
}
......@@ -119,20 +134,6 @@ QStringRef BackwardsScanner::textRef(int index) const
return _text.midRef(firstToken.begin(), firstToken.length());
}
int BackwardsScanner::previousBlockState(const QTextBlock &block)
{
const QTextBlock prevBlock = block.previous();
if (prevBlock.isValid()) {
int state = prevBlock.userState();
if (state != -1)
return state;
}
return 0;
}
int BackwardsScanner::size() const
{
return _tokens.size();
......
......@@ -36,16 +36,18 @@
namespace CPlusPlus {
class TokenCache;
class CPLUSPLUS_EXPORT BackwardsScanner
{
enum { MAX_BLOCK_COUNT = 10 };
public:
BackwardsScanner(const QTextCursor &cursor,
const QString &suffix = QString(),
int maxBlockCount = MAX_BLOCK_COUNT);
BackwardsScanner(TokenCache *cache,
const QTextCursor &cursor,
int maxBlockCount = MAX_BLOCK_COUNT,
const QString &suffix = QString());
int state() const;
int startToken() const;
int startPosition() const;
......@@ -67,20 +69,18 @@ public:
int startOfMatchingBrace(int index) const;
int startOfBlock(int index) const;
static int previousBlockState(const QTextBlock &block);
int size() const;
private:
const SimpleToken &fetchToken(int i);
private:
TokenCache *_tokenCache;
QList<SimpleToken> _tokens;
int _offset;
int _blocksTokenized;
QTextBlock _block;
QString _text;
SimpleLexer _tokenize;
int _maxBlockCount;
int _startToken;
};
......
......@@ -30,6 +30,7 @@
#include "ExpressionUnderCursor.h"
#include "SimpleLexer.h"
#include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h>
#include <QTextCursor>
......@@ -37,8 +38,8 @@
using namespace CPlusPlus;
ExpressionUnderCursor::ExpressionUnderCursor()
: _jumpedComma(false)
ExpressionUnderCursor::ExpressionUnderCursor(TokenCache *tokenCache)
: _tokenCache(tokenCache), _jumpedComma(false)
{ }
ExpressionUnderCursor::~ExpressionUnderCursor()
......@@ -218,7 +219,7 @@ bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk)
QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
{
BackwardsScanner scanner(cursor);
BackwardsScanner scanner(_tokenCache, cursor);
_jumpedComma = false;
......@@ -232,7 +233,7 @@ QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
{
BackwardsScanner scanner(cursor);
BackwardsScanner scanner(_tokenCache, cursor);
int index = scanner.startToken();
......
......@@ -43,11 +43,12 @@ namespace CPlusPlus {
class BackwardsScanner;
class SimpleToken;
class TokenCache;
class CPLUSPLUS_EXPORT ExpressionUnderCursor
{
public:
ExpressionUnderCursor();
ExpressionUnderCursor(TokenCache *tokenCache);
~ExpressionUnderCursor();
QString operator()(const QTextCursor &cursor);
......@@ -59,6 +60,7 @@ private:
bool isAccessToken(const SimpleToken &tk);
private:
TokenCache *_tokenCache;
bool _jumpedComma;
};
......
......@@ -28,6 +28,7 @@
**************************************************************************/
#include "MatchingText.h"
#include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h>
......@@ -75,7 +76,8 @@ static bool isCompleteCharLiteral(const BackwardsScanner &tk, int index)
return false;
}
MatchingText::MatchingText()
MatchingText::MatchingText(TokenCache *tokenCache)
: _tokenCache(tokenCache)
{ }
bool MatchingText::shouldInsertMatchingText(const QTextCursor &tc)
......@@ -151,7 +153,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri
if (text.isEmpty() || !shouldInsertMatchingText(la))
return QString();
BackwardsScanner tk(tc, textToProcess.left(*skippedChars), MAX_NUM_LINES);
BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES, textToProcess.left(*skippedChars));
const int startToken = tk.startToken();
int index = startToken;
......@@ -211,7 +213,7 @@ bool MatchingText::shouldInsertNewline(const QTextCursor &tc) const
QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
{
BackwardsScanner tk(tc, QString(), MAX_NUM_LINES);
BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES);
int index = tk.startToken();
if (tk[index - 1].isNot(T_LBRACE))
......
......@@ -35,11 +35,12 @@
namespace CPlusPlus {
class BackwardsScanner;
class TokenCache;
class CPLUSPLUS_EXPORT MatchingText
{
public:
MatchingText();
MatchingText(TokenCache *tokenCache);
static bool shouldInsertMatchingText(const QTextCursor &tc);
static bool shouldInsertMatchingText(const QChar &lookAhead);
......@@ -50,6 +51,8 @@ public:
private:
bool shouldInsertNewline(const QTextCursor &tc) const;
TokenCache *_tokenCache;
};
} // end of namespace CPlusPlus
......
#include "SimpleLexer.h"
#include "TokenCache.h"
#include <QtCore/QDebug>
using namespace CPlusPlus;
TokenCache::TokenCache()
: m_doc(0)
, m_revision(-1)
{}
void TokenCache::setDocument(QTextDocument *doc)
{
m_doc = doc;
m_revision = -1;
}
QList<SimpleToken> TokenCache::tokensForBlock(const QTextBlock &block) const
{
Q_ASSERT(m_doc);
const int documentRevision = m_doc->revision();
if (documentRevision != m_revision) {
m_tokensByBlock.clear();
m_revision = documentRevision;
// qDebug() << "** revision changed to" << documentRevision;
}
const int blockNr = block.blockNumber();
if (m_tokensByBlock.contains(blockNr)) {
// qDebug()<<"Cache hit on line" << line;
return m_tokensByBlock.value(blockNr);
} else {
// qDebug()<<"Cache miss on line" << line;
SimpleLexer tokenize;
tokenize.setObjCEnabled(true);
tokenize.setQtMocRunEnabled(true);
tokenize.setSkipComments(false);
const int prevState = previousBlockState(block);
QList<SimpleToken> tokens = tokenize(block.text(), prevState);
m_tokensByBlock.insert(blockNr, tokens);
return tokens;
}
}
int TokenCache::previousBlockState(const QTextBlock &block)
{
const QTextBlock prevBlock = block.previous();
if (prevBlock.isValid()) {
int state = prevBlock.userState();
if (state != -1)
return state;
}
return 0;
}
#ifndef TOKENCACHE_H
#define TOKENCACHE_H
#include <CPlusPlusForwardDeclarations.h>
#include <cplusplus/SimpleLexer.h>
#include <QtCore/QHash>
#include <QtCore/QList>
#include <QtGui/QTextBlock>
#include <QtGui/QTextDocument>
namespace CPlusPlus {
class CPLUSPLUS_EXPORT TokenCache
{
public:
TokenCache();
void setDocument(QTextDocument *doc);
QList<CPlusPlus::SimpleToken> tokensForBlock(const QTextBlock &block) const;
static int previousBlockState(const QTextBlock &block);
private:
QTextDocument *m_doc;
mutable int m_revision;
mutable QHash<int, QList<CPlusPlus::SimpleToken> > m_tokensByBlock;
};
} // namespace CPlusPlus
#endif // TOKENCACHE_H
......@@ -28,12 +28,14 @@
**************************************************************************/
#include "TokenUnderCursor.h"
#include "BackwardsScanner.h"
#include "TokenCache.h"
#include "TokenCache.h"
#include <Token.h>
#include <QTextCursor>
#include <QTextBlock>
#include <climits>
#include <QTextDocument>
using namespace CPlusPlus;
......@@ -43,17 +45,13 @@ TokenUnderCursor::TokenUnderCursor()
TokenUnderCursor::~TokenUnderCursor()
{ }
SimpleToken TokenUnderCursor::operator()(const QTextCursor &cursor, QTextBlock *b)
SimpleToken TokenUnderCursor::operator()(TokenCache *cache, const QTextCursor &cursor, QTextBlock *b)
{
SimpleLexer tokenize;
tokenize.setObjCEnabled(true);
tokenize.setSkipComments(false);
QTextBlock block = cursor.block();
int column = cursor.position() - cursor.block().position();
_text = block.text();
_tokens = tokenize(_text, BackwardsScanner::previousBlockState(block));
_tokens = cache->tokensForBlock(block);
for (int index = _tokens.size() - 1; index != -1; --index) {
const SimpleToken &tk = _tokens.at(index);
if (tk.position() < column) {
......
......@@ -40,13 +40,15 @@ QT_END_NAMESPACE
namespace CPlusPlus {
class TokenCache;
class CPLUSPLUS_EXPORT TokenUnderCursor
{
public:
TokenUnderCursor();
~TokenUnderCursor();
SimpleToken operator()(const QTextCursor &cursor, QTextBlock *block = 0);
SimpleToken operator()(TokenCache *cache, const QTextCursor &cursor, QTextBlock *block = 0);
const QList<SimpleToken> &tokens() const
{ return _tokens; }
......
......@@ -15,7 +15,8 @@ HEADERS += \
$$PWD/TokenUnderCursor.h \
$$PWD/BackwardsScanner.h \
$$PWD/MatchingText.h \
$$PWD/OverviewModel.h
$$PWD/OverviewModel.h \
$$PWD/TokenCache.h
SOURCES += \
$$PWD/Icons.cpp \
......@@ -23,7 +24,8 @@ SOURCES += \
$$PWD/TokenUnderCursor.cpp \
$$PWD/BackwardsScanner.cpp \
$$PWD/MatchingText.cpp \
$$PWD/OverviewModel.cpp
$$PWD/OverviewModel.cpp \
$$PWD/TokenCache.cpp
}
HEADERS += \
......
......@@ -56,6 +56,7 @@
#include <cplusplus/BackwardsScanner.h>
#include <cplusplus/FastPreprocessor.h>
#include <cplusplus/CheckUndefinedSymbols.h>
#include <cplusplus/TokenCache.h>
#include <cpptools/cppmodelmanagerinterface.h>
......@@ -534,7 +535,7 @@ struct FindCanonicalSymbol
SemanticInfo info;
FindCanonicalSymbol(CPPEditor *editor, const SemanticInfo &info)
: editor(editor), info(info)
: editor(editor), expressionUnderCursor(editor->tokenCache()), info(info)
{
typeOfExpression.init(info.doc, info.snapshot);
}
......@@ -772,6 +773,11 @@ void CPPEditor::cut()
finishRename();
}
TokenCache *CPPEditor::tokenCache() const
{
return m_modelManager->tokenCache(editableInterface());
}
void CPPEditor::startRename()
{
m_inRenameChanged = false;
......@@ -1251,7 +1257,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(true);
const QString blockText = cursor.block().text();
const QList<SimpleToken> tokens = tokenize(blockText, BackwardsScanner::previousBlockState(cursor.block()));
const QList<SimpleToken> tokens = tokenize(blockText, TokenCache::previousBlockState(cursor.block()));
bool recognizedQtMethod = false;
......@@ -1301,7 +1307,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
static TokenUnderCursor tokenUnderCursor;
QTextBlock block;
const SimpleToken tk = tokenUnderCursor(tc, &block);
const SimpleToken tk = tokenUnderCursor(tokenCache(), tc, &block);
beginOfToken = block.position() + tk.begin();
endOfToken = block.position() + tk.end();
......@@ -1331,7 +1337,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
return link;
// Evaluate the type of the expression under the cursor
ExpressionUnderCursor expressionUnderCursor;
ExpressionUnderCursor expressionUnderCursor(tokenCache());
const QString expression = expressionUnderCursor(tc);
TypeOfExpression typeOfExpression;
......@@ -1430,13 +1436,13 @@ bool CPPEditor::isElectricCharacter(const QChar &ch) const
QString CPPEditor::insertMatchingBrace(const QTextCursor &tc, const QString &text,
const QChar &la, int *skippedChars) const
{
MatchingText m;
MatchingText m(tokenCache());
return m.insertMatchingBrace(tc, text, la, skippedChars);
}
QString CPPEditor::insertParagraphSeparator(const QTextCursor &tc) const
{
MatchingText m;
MatchingText m(tokenCache());
return m.insertParagraphSeparator(tc);
}
......@@ -1460,7 +1466,7 @@ bool CPPEditor::contextAllowsAutoParentheses(const QTextCursor &cursor,
bool CPPEditor::isInComment(const QTextCursor &cursor) const
{
CPlusPlus::TokenUnderCursor tokenUnderCursor;
const SimpleToken tk = tokenUnderCursor(cursor);
const SimpleToken tk = tokenUnderCursor(tokenCache(), cursor);
if (tk.isComment()) {
const int pos = cursor.selectionEnd() - cursor.block().position();
......@@ -1515,7 +1521,7 @@ void CPPEditor::indentBlock(QTextDocument *doc, QTextBlock block, QChar typedCha
const TabSettings &ts = tabSettings();
BackwardsScanner tk(tc, QString(), 400);
BackwardsScanner tk(tokenCache(), tc, 400);
const int tokenCount = tk.startToken();
if (tokenCount != 0) {
......
......@@ -48,6 +48,7 @@ QT_END_NAMESPACE
namespace CPlusPlus {
class OverviewModel;
class Symbol;
class TokenCache;
}
namespace CppTools {
......@@ -198,6 +199,8 @@ public:
virtual void paste(); // reimplemented from BaseTextEditor
virtual void cut(); // reimplemented from BaseTextEditor
CPlusPlus::TokenCache *tokenCache() const;
public Q_SLOTS:
virtual void setFontSettings(const TextEditor::FontSettings &);
void setSortedMethodOverview(bool sort);
......
......@@ -62,6 +62,7 @@ void CppHighlighter::highlightBlock(const QString &text)
tokenize.setObjCEnabled(false);
int initialState = state;
// qDebug() << currentBlock().document()->revision()<<"CppHighlighter::highlightBlock for block#" << currentBlock().blockNumber();
const QList<SimpleToken> tokens = tokenize(text, initialState);
state = tokenize.state(); // refresh the state
......
......@@ -254,7 +254,7 @@ void CppHoverHandler::updateHelpIdAndTooltip(TextEditor::ITextEditor *editor, in
}
// Fetch the expression's code
ExpressionUnderCursor expressionUnderCursor;
ExpressionUnderCursor expressionUnderCursor(m_modelManager->tokenCache(editor));
const QString expression = expressionUnderCursor(tc);
const QList<LookupItem> types = typeOfExpression(expression, scope);
......
......@@ -37,6 +37,10 @@ class QHelpEngineCore;
class QPoint;
QT_END_NAMESPACE
namespace CPlusPlus {
class TokenCache;
}
namespace Core {
class IEditor;
}
......
......@@ -65,7 +65,6 @@
#include <utils/faketooltip.h>
#include <utils/qtcassert.h>
#include <QtCore/QDebug>
#include <QtCore/QMap>
#include <QtCore/QFile>
#include <QtGui/QAction>
......@@ -454,7 +453,8 @@ QIcon CppCodeCompletion::iconForSymbol(Symbol *symbol) const
/*
Searches backwards for an access operator.
*/
static int startOfOperator(TextEditor::ITextEditable *editor,
static int startOfOperator(TokenCache *tokenCache,
TextEditor::ITextEditable *editor,
int pos, unsigned *kind,
bool wantFunctionCall)
{
......@@ -547,7 +547,7 @@ static int startOfOperator(TextEditor::ITextEditable *editor,
}
if (completionKind == T_COMMA) {
ExpressionUnderCursor expressionUnderCursor;
ExpressionUnderCursor expressionUnderCursor(tokenCache);
if (expressionUnderCursor.startOfFunctionCall(tc) == -1) {
completionKind = T_EOF_SYMBOL;
start = pos;
......@@ -555,7 +555,7 @@ static int startOfOperator(TextEditor::ITextEditable *editor,
}
static CPlusPlus::TokenUnderCursor tokenUnderCursor;
const SimpleToken tk = tokenUnderCursor(tc);
const SimpleToken tk = tokenUnderCursor(tokenCache, tc);
if (completionKind == T_DOXY_COMMENT && !(tk.is(T_DOXY_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))) {
completionKind = T_EOF_SYMBOL;
......@@ -634,9 +634,10 @@ int CppCodeCompletion::startPosition() const
bool CppCodeCompletion::triggersCompletion(TextEditor::ITextEditable *editor)
{
const int pos = editor->position();
TokenCache *tokenCache = m_manager->tokenCache(editor);
unsigned token = T_EOF_SYMBOL;
if (startOfOperator(editor, pos, &token, /*want function call=*/ true) != pos) {
if (startOfOperator(tokenCache, editor, pos, &token, /*want function call=*/ true) != pos) {
if (token == T_POUND) {
if (TextEditor::BaseTextEditor *edit = qobject_cast<TextEditor::BaseTextEditor *>(editor->widget())) {
QTextCursor tc(edit->document());
......@@ -684,7 +685,8 @@ int CppCodeCompletion::startCompletionHelper(TextEditor::ITextEditable *editor)
while (editor->characterAt(endOfOperator - 1).isSpace())
--endOfOperator;
int endOfExpression = startOfOperator(editor, endOfOperator,
TokenCache *tokenCache = m_manager->tokenCache(editor);
int endOfExpression = startOfOperator(tokenCache, editor, endOfOperator,
&m_completionOperator,
/*want function call =*/ true);
......@@ -725,7 +727,7 @@ int CppCodeCompletion::startCompletionHelper(TextEditor::ITextEditable *editor)
return m_startPosition;
}
ExpressionUnderCursor expressionUnderCursor;
ExpressionUnderCursor expressionUnderCursor(m_manager->tokenCache(editor));
QTextCursor tc(edit->document());
if (m_completionOperator == T_COMMA) {
......@@ -800,13 +802,13 @@ int CppCodeCompletion::startCompletionInternal(TextEditor::BaseTextEditor *edit,
}
}
if (debug)
qDebug() << "scope:" << scope->owner()->fileName() << scope->owner()->line() << scope->owner()->column();
// if (debug)
// qDebug() << "scope:" << scope->owner()->fileName() << scope->owner()->line() << scope->owner()->column();
QList<LookupItem> results = typeOfExpression(expression, scope, TypeOfExpression::Preprocess);
if (debug)
qDebug() << "got:" << results.size() << "results";
// if (debug)