Commit e3e8b1a5 authored by Erik Verbruggen's avatar Erik Verbruggen

Removed the TokenCache.

parent bb8aed62
......@@ -27,48 +27,27 @@
**
**************************************************************************/
#include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h>
#include <QtGui/QTextCursor>
#include <QTextDocument>
using namespace CPlusPlus;
BackwardsScanner::BackwardsScanner(TokenCache *tokenCache, const QTextCursor &cursor, int maxBlockCount, const QString &suffix)
: _tokenCache(tokenCache)
, _offset(0)
BackwardsScanner::BackwardsScanner(const QTextCursor &cursor, int maxBlockCount, const QString &suffix)
: _offset(0)
, _blocksTokenized(0)
, _block(cursor.block())
, _maxBlockCount(maxBlockCount)
{
int pos = cursor.position() - cursor.block().position();
_text = _block.text().left(pos);
_text += suffix;
_tokenize.setQtMocRunEnabled(true);
_tokenize.setSkipComments(true);
_tokenize.setObjCEnabled(true);
_text = _block.text().left(cursor.position() - cursor.block().position());
_tokens.append(tokenCache->tokensForBlock(_block));
if (! suffix.isEmpty())
_text += suffix;
for (int i = _tokens.size() - 1; i >= 0; --i) {
const int tokenEnd = _tokens.at(i).end();
if ((tokenEnd < pos) ||
(tokenEnd == pos && suffix.isEmpty())) {
break;
} else {
_tokens.removeAt(i);
}
}
QString remainingText;
if (!_tokens.isEmpty())
remainingText = _text.mid(_tokens.last().end());
if (!remainingText.isEmpty()) {
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(true);
tokenize.setSkipComments(true);
tokenize.setObjCEnabled(true);
_tokens.append(tokenize(remainingText, TokenCache::previousBlockState(_block)));
}
_tokens.append(_tokenize(_text, previousBlockState(_block)));
_startToken = _tokens.size();
}
......@@ -90,7 +69,6 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
} else {
++_blocksTokenized;
QList<SimpleToken> newTokens = _tokenCache->tokensForBlock(_block);
QString blockText = _block.text();
_text.prepend(QLatin1Char('\n'));
_text.prepend(blockText);
......@@ -102,8 +80,8 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
adaptedTokens.append(t);
}
_tokens = newTokens;
_offset += newTokens.size();
_tokens = _tokenize(blockText, previousBlockState(_block));
_offset += _tokens.size();
_tokens += adaptedTokens;
}
}
......@@ -260,3 +238,18 @@ QString BackwardsScanner::indentationString(int index) const
const int newlinePos = qMax(0, _text.lastIndexOf(QLatin1Char('\n'), tokenAfterNewline.position()));
return _text.mid(newlinePos, tokenAfterNewline.position() - newlinePos);
}
int BackwardsScanner::previousBlockState(const QTextBlock &block)
{
const QTextBlock prevBlock = block.previous();
if (prevBlock.isValid()) {
int state = prevBlock.userState();
if (state != -1)
return state;
}
return 0;
}
......@@ -36,15 +36,12 @@
namespace CPlusPlus {
class TokenCache;
class CPLUSPLUS_EXPORT BackwardsScanner
{
enum { MAX_BLOCK_COUNT = 10 };
public:
BackwardsScanner(TokenCache *cache,
const QTextCursor &cursor,
BackwardsScanner(const QTextCursor &cursor,
int maxBlockCount = MAX_BLOCK_COUNT,
const QString &suffix = QString());
......@@ -71,15 +68,17 @@ public:
int size() const;
static int previousBlockState(const QTextBlock &block);
private:
const SimpleToken &fetchToken(int tokenIndex);
private:
TokenCache *_tokenCache;
QList<SimpleToken> _tokens;
int _offset;
int _blocksTokenized;
QTextBlock _block;
SimpleLexer _tokenize;
QString _text;
int _maxBlockCount;
int _startToken;
......
......@@ -30,7 +30,6 @@
#include "ExpressionUnderCursor.h"
#include "SimpleLexer.h"
#include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h>
#include <QTextCursor>
......@@ -38,8 +37,8 @@
using namespace CPlusPlus;
ExpressionUnderCursor::ExpressionUnderCursor(TokenCache *tokenCache)
: _tokenCache(tokenCache), _jumpedComma(false)
ExpressionUnderCursor::ExpressionUnderCursor()
: _jumpedComma(false)
{ }
ExpressionUnderCursor::~ExpressionUnderCursor()
......@@ -219,7 +218,7 @@ bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk)
QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
{
BackwardsScanner scanner(_tokenCache, cursor);
BackwardsScanner scanner(cursor);
_jumpedComma = false;
......@@ -233,7 +232,7 @@ QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
{
BackwardsScanner scanner(_tokenCache, cursor);
BackwardsScanner scanner(cursor);
int index = scanner.startToken();
......
......@@ -43,12 +43,11 @@ namespace CPlusPlus {
class BackwardsScanner;
class SimpleToken;
class TokenCache;
class CPLUSPLUS_EXPORT ExpressionUnderCursor
{
public:
ExpressionUnderCursor(TokenCache *tokenCache);
ExpressionUnderCursor();
~ExpressionUnderCursor();
QString operator()(const QTextCursor &cursor);
......@@ -60,7 +59,6 @@ private:
bool isAccessToken(const SimpleToken &tk);
private:
TokenCache *_tokenCache;
bool _jumpedComma;
};
......
......@@ -28,7 +28,6 @@
**************************************************************************/
#include "MatchingText.h"
#include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h>
......@@ -76,10 +75,6 @@ static bool isCompleteCharLiteral(const BackwardsScanner &tk, int index)
return false;
}
MatchingText::MatchingText(TokenCache *tokenCache)
: _tokenCache(tokenCache)
{ }
bool MatchingText::shouldInsertMatchingText(const QTextCursor &tc)
{
QTextDocument *doc = tc.document();
......@@ -153,7 +148,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri
if (text.isEmpty() || !shouldInsertMatchingText(la))
return QString();
BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES, textToProcess.left(*skippedChars));
BackwardsScanner tk(tc, MAX_NUM_LINES, textToProcess.left(*skippedChars));
const int startToken = tk.startToken();
int index = startToken;
......@@ -213,7 +208,7 @@ bool MatchingText::shouldInsertNewline(const QTextCursor &tc) const
QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
{
BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES);
BackwardsScanner tk(tc, MAX_NUM_LINES);
int index = tk.startToken();
if (tk[index - 1].isNot(T_LBRACE))
......
......@@ -40,8 +40,6 @@ class TokenCache;
class CPLUSPLUS_EXPORT MatchingText
{
public:
MatchingText(TokenCache *tokenCache);
static bool shouldInsertMatchingText(const QTextCursor &tc);
static bool shouldInsertMatchingText(QChar lookAhead);
......@@ -51,8 +49,6 @@ public:
private:
bool shouldInsertNewline(const QTextCursor &tc) const;
TokenCache *_tokenCache;
};
} // end of namespace CPlusPlus
......
......@@ -170,4 +170,36 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
return tokens;
}
int SimpleLexer::tokenAt(const QList<SimpleToken> &tokens, int offset)
{
for (int index = tokens.size() - 1; index >= 0; --index) {
const SimpleToken &tk = tokens.at(index);
if (tk.position() <= offset && tk.end() >= offset)
return index;
}
return -1;
}
SimpleToken SimpleLexer::tokenAt(const QString &text,
int offset,
int state,
bool qtMocRunEnabled)
{
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(qtMocRunEnabled);
const QList<SimpleToken> tokens = tokenize(text, state);
const int tokenIdx = tokenAt(tokens, offset);
return (tokenIdx == -1) ? SimpleToken() : tokens.at(tokenIdx);
}
int SimpleLexer::tokenBefore(const QList<SimpleToken> &tokens, int offset)
{
for (int index = tokens.size() - 1; index >= 0; --index) {
const SimpleToken &tk = tokens.at(index);
if (tk.position() <= offset)
return index;
}
return -1;
}
......@@ -126,6 +126,11 @@ public:
int state() const
{ return _lastState; }
static int tokenAt(const QList<SimpleToken> &tokens, int offset);
static SimpleToken tokenAt(const QString &text, int offset, int state, bool qtMocRunEnabled = false);
static int tokenBefore(const QList<SimpleToken> &tokens, int offset);
private:
int _lastState;
bool _skipComments: 1;
......
#include "SimpleLexer.h"
#include "TokenCache.h"
using namespace CPlusPlus;
TokenCache::TokenCache(QTextDocument *doc)
: m_doc(doc)
, m_revision(-1)
{}
void TokenCache::setDocument(QTextDocument *doc)
{
m_doc = doc;
m_revision = -1;
}
QList<SimpleToken> TokenCache::tokensForBlock(const QTextBlock &block) const
{
Q_ASSERT(m_doc);
Q_ASSERT(m_doc == block.document());
const int documentRevision = m_doc->revision();
if (documentRevision != m_revision) {
m_tokensByBlock.clear();
m_revision = documentRevision;
}
const int blockNr = block.blockNumber();
if (m_tokensByBlock.contains(blockNr)) {
return m_tokensByBlock.value(blockNr);
} else {
SimpleLexer tokenize;
tokenize.setObjCEnabled(true);
tokenize.setQtMocRunEnabled(true);
tokenize.setSkipComments(false);
const int prevState = previousBlockState(block);
QList<SimpleToken> tokens = tokenize(block.text(), prevState);
m_tokensByBlock.insert(blockNr, tokens);
return tokens;
}
}
SimpleToken TokenCache::tokenUnderCursor(const QTextCursor &cursor) const
{
const QTextBlock block = cursor.block();
const QList<SimpleToken> tokens = tokensForBlock(block);
const int column = cursor.position() - block.position();
for (int index = tokens.size() - 1; index >= 0; --index) {
const SimpleToken &tk = tokens.at(index);
if (tk.position() < column)
return tk;
}
return SimpleToken();
}
QString TokenCache::text(const QTextBlock &block, int tokenIndex) const
{
SimpleToken tk = tokensForBlock(block).at(tokenIndex);
return block.text().mid(tk.position(), tk.length());
}
int TokenCache::previousBlockState(const QTextBlock &block)
{
const QTextBlock prevBlock = block.previous();
if (prevBlock.isValid()) {
int state = prevBlock.userState();
if (state != -1)
return state;
}
return 0;
}
#ifndef TOKENCACHE_H
#define TOKENCACHE_H
#include "SimpleLexer.h"
#include <CPlusPlusForwardDeclarations.h>
#include <QtCore/QHash>
#include <QtCore/QList>
#include <QtGui/QTextBlock>
#include <QtGui/QTextCursor>
#include <QtGui/QTextDocument>
namespace CPlusPlus {
class CPLUSPLUS_EXPORT TokenCache
{
public:
TokenCache(QTextDocument *doc = 0);
void setDocument(QTextDocument *doc);
QList<CPlusPlus::SimpleToken> tokensForBlock(const QTextBlock &block) const;
CPlusPlus::SimpleToken tokenUnderCursor(const QTextCursor &cursor) const;
QString text(const QTextBlock &block, int tokenIndex) const;
static int previousBlockState(const QTextBlock &block);
private:
QTextDocument *m_doc;
mutable int m_revision;
mutable QHash<int, QList<CPlusPlus::SimpleToken> > m_tokensByBlock;
};
} // namespace CPlusPlus
#endif // TOKENCACHE_H
......@@ -14,16 +14,14 @@ HEADERS += \
$$PWD/ExpressionUnderCursor.h \
$$PWD/BackwardsScanner.h \
$$PWD/MatchingText.h \
$$PWD/OverviewModel.h \
$$PWD/TokenCache.h
$$PWD/OverviewModel.h
SOURCES += \
$$PWD/Icons.cpp \
$$PWD/ExpressionUnderCursor.cpp \
$$PWD/BackwardsScanner.cpp \
$$PWD/MatchingText.cpp \
$$PWD/OverviewModel.cpp \
$$PWD/TokenCache.cpp
$$PWD/OverviewModel.cpp
}
HEADERS += \
......
......@@ -55,7 +55,6 @@
#include <cplusplus/BackwardsScanner.h>
#include <cplusplus/FastPreprocessor.h>
#include <cplusplus/CheckUndefinedSymbols.h>
#include <cplusplus/TokenCache.h>
#include <cpptools/cppmodelmanagerinterface.h>
#include <cpptools/cpptoolsconstants.h>
......@@ -530,7 +529,7 @@ struct FindCanonicalSymbol
SemanticInfo info;
FindCanonicalSymbol(CPPEditor *editor, const SemanticInfo &info)
: editor(editor), expressionUnderCursor(editor->tokenCache()), info(info)
: editor(editor), info(info)
{
typeOfExpression.init(info.doc, info.snapshot);
}
......@@ -724,11 +723,6 @@ void CPPEditor::cut()
finishRename();
}
TokenCache *CPPEditor::tokenCache() const
{
return m_modelManager->tokenCache(editableInterface());
}
CppTools::CppModelManagerInterface *CPPEditor::modelManager() const
{
return m_modelManager;
......@@ -1227,7 +1221,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(true);
const QString blockText = cursor.block().text();
const QList<SimpleToken> tokens = tokenize(blockText, TokenCache::previousBlockState(cursor.block()));
const QList<SimpleToken> tokens = tokenize(blockText, BackwardsScanner::previousBlockState(cursor.block()));
bool recognizedQtMethod = false;
......@@ -1275,7 +1269,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
if (! recognizedQtMethod) {
const QTextBlock block = tc.block();
const SimpleToken tk = tokenCache()->tokenUnderCursor(tc);
const SimpleToken tk = SimpleLexer::tokenAt(block.text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(block), true);
beginOfToken = block.position() + tk.begin();
endOfToken = block.position() + tk.end();
......@@ -1305,7 +1299,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
return link;
// Evaluate the type of the expression under the cursor
ExpressionUnderCursor expressionUnderCursor(tokenCache());
ExpressionUnderCursor expressionUnderCursor;
const QString expression = expressionUnderCursor(tc);
TypeOfExpression typeOfExpression;
......@@ -1408,13 +1402,13 @@ bool CPPEditor::isElectricCharacter(QChar ch) const
QString CPPEditor::insertMatchingBrace(const QTextCursor &tc, const QString &text,
QChar la, int *skippedChars) const
{
MatchingText m(tokenCache());
MatchingText m;
return m.insertMatchingBrace(tc, text, la, skippedChars);
}
QString CPPEditor::insertParagraphSeparator(const QTextCursor &tc) const
{
MatchingText m(tokenCache());
MatchingText m;
return m.insertParagraphSeparator(tc);
}
......@@ -1437,7 +1431,7 @@ bool CPPEditor::contextAllowsAutoParentheses(const QTextCursor &cursor,
bool CPPEditor::contextAllowsElectricCharacters(const QTextCursor &cursor) const
{
const SimpleToken tk = tokenCache()->tokenUnderCursor(cursor);
const SimpleToken tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
// XXX Duplicated from CPPEditor::isInComment to avoid tokenizing twice
if (tk.isComment()) {
......@@ -1468,7 +1462,7 @@ bool CPPEditor::contextAllowsElectricCharacters(const QTextCursor &cursor) const
bool CPPEditor::isInComment(const QTextCursor &cursor) const
{
const SimpleToken tk = tokenCache()->tokenUnderCursor(cursor);
const SimpleToken tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
if (tk.isComment()) {
const int pos = cursor.selectionEnd() - cursor.block().position();
......@@ -1522,7 +1516,7 @@ void CPPEditor::indentBlock(QTextDocument *doc, QTextBlock block, QChar typedCha
const TabSettings &ts = tabSettings();
BackwardsScanner tk(tokenCache(), tc, 400);
BackwardsScanner tk(tc, 400);
const int tokenCount = tk.startToken();
if (tokenCount != 0) {
......
......@@ -48,7 +48,6 @@ QT_END_NAMESPACE
namespace CPlusPlus {
class OverviewModel;
class Symbol;
class TokenCache;
}
namespace CppTools {
......@@ -200,8 +199,6 @@ public:
virtual void paste(); // reimplemented from BaseTextEditor
virtual void cut(); // reimplemented from BaseTextEditor
CPlusPlus::TokenCache *tokenCache() const;
CppTools::CppModelManagerInterface *modelManager() const;
virtual void setMimeType(const QString &mt);
......
......@@ -232,7 +232,7 @@ void CppHoverHandler::updateHelpIdAndTooltip(TextEditor::ITextEditor *editor, in
}
// Fetch the expression's code
ExpressionUnderCursor expressionUnderCursor(m_modelManager->tokenCache(editor));
ExpressionUnderCursor expressionUnderCursor;
const QString expression = expressionUnderCursor(tc);
const QList<LookupItem> types = typeOfExpression(expression, scope);
......
......@@ -36,10 +36,6 @@ QT_BEGIN_NAMESPACE
class QPoint;
QT_END_NAMESPACE
namespace CPlusPlus {
class TokenCache;
}
namespace Core {
class IEditor;
}
......
......@@ -452,8 +452,7 @@ QIcon CppCodeCompletion::iconForSymbol(Symbol *symbol) const
/*
Searches backwards for an access operator.
*/
static int startOfOperator(TokenCache *tokenCache,
TextEditor::ITextEditable *editor,
static int startOfOperator(TextEditor::ITextEditable *editor,
int pos, unsigned *kind,
bool wantFunctionCall)
{
......@@ -546,14 +545,19 @@ static int startOfOperator(TokenCache *tokenCache,
}
if (completionKind == T_COMMA) {
ExpressionUnderCursor expressionUnderCursor(tokenCache);
ExpressionUnderCursor expressionUnderCursor;
if (expressionUnderCursor.startOfFunctionCall(tc) == -1) {
completionKind = T_EOF_SYMBOL;
start = pos;
}
}
const SimpleToken tk = tokenCache->tokenUnderCursor(tc);
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(true);
tokenize.setSkipComments(false);
const QList<SimpleToken> &tokens = tokenize(tc.block().text());
const int tokenIdx = SimpleLexer::tokenAt(tokens, tc.positionInBlock());
const SimpleToken &tk = (tokenIdx == -1) ? SimpleToken() : tokens.at(tokenIdx);
if (completionKind == T_DOXY_COMMENT && !(tk.is(T_DOXY_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))) {
completionKind = T_EOF_SYMBOL;
......@@ -573,7 +577,6 @@ static int startOfOperator(TokenCache *tokenCache,
start = pos;
}
else if (completionKind == T_LPAREN) {
const QList<SimpleToken> &tokens = tokenCache->tokensForBlock(tc.block());
int i = 0;
for (; i < tokens.size(); ++i) {
const SimpleToken &token = tokens.at(i);
......@@ -595,11 +598,12 @@ static int startOfOperator(TokenCache *tokenCache,
// Check for include preprocessor directive
else if (completionKind == T_STRING_LITERAL || completionKind == T_ANGLE_STRING_LITERAL || completionKind == T_SLASH) {
bool include = false;
const QList<SimpleToken> &tokens = tokenCache->tokensForBlock(tc.block());
if (tokens.size() >= 3) {
if (tokens.at(0).is(T_POUND) && tokens.at(1).is(T_IDENTIFIER) && (tokens.at(2).is(T_STRING_LITERAL) ||
tokens.at(2).is(T_ANGLE_STRING_LITERAL))) {
QString directive = tokenCache->text(tc.block(), 1);
const SimpleToken &directiveToken = tokens.at(1);
QString directive = tc.block().text().mid(directiveToken.position(),
directiveToken.length());
if (directive == QLatin1String("include") ||
directive == QLatin1String("include_next") ||
directive == QLatin1String("import")) {
......@@ -632,10 +636,9 @@ int CppCodeCompletion::startPosition() const
bool CppCodeCompletion::triggersCompletion(TextEditor::ITextEditable *editor)
{
const int pos = editor->position();
TokenCache *tokenCache = m_manager->tokenCache(editor);
unsigned token = T_EOF_SYMBOL;
if (startOfOperator(tokenCache, editor, pos, &token, /*want function call=*/ true) != pos) {