Revert "Introduced a token cache for the C++ editor."

This reverts commit c2393df0.

src/libs/cplusplus/BackwardsScanner.cpp

@@ -27,48 +27,33 @@
 **
 **************************************************************************/
 #include "BackwardsScanner.h"
-#include "TokenCache.h"
 #include <Token.h>
 #include <QtGui/QTextCursor>
-#include <QTextDocument>
 
 using namespace CPlusPlus;
 
-BackwardsScanner::BackwardsScanner(TokenCache *tokenCache, const QTextCursor &cursor, int maxBlockCount, const QString &suffix)
-    : _tokenCache(tokenCache)
-    , _offset(0)
+BackwardsScanner::BackwardsScanner(const QTextCursor &cursor, const QString &suffix, int maxBlockCount)
+    : _offset(0)
     , _blocksTokenized(0)
     , _block(cursor.block())
     , _maxBlockCount(maxBlockCount)
 {
-    int pos = cursor.position() - cursor.block().position();
-    _text = _block.text().left(pos);
-
-    if (suffix.isEmpty()) {
-        _tokens.append(tokenCache->tokensForBlock(_block));
-        int last = -1;
-        for (int i = _tokens.size() - 1; i >= 0; --i) {
-            if (_tokens.at(i).begin() < pos) {
-                last = i;
-                break;
-            }
-        }
-        for (int i = _tokens.size() - 1; i > last && i >= 0; --i)
-            _tokens.removeAt(i);
-    } else {
-        SimpleLexer tokenize;
-        tokenize.setQtMocRunEnabled(true);
-        tokenize.setSkipComments(true);
-        tokenize.setObjCEnabled(true);
+    _tokenize.setQtMocRunEnabled(true);
+    _tokenize.setSkipComments(true);
+    _tokenize.setObjCEnabled(true);
+    _text = _block.text().left(cursor.position() - cursor.block().position());
 
+    if (! suffix.isEmpty())
         _text += suffix;
 
-        _tokens.append(tokenize(_text, TokenCache::previousBlockState(_block)));
-    }
+    _tokens.append(_tokenize(_text, previousBlockState(_block)));
 
     _startToken = _tokens.size();
 }
 
+int BackwardsScanner::state() const
+{ return _tokenize.state(); }
+
 SimpleToken BackwardsScanner::LA(int index) const
 { return const_cast<BackwardsScanner *>(this)->fetchToken(_startToken - index); }
 
@@ -98,7 +83,7 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
                 adaptedTokens.append(t);
             }
 
-            _tokens = _tokenCache->tokensForBlock(_block);
+            _tokens = _tokenize(blockText, previousBlockState(_block));
             _offset += _tokens.size();
             _tokens += adaptedTokens;
         }
@@ -134,6 +119,20 @@ QStringRef BackwardsScanner::textRef(int index) const
     return _text.midRef(firstToken.begin(), firstToken.length());
 }
 
+int BackwardsScanner::previousBlockState(const QTextBlock &block)
+{
+    const QTextBlock prevBlock = block.previous();
+
+    if (prevBlock.isValid()) {
+        int state = prevBlock.userState();
+
+        if (state != -1)
+            return state;
+    }
+
+    return 0;
+}
+
 int BackwardsScanner::size() const
 {
     return _tokens.size();

src/libs/cplusplus/BackwardsScanner.h

@@ -36,18 +36,16 @@
 
 namespace CPlusPlus {
 
-class TokenCache;
-
 class CPLUSPLUS_EXPORT BackwardsScanner
 {
     enum { MAX_BLOCK_COUNT = 10 };
 
 public:
-    BackwardsScanner(TokenCache *cache,
-                     const QTextCursor &cursor,
-                     int maxBlockCount = MAX_BLOCK_COUNT,
-                     const QString &suffix = QString());
+    BackwardsScanner(const QTextCursor &cursor,
+                     const QString &suffix = QString(),
+                     int maxBlockCount = MAX_BLOCK_COUNT);
 
+    int state() const;
     int startToken() const;
 
     int startPosition() const;
@@ -69,18 +67,20 @@ public:
     int startOfMatchingBrace(int index) const;
     int startOfBlock(int index) const;
 
+    static int previousBlockState(const QTextBlock &block);
+
     int size() const;
 
 private:
     const SimpleToken &fetchToken(int tokenIndex);
 
 private:
-    TokenCache *_tokenCache;
     QList<SimpleToken> _tokens;
     int _offset;
     int _blocksTokenized;
     QTextBlock _block;
     QString _text;
+    SimpleLexer _tokenize;
     int _maxBlockCount;
     int _startToken;
 };

src/libs/cplusplus/ExpressionUnderCursor.cpp

@@ -30,7 +30,6 @@
 #include "ExpressionUnderCursor.h"
 #include "SimpleLexer.h"
 #include "BackwardsScanner.h"
-#include "TokenCache.h"
 #include <Token.h>
 
 #include <QTextCursor>
@@ -38,8 +37,8 @@
 
 using namespace CPlusPlus;
 
-ExpressionUnderCursor::ExpressionUnderCursor(TokenCache *tokenCache)
-    : _tokenCache(tokenCache), _jumpedComma(false)
+ExpressionUnderCursor::ExpressionUnderCursor()
+    : _jumpedComma(false)
 { }
 
 ExpressionUnderCursor::~ExpressionUnderCursor()
@@ -219,7 +218,7 @@ bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk)
 
 QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
 {
-    BackwardsScanner scanner(_tokenCache, cursor);
+    BackwardsScanner scanner(cursor);
 
     _jumpedComma = false;
 
@@ -233,7 +232,7 @@ QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
 
 int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
 {
-    BackwardsScanner scanner(_tokenCache, cursor);
+    BackwardsScanner scanner(cursor);
 
     int index = scanner.startToken();
 

src/libs/cplusplus/ExpressionUnderCursor.h

@@ -43,12 +43,11 @@ namespace CPlusPlus {
 
 class BackwardsScanner;
 class SimpleToken;
-class TokenCache;
 
 class CPLUSPLUS_EXPORT ExpressionUnderCursor
 {
 public:
-    ExpressionUnderCursor(TokenCache *tokenCache);
+    ExpressionUnderCursor();
     ~ExpressionUnderCursor();
 
     QString operator()(const QTextCursor &cursor);
@@ -60,7 +59,6 @@ private:
     bool isAccessToken(const SimpleToken &tk);
 
 private:
-    TokenCache *_tokenCache;
     bool _jumpedComma;
 };
 

src/libs/cplusplus/MatchingText.cpp

@@ -28,7 +28,6 @@
 **************************************************************************/
 #include "MatchingText.h"
 #include "BackwardsScanner.h"
-#include "TokenCache.h"
 
 #include <Token.h>
 
@@ -76,8 +75,7 @@ static bool isCompleteCharLiteral(const BackwardsScanner &tk, int index)
     return false;
 }
 
-MatchingText::MatchingText(TokenCache *tokenCache)
-    : _tokenCache(tokenCache)
+MatchingText::MatchingText()
 { }
 
 bool MatchingText::shouldInsertMatchingText(const QTextCursor &tc)
@@ -153,7 +151,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri
     if (text.isEmpty() || !shouldInsertMatchingText(la))
         return QString();
 
-    BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES, textToProcess.left(*skippedChars));
+    BackwardsScanner tk(tc, textToProcess.left(*skippedChars), MAX_NUM_LINES);
     const int startToken = tk.startToken();
     int index = startToken;
 
@@ -213,7 +211,7 @@ bool MatchingText::shouldInsertNewline(const QTextCursor &tc) const
 
 QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
 {
-    BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES);
+    BackwardsScanner tk(tc, QString(), MAX_NUM_LINES);
     int index = tk.startToken();
 
     if (tk[index - 1].isNot(T_LBRACE))

src/libs/cplusplus/MatchingText.h

@@ -35,12 +35,11 @@
 namespace CPlusPlus {
 
 class BackwardsScanner;
-class TokenCache;
 
 class CPLUSPLUS_EXPORT MatchingText
 {
 public:
-    MatchingText(TokenCache *tokenCache);
+    MatchingText();
 
     static bool shouldInsertMatchingText(const QTextCursor &tc);
     static bool shouldInsertMatchingText(const QChar &lookAhead);
@@ -51,8 +50,6 @@ public:
 
 private:
     bool shouldInsertNewline(const QTextCursor &tc) const;
-
-    TokenCache *_tokenCache;
 };
 
 } // end of namespace CPlusPlus

src/libs/cplusplus/TokenCache.cpp

-#include "SimpleLexer.h"
-#include "TokenCache.h"
-
-#include <QtCore/QDebug>
-
-using namespace CPlusPlus;
-
-TokenCache::TokenCache()
-    : m_doc(0)
-    , m_revision(-1)
-{}
-
-void TokenCache::setDocument(QTextDocument *doc)
-{
-    m_doc = doc;
-    m_revision = -1;
-}
-
-QList<SimpleToken> TokenCache::tokensForBlock(const QTextBlock &block) const
-{
-    Q_ASSERT(m_doc);
-
-    const int documentRevision = m_doc->revision();
-
-    if (documentRevision != m_revision) {
-        m_tokensByBlock.clear();
-        m_revision = documentRevision;
-//        qDebug() << "** revision changed to" << documentRevision;
-    }
-
-    const int blockNr = block.blockNumber();
-
-    if (m_tokensByBlock.contains(blockNr)) {
-//        qDebug()<<"Cache hit on line" << line;
-        return m_tokensByBlock.value(blockNr);
-    } else {
-//        qDebug()<<"Cache miss on line" << line;
-
-        SimpleLexer tokenize;
-        tokenize.setObjCEnabled(true);
-        tokenize.setQtMocRunEnabled(true);
-        tokenize.setSkipComments(false);
-
-        const int prevState = previousBlockState(block);
-        QList<SimpleToken> tokens = tokenize(block.text(), prevState);
-        m_tokensByBlock.insert(blockNr, tokens);
-
-        return tokens;
-    }
-}
-
-int TokenCache::previousBlockState(const QTextBlock &block)
-{
-    const QTextBlock prevBlock = block.previous();
-
-    if (prevBlock.isValid()) {
-        int state = prevBlock.userState();
-
-        if (state != -1)
-            return state;
-    }
-
-    return 0;
-}

src/libs/cplusplus/TokenCache.h

-#ifndef TOKENCACHE_H
-#define TOKENCACHE_H
-
-#include <CPlusPlusForwardDeclarations.h>
-#include <cplusplus/SimpleLexer.h>
-
-#include <QtCore/QHash>
-#include <QtCore/QList>
-
-#include <QtGui/QTextBlock>
-#include <QtGui/QTextDocument>
-
-namespace CPlusPlus {
-
-class CPLUSPLUS_EXPORT TokenCache
-{
-public:
-    TokenCache();
-
-    void setDocument(QTextDocument *doc);
-
-    QList<CPlusPlus::SimpleToken> tokensForBlock(const QTextBlock &block) const;
-
-    static int previousBlockState(const QTextBlock &block);
-
-private:
-    QTextDocument *m_doc;
-
-    mutable int m_revision;
-    mutable QHash<int, QList<CPlusPlus::SimpleToken> > m_tokensByBlock;
-};
-
-} // namespace CPlusPlus
-
-#endif // TOKENCACHE_H

src/libs/cplusplus/TokenUnderCursor.cpp

@@ -28,14 +28,12 @@
 **************************************************************************/
 
 #include "TokenUnderCursor.h"
-#include "TokenCache.h"
-#include "TokenCache.h"
+#include "BackwardsScanner.h"
 #include <Token.h>
 
 #include <QTextCursor>
 #include <QTextBlock>
 #include <climits>
-#include <QTextDocument>
 
 using namespace CPlusPlus;
 
@@ -45,13 +43,17 @@ TokenUnderCursor::TokenUnderCursor()
 TokenUnderCursor::~TokenUnderCursor()
 { }
 
-SimpleToken TokenUnderCursor::operator()(TokenCache *cache, const QTextCursor &cursor, QTextBlock *b)
+SimpleToken TokenUnderCursor::operator()(const QTextCursor &cursor, QTextBlock *b)
 {
+    SimpleLexer tokenize;
+    tokenize.setObjCEnabled(true);
+    tokenize.setSkipComments(false);
+
     QTextBlock block = cursor.block();
     int column = cursor.position() - cursor.block().position();
 
     _text = block.text();
-    _tokens = cache->tokensForBlock(block);
+    _tokens = tokenize(_text, BackwardsScanner::previousBlockState(block));
     for (int index = _tokens.size() - 1; index != -1; --index) {
         const SimpleToken &tk = _tokens.at(index);
         if (tk.position() < column) {

src/libs/cplusplus/TokenUnderCursor.h

@@ -40,15 +40,13 @@ QT_END_NAMESPACE
 
 namespace CPlusPlus {
 
-class TokenCache;
-
 class CPLUSPLUS_EXPORT TokenUnderCursor
 {
 public:
     TokenUnderCursor();
     ~TokenUnderCursor();
 
-    SimpleToken operator()(TokenCache *cache, const QTextCursor &cursor, QTextBlock *block = 0);
+    SimpleToken operator()(const QTextCursor &cursor, QTextBlock *block = 0);
 
     const QList<SimpleToken> &tokens() const
     { return _tokens; }

src/libs/cplusplus/cplusplus-lib.pri

@@ -15,8 +15,7 @@ HEADERS += \
     $$PWD/TokenUnderCursor.h \
     $$PWD/BackwardsScanner.h \
     $$PWD/MatchingText.h \
-    $$PWD/OverviewModel.h \
-    $$PWD/TokenCache.h
+    $$PWD/OverviewModel.h
 
 SOURCES += \
     $$PWD/Icons.cpp \
@@ -24,8 +23,7 @@ SOURCES += \
     $$PWD/TokenUnderCursor.cpp \
     $$PWD/BackwardsScanner.cpp \
     $$PWD/MatchingText.cpp \
-    $$PWD/OverviewModel.cpp \
-    $$PWD/TokenCache.cpp
+    $$PWD/OverviewModel.cpp
 }
 
 HEADERS += \

src/plugins/cppeditor/cppeditor.cpp

@@ -56,7 +56,6 @@
 #include <cplusplus/BackwardsScanner.h>
 #include <cplusplus/FastPreprocessor.h>
 #include <cplusplus/CheckUndefinedSymbols.h>
-#include <cplusplus/TokenCache.h>
 
 #include <cpptools/cppmodelmanagerinterface.h>
 
@@ -535,7 +534,7 @@ struct FindCanonicalSymbol
     SemanticInfo info;
 
     FindCanonicalSymbol(CPPEditor *editor, const SemanticInfo &info)
-        : editor(editor), expressionUnderCursor(editor->tokenCache()), info(info)
+        : editor(editor), info(info)
     {
         typeOfExpression.init(info.doc, info.snapshot);
     }
@@ -773,11 +772,6 @@ void CPPEditor::cut()
     finishRename();
 }
 
-TokenCache *CPPEditor::tokenCache() const
-{
-    return m_modelManager->tokenCache(editableInterface());
-}
-
 void CPPEditor::startRename()
 {
     m_inRenameChanged = false;
@@ -1257,7 +1251,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
     SimpleLexer tokenize;
     tokenize.setQtMocRunEnabled(true);
     const QString blockText = cursor.block().text();
-    const QList<SimpleToken> tokens = tokenize(blockText, TokenCache::previousBlockState(cursor.block()));
+    const QList<SimpleToken> tokens = tokenize(blockText, BackwardsScanner::previousBlockState(cursor.block()));
 
     bool recognizedQtMethod = false;
 
@@ -1307,7 +1301,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
         static TokenUnderCursor tokenUnderCursor;
 
         QTextBlock block;
-        const SimpleToken tk = tokenUnderCursor(tokenCache(), tc, &block);
+        const SimpleToken tk = tokenUnderCursor(tc, &block);
 
         beginOfToken = block.position() + tk.begin();
         endOfToken = block.position() + tk.end();
@@ -1337,7 +1331,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
         return link;
 
     // Evaluate the type of the expression under the cursor
-    ExpressionUnderCursor expressionUnderCursor(tokenCache());
+    ExpressionUnderCursor expressionUnderCursor;
     const QString expression = expressionUnderCursor(tc);
 
     TypeOfExpression typeOfExpression;
@@ -1436,13 +1430,13 @@ bool CPPEditor::isElectricCharacter(const QChar &ch) const
 QString CPPEditor::insertMatchingBrace(const QTextCursor &tc, const QString &text,
                                        const QChar &la, int *skippedChars) const
 {
-    MatchingText m(tokenCache());
+    MatchingText m;
     return m.insertMatchingBrace(tc, text, la, skippedChars);
 }
 
 QString CPPEditor::insertParagraphSeparator(const QTextCursor &tc) const
 {
-    MatchingText m(tokenCache());
+    MatchingText m;
     return m.insertParagraphSeparator(tc);
 }
 
@@ -1466,7 +1460,7 @@ bool CPPEditor::contextAllowsAutoParentheses(const QTextCursor &cursor,
 bool CPPEditor::isInComment(const QTextCursor &cursor) const
 {
     CPlusPlus::TokenUnderCursor tokenUnderCursor;
-    const SimpleToken tk = tokenUnderCursor(tokenCache(), cursor);
+    const SimpleToken tk = tokenUnderCursor(cursor);
 
     if (tk.isComment()) {
         const int pos = cursor.selectionEnd() - cursor.block().position();
@@ -1521,7 +1515,7 @@ void CPPEditor::indentBlock(QTextDocument *doc, QTextBlock block, QChar typedCha
 
     const TabSettings &ts = tabSettings();
 
-    BackwardsScanner tk(tokenCache(), tc, 400);
+    BackwardsScanner tk(tc, QString(), 400);
     const int tokenCount = tk.startToken();
 
     if (tokenCount != 0) {

src/plugins/cppeditor/cppeditor.h

@@ -48,7 +48,6 @@ QT_END_NAMESPACE
 namespace CPlusPlus {
 class OverviewModel;
 class Symbol;
-class TokenCache;
 }
 
 namespace CppTools {
@@ -199,8 +198,6 @@ public:
     virtual void paste(); // reimplemented from BaseTextEditor
     virtual void cut(); // reimplemented from BaseTextEditor
 
-    CPlusPlus::TokenCache *tokenCache() const;
-
 public Q_SLOTS:
     virtual void setFontSettings(const TextEditor::FontSettings &);
     void setSortedMethodOverview(bool sort);

src/plugins/cppeditor/cpphighlighter.cpp

@@ -62,7 +62,6 @@ void CppHighlighter::highlightBlock(const QString &text)
     tokenize.setObjCEnabled(false);
 
     int initialState = state;
-//    qDebug() << currentBlock().document()->revision()<<"CppHighlighter::highlightBlock for block#" << currentBlock().blockNumber();
     const QList<SimpleToken> tokens = tokenize(text, initialState);
     state = tokenize.state(); // refresh the state
 

src/plugins/cppeditor/cpphoverhandler.cpp

@@ -254,7 +254,7 @@ void CppHoverHandler::updateHelpIdAndTooltip(TextEditor::ITextEditor *editor, in
         }
 
         // Fetch the expression's code
-        ExpressionUnderCursor expressionUnderCursor(m_modelManager->tokenCache(editor));
+        ExpressionUnderCursor expressionUnderCursor;
         const QString expression = expressionUnderCursor(tc);
 
         const QList<LookupItem> types = typeOfExpression(expression, scope);

src/plugins/cppeditor/cpphoverhandler.h

@@ -37,10 +37,6 @@ class QHelpEngineCore;
 class QPoint;
 QT_END_NAMESPACE
 
-namespace CPlusPlus {
-class TokenCache;
-}
-
 namespace Core {
 class IEditor;
 }

src/plugins/cpptools/cppcodecompletion.cpp

@@ -65,6 +65,7 @@
 #include <utils/faketooltip.h>
 #include <utils/qtcassert.h>
 
+#include <QtCore/QDebug>
 #include <QtCore/QMap>
 #include <QtCore/QFile>
 #include <QtGui/QAction>
@@ -453,8 +454,7 @@ QIcon CppCodeCompletion::iconForSymbol(Symbol *symbol) const
 /*
   Searches backwards for an access operator.
 */
-static int startOfOperator(TokenCache *tokenCache,
-                           TextEditor::ITextEditable *editor,
+static int startOfOperator(TextEditor::ITextEditable *editor,
                            int pos, unsigned *kind,
                            bool wantFunctionCall)
 {
@@ -547,7 +547,7 @@ static int startOfOperator(TokenCache *tokenCache,
     }
 
     if (completionKind == T_COMMA) {
-        ExpressionUnderCursor expressionUnderCursor(tokenCache);
+        ExpressionUnderCursor expressionUnderCursor;
         if (expressionUnderCursor.startOfFunctionCall(tc) == -1) {
             completionKind = T_EOF_SYMBOL;
             start = pos;
@@ -555,7 +555,7 @@ static int startOfOperator(TokenCache *tokenCache,
     }
 
     static CPlusPlus::TokenUnderCursor tokenUnderCursor;
-    const SimpleToken tk = tokenUnderCursor(tokenCache, tc);
+    const SimpleToken tk = tokenUnderCursor(tc);
 
     if (completionKind == T_DOXY_COMMENT && !(tk.is(T_DOXY_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))) {
         completionKind = T_EOF_SYMBOL;
@@ -634,10 +634,9 @@ int CppCodeCompletion::startPosition() const
 bool CppCodeCompletion::triggersCompletion(TextEditor::ITextEditable *editor)
 {
     const int pos = editor->position();
-    TokenCache *tokenCache = m_manager->tokenCache(editor);
     unsigned token = T_EOF_SYMBOL;
 
-    if (startOfOperator(tokenCache, editor, pos, &token, /*want function call=*/ true) != pos) {
+    if (startOfOperator(editor, pos, &token, /*want function call=*/ true) != pos) {
         if (token == T_POUND) {
             if (TextEditor::BaseTextEditor *edit = qobject_cast<TextEditor::BaseTextEditor *>(editor->widget())) {
                 QTextCursor tc(edit->document());
@@ -685,8 +684,7 @@ int CppCodeCompletion::startCompletionHelper(TextEditor::ITextEditable *editor)
     while (editor->characterAt(endOfOperator - 1).isSpace())
         --endOfOperator;
 
-    TokenCache *tokenCache = m_manager->tokenCache(editor);
-    int endOfExpression = startOfOperator(tokenCache, editor, endOfOperator,
+    int endOfExpression = startOfOperator(editor, endOfOperator,
                                           &m_completionOperator,
                                           /*want function call =*/ true);
 
@@ -727,7 +725,7 @@ int CppCodeCompletion::startCompletionHelper(TextEditor::ITextEditable *editor)
         return m_startPosition;
     }
 
-    ExpressionUnderCursor expressionUnderCursor(m_manager->tokenCache(editor));
+    ExpressionUnderCursor expressionUnderCursor;
     QTextCursor tc(edit->document());
 
     if (m_completionOperator == T_COMMA) {
@@ -802,13 +800,13 @@ int CppCodeCompletion::startCompletionInternal(TextEditor::BaseTextEditor *edit,
         }
     }
 
-//    if (debug)
-//        qDebug() << "scope:" << scope->owner()->fileName() << scope->owner()->line() << scope->owner()->column();
+    if (debug)
+        qDebug() << "scope:" << scope->owner()->fileName() << scope->owner()->line() << scope->owner()->column();
 
     QList<LookupItem> results = typeOfExpression(expression, scope, TypeOfExpression::Preprocess);
 
-//    if (debug)
-//        qDebug() << "got:" << results.size() << "results";
+    if (debug)