From 8e4fb678fd68cbb01d5548ba07dfcb2927868df4 Mon Sep 17 00:00:00 2001
From: Erik Verbruggen <erik.verbruggen@nokia.com>
Date: Tue, 29 Jun 2010 17:57:15 +0200
Subject: [PATCH] Removing SimpleToken
---
src/libs/cplusplus/BackwardsScanner.cpp | 33 ++++----
src/libs/cplusplus/BackwardsScanner.h | 8 +-
src/libs/cplusplus/ExpressionUnderCursor.cpp | 8 +-
src/libs/cplusplus/ExpressionUnderCursor.h | 3 +-
src/libs/cplusplus/MatchingText.cpp | 8 +-
src/libs/cplusplus/SimpleLexer.cpp | 80 +++++---------------
src/libs/cplusplus/SimpleLexer.h | 78 ++-----------------
src/plugins/cppeditor/cppeditor.cpp | 24 +++---
src/plugins/cppeditor/cpphighlighter.cpp | 58 +++++++-------
src/plugins/cpptools/cppcodecompletion.cpp | 18 ++---
10 files changed, 105 insertions(+), 213 deletions(-)
diff --git a/src/libs/cplusplus/BackwardsScanner.cpp b/src/libs/cplusplus/BackwardsScanner.cpp
index e6c31691769..93db1209b94 100644
--- a/src/libs/cplusplus/BackwardsScanner.cpp
+++ b/src/libs/cplusplus/BackwardsScanner.cpp
@@ -52,19 +52,19 @@ BackwardsScanner::BackwardsScanner(const QTextCursor &cursor, int maxBlockCount,
_startToken = _tokens.size();
}
-SimpleToken BackwardsScanner::LA(int index) const
+Token BackwardsScanner::LA(int index) const
{ return const_cast<BackwardsScanner *>(this)->fetchToken(_startToken - index); }
-SimpleToken BackwardsScanner::operator[](int index) const
+Token BackwardsScanner::operator[](int index) const
{ return const_cast<BackwardsScanner *>(this)->fetchToken(index); }
-const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
+const Token &BackwardsScanner::fetchToken(int tokenIndex)
{
while (_offset + tokenIndex < 0) {
_block = _block.previous();
if (_blocksTokenized == _maxBlockCount || !_block.isValid()) {
++_offset;
- _tokens.prepend(SimpleToken()); // sentinel
+ _tokens.prepend(Token()); // sentinel
break;
} else {
++_blocksTokenized;
@@ -73,10 +73,10 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
_text.prepend(QLatin1Char('\n'));
_text.prepend(blockText);
- QList<SimpleToken> adaptedTokens;
+ QList<Token> adaptedTokens;
for (int i = 0; i < _tokens.size(); ++i) {
- SimpleToken t = _tokens.at(i);
- t.setPosition(t.position() + blockText.length() + 1);
+ Token t = _tokens.at(i);
+ t.offset += + blockText.length() + 1;
adaptedTokens.append(t);
}
@@ -100,19 +100,19 @@ QString BackwardsScanner::text() const
QString BackwardsScanner::mid(int index) const
{
- const SimpleToken &firstToken = _tokens.at(index + _offset);
+ const Token &firstToken = _tokens.at(index + _offset);
return _text.mid(firstToken.begin());
}
QString BackwardsScanner::text(int index) const
{
- const SimpleToken &firstToken = _tokens.at(index + _offset);
+ const Token &firstToken = _tokens.at(index + _offset);
return _text.mid(firstToken.begin(), firstToken.length());
}
QStringRef BackwardsScanner::textRef(int index) const
{
- const SimpleToken &firstToken = _tokens.at(index + _offset);
+ const Token &firstToken = _tokens.at(index + _offset);
return _text.midRef(firstToken.begin(), firstToken.length());
}
@@ -181,11 +181,11 @@ int BackwardsScanner::startOfLine(int index) const
const BackwardsScanner tk(*this);
forever {
- const SimpleToken &tok = tk[index - 1];
+ const Token &tok = tk[index - 1];
if (tok.is(T_EOF_SYMBOL))
break;
- else if (tok.followsNewline())
+ else if (tok.newline())
return index - 1;
--index;
@@ -201,7 +201,7 @@ int BackwardsScanner::startOfBlock(int index) const
const int start = index;
forever {
- SimpleToken token = tk[index - 1];
+ Token token = tk[index - 1];
if (token.is(T_EOF_SYMBOL)) {
break;
@@ -234,9 +234,10 @@ int BackwardsScanner::startOfBlock(int index) const
QString BackwardsScanner::indentationString(int index) const
{
- const SimpleToken tokenAfterNewline = operator[](startOfLine(index + 1));
- const int newlinePos = qMax(0, _text.lastIndexOf(QLatin1Char('\n'), tokenAfterNewline.position()));
- return _text.mid(newlinePos, tokenAfterNewline.position() - newlinePos);
+ const Token tokenAfterNewline = operator[](startOfLine(index + 1));
+ const int newlinePos = qMax(0, _text.lastIndexOf(QLatin1Char('\n'),
+ tokenAfterNewline.begin()));
+ return _text.mid(newlinePos, tokenAfterNewline.begin() - newlinePos);
}
diff --git a/src/libs/cplusplus/BackwardsScanner.h b/src/libs/cplusplus/BackwardsScanner.h
index e348c72a27d..26b45f5667c 100644
--- a/src/libs/cplusplus/BackwardsScanner.h
+++ b/src/libs/cplusplus/BackwardsScanner.h
@@ -55,10 +55,10 @@ public:
QString text(int index) const;
QStringRef textRef(int index) const;
// 1-based
- SimpleToken LA(int index) const;
+ Token LA(int index) const;
// n-la token is [startToken - n]
- SimpleToken operator[](int index) const; // ### deprecate
+ Token operator[](int index) const; // ### deprecate
QString indentationString(int index) const;
@@ -71,10 +71,10 @@ public:
static int previousBlockState(const QTextBlock &block);
private:
- const SimpleToken &fetchToken(int tokenIndex);
+ const Token &fetchToken(int tokenIndex);
private:
- QList<SimpleToken> _tokens;
+ QList<Token> _tokens;
int _offset;
int _blocksTokenized;
QTextBlock _block;
diff --git a/src/libs/cplusplus/ExpressionUnderCursor.cpp b/src/libs/cplusplus/ExpressionUnderCursor.cpp
index c840ad398dd..d70bf3428eb 100644
--- a/src/libs/cplusplus/ExpressionUnderCursor.cpp
+++ b/src/libs/cplusplus/ExpressionUnderCursor.cpp
@@ -56,7 +56,7 @@ int ExpressionUnderCursor::startOfExpression(BackwardsScanner &tk, int index)
index = startOfExpression_helper(tk, index);
if (_jumpedComma) {
- const SimpleToken &tok = tk[index - 1];
+ const Token &tok = tk[index - 1];
switch (tok.kind()) {
case T_COMMA:
@@ -204,7 +204,7 @@ int ExpressionUnderCursor::startOfExpression_helper(BackwardsScanner &tk, int in
return index;
}
-bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk)
+bool ExpressionUnderCursor::isAccessToken(const Token &tk)
{
switch (tk.kind()) {
case T_COLON_COLON:
@@ -237,12 +237,12 @@ int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
int index = scanner.startToken();
forever {
- const SimpleToken &tk = scanner[index - 1];
+ const Token &tk = scanner[index - 1];
if (tk.is(T_EOF_SYMBOL))
break;
else if (tk.is(T_LPAREN))
- return scanner.startPosition() + tk.position();
+ return scanner.startPosition() + tk.begin();
else if (tk.is(T_RPAREN)) {
int matchingBrace = scanner.startOfMatchingBrace(index);
diff --git a/src/libs/cplusplus/ExpressionUnderCursor.h b/src/libs/cplusplus/ExpressionUnderCursor.h
index 3972aca6e4f..a18b40b6e9c 100644
--- a/src/libs/cplusplus/ExpressionUnderCursor.h
+++ b/src/libs/cplusplus/ExpressionUnderCursor.h
@@ -42,7 +42,6 @@ QT_END_NAMESPACE
namespace CPlusPlus {
class BackwardsScanner;
-class SimpleToken;
class CPLUSPLUS_EXPORT ExpressionUnderCursor
{
@@ -56,7 +55,7 @@ public:
private:
int startOfExpression(BackwardsScanner &tk, int index);
int startOfExpression_helper(BackwardsScanner &tk, int index);
- bool isAccessToken(const SimpleToken &tk);
+ bool isAccessToken(const Token &tk);
private:
bool _jumpedComma;
diff --git a/src/libs/cplusplus/MatchingText.cpp b/src/libs/cplusplus/MatchingText.cpp
index b14348e117b..6dc7515d8d0 100644
--- a/src/libs/cplusplus/MatchingText.cpp
+++ b/src/libs/cplusplus/MatchingText.cpp
@@ -152,7 +152,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri
const int startToken = tk.startToken();
int index = startToken;
- const SimpleToken &token = tk[index - 1];
+ const Token &token = tk[index - 1];
if (text.at(0) == QLatin1Char('"') && (token.is(T_STRING_LITERAL) || token.is(T_WIDE_STRING_LITERAL))) {
if (text.length() != 1)
@@ -220,7 +220,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
--index; // consume the `{'
- const SimpleToken &token = tk[index - 1];
+ const Token &token = tk[index - 1];
if (token.is(T_STRING_LITERAL) && tk[index - 2].is(T_EXTERN)) {
// recognized extern "C"
@@ -230,7 +230,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
int i = index - 1;
forever {
- const SimpleToken ¤t = tk[i - 1];
+ const Token ¤t = tk[i - 1];
if (current.is(T_EOF_SYMBOL))
break;
@@ -290,7 +290,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
}
// look at the token before the matched brace
- const SimpleToken &tokenBeforeBrace = tk[lparenIndex - 1];
+ const Token &tokenBeforeBrace = tk[lparenIndex - 1];
if (tokenBeforeBrace.is(T_IF)) {
// recognized an if statement
diff --git a/src/libs/cplusplus/SimpleLexer.cpp b/src/libs/cplusplus/SimpleLexer.cpp
index 83f3a5189c2..1efa043e7bd 100644
--- a/src/libs/cplusplus/SimpleLexer.cpp
+++ b/src/libs/cplusplus/SimpleLexer.cpp
@@ -37,47 +37,6 @@
using namespace CPlusPlus;
-SimpleToken::SimpleToken(const Token &token)
- : _kind(token.f.kind)
- , _flags(0)
- , _position(token.begin())
- , _length(token.f.length)
-{
- f._whitespace = token.f.whitespace;
- f._newline = token.f.newline;
-}
-
-bool SimpleToken::isLiteral() const
-{
- return _kind >= T_FIRST_LITERAL && _kind <= T_LAST_LITERAL;
-}
-
-bool SimpleToken::isOperator() const
-{
- return _kind >= T_FIRST_OPERATOR && _kind <= T_LAST_OPERATOR;
-}
-
-bool SimpleToken::isKeyword() const
-{
- return _kind >= T_FIRST_KEYWORD && _kind < T_FIRST_QT_KEYWORD;
-}
-
-bool SimpleToken::isComment() const
-{
- return _kind == T_COMMENT || _kind == T_DOXY_COMMENT ||
- _kind == T_CPP_COMMENT || _kind == T_CPP_DOXY_COMMENT;
-}
-
-bool SimpleToken::isObjCAtKeyword() const
-{
- return _kind >= T_FIRST_OBJC_AT_KEYWORD && _kind <= T_LAST_OBJC_AT_KEYWORD;
-}
-
-const char *SimpleToken::name() const
-{
- return Token::name(_kind);
-}
-
SimpleLexer::SimpleLexer()
: _lastState(0),
_skipComments(false),
@@ -119,9 +78,9 @@ void SimpleLexer::setSkipComments(bool skipComments)
_skipComments = skipComments;
}
-QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
+QList<Token> SimpleLexer::operator()(const QString &text, int state)
{
- QList<SimpleToken> tokens;
+ QList<Token> tokens;
const QByteArray bytes = text.toLatin1();
const char *firstChar = bytes.constData();
@@ -131,6 +90,7 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
lex.setQtMocRunEnabled(_qtMocRunEnabled);
lex.setObjCEnabled(_objCEnabled);
lex.setStartWithNewline(true);
+ lex.setObjCEnabled(_objCEnabled);
if (! _skipComments)
lex.setScanCommentTokens(true);
@@ -147,57 +107,53 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
break;
QStringRef spell = text.midRef(lex.tokenOffset(), lex.tokenLength());
- SimpleToken simpleTk(tk);
lex.setScanAngleStringLiteralTokens(false);
if (tk.f.newline && tk.is(T_POUND))
inPreproc = true;
- else if (inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
+ else if (inPreproc && tokens.size() == 1 && tk.is(T_IDENTIFIER) &&
spell == QLatin1String("include"))
lex.setScanAngleStringLiteralTokens(true);
else if (_objCEnabled
- && inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
+ && inPreproc && tokens.size() == 1 && tk.is(T_IDENTIFIER) &&
spell == QLatin1String("import"))
lex.setScanAngleStringLiteralTokens(true);
- if (_objCEnabled && tk.is(T_IDENTIFIER))
- simpleTk.f._objcTypeQualifier = (classifyObjectiveCContextKeyword(firstChar + tk.offset, tk.f.length) != Token_identifier);
-
- tokens.append(simpleTk);
+ tokens.append(tk);
}
_lastState = lex.state();
return tokens;
}
-int SimpleLexer::tokenAt(const QList<SimpleToken> &tokens, int offset)
+int SimpleLexer::tokenAt(const QList<Token> &tokens, unsigned offset)
{
for (int index = tokens.size() - 1; index >= 0; --index) {
- const SimpleToken &tk = tokens.at(index);
- if (tk.position() <= offset && tk.end() >= offset)
+ const Token &tk = tokens.at(index);
+ if (tk.begin() <= offset && tk.end() >= offset)
return index;
}
return -1;
}
-SimpleToken SimpleLexer::tokenAt(const QString &text,
- int offset,
- int state,
- bool qtMocRunEnabled)
+Token SimpleLexer::tokenAt(const QString &text,
+ unsigned offset,
+ int state,
+ bool qtMocRunEnabled)
{
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(qtMocRunEnabled);
- const QList<SimpleToken> tokens = tokenize(text, state);
+ const QList<Token> tokens = tokenize(text, state);
const int tokenIdx = tokenAt(tokens, offset);
- return (tokenIdx == -1) ? SimpleToken() : tokens.at(tokenIdx);
+ return (tokenIdx == -1) ? Token() : tokens.at(tokenIdx);
}
-int SimpleLexer::tokenBefore(const QList<SimpleToken> &tokens, int offset)
+int SimpleLexer::tokenBefore(const QList<Token> &tokens, unsigned offset)
{
for (int index = tokens.size() - 1; index >= 0; --index) {
- const SimpleToken &tk = tokens.at(index);
- if (tk.position() <= offset)
+ const Token &tk = tokens.at(index);
+ if (tk.begin() <= offset)
return index;
}
diff --git a/src/libs/cplusplus/SimpleLexer.h b/src/libs/cplusplus/SimpleLexer.h
index cf451cacc6b..b745b1b8099 100644
--- a/src/libs/cplusplus/SimpleLexer.h
+++ b/src/libs/cplusplus/SimpleLexer.h
@@ -39,73 +39,6 @@ namespace CPlusPlus {
class SimpleLexer;
class Token;
-class CPLUSPLUS_EXPORT SimpleToken
-{
-public:
- SimpleToken(const Token &token);
-
- SimpleToken()
- : _kind(0)
- , _flags(0)
- , _position(0)
- , _length(0)
- { }
-
- inline int kind() const
- { return _kind; }
-
- inline int position() const
- { return _position; }
-
- inline int length() const
- { return _length; }
-
- inline int begin() const
- { return _position; }
-
- inline int end() const
- { return _position + _length; }
-
- inline bool followsNewline() const
- { return f._newline; }
-
- inline bool followsWhitespace() const
- { return f._whitespace; }
-
- inline bool is(int k) const { return _kind == k; }
- inline bool isNot(int k) const { return _kind != k; }
-
- bool isLiteral() const;
- bool isOperator() const;
- bool isKeyword() const;
- bool isComment() const;
- bool isObjCAtKeyword() const;
- bool isObjCTypeQualifier() const { return f._objcTypeQualifier; }
-
- const char *name() const;
-
- // internal
- inline void setPosition(int position)
- { _position = position; }
-
-public:
- short _kind;
- union {
- short _flags;
-
- struct {
- unsigned _newline: 1;
- unsigned _whitespace: 1;
- unsigned _objcTypeQualifier: 1;
- } f;
- };
-
- int _position;
- int _length;
-
- friend class SimpleLexer;
-};
-
class CPLUSPLUS_EXPORT SimpleLexer
{
public:
@@ -121,15 +54,18 @@ public:
bool objCEnabled() const;
void setObjCEnabled(bool onoff);
- QList<SimpleToken> operator()(const QString &text, int state = 0);
+ QList<Token> operator()(const QString &text, int state = 0);
int state() const
{ return _lastState; }
- static int tokenAt(const QList<SimpleToken> &tokens, int offset);
- static SimpleToken tokenAt(const QString &text, int offset, int state, bool qtMocRunEnabled = false);
+ static int tokenAt(const QList<Token> &tokens, unsigned offset);
+ static Token tokenAt(const QString &text,
+ unsigned offset,
+ int state,
+ bool qtMocRunEnabled = false);
- static int tokenBefore(const QList<SimpleToken> &tokens, int offset);
+ static int tokenBefore(const QList<Token> &tokens, unsigned offset);
private:
int _lastState;
diff --git a/src/plugins/cppeditor/cppeditor.cpp b/src/plugins/cppeditor/cppeditor.cpp
index 99e5bf0e89f..f2f55510942 100644
--- a/src/plugins/cppeditor/cppeditor.cpp
+++ b/src/plugins/cppeditor/cppeditor.cpp
@@ -1221,14 +1221,14 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(true);
const QString blockText = cursor.block().text();
- const QList<SimpleToken> tokens = tokenize(blockText, BackwardsScanner::previousBlockState(cursor.block()));
+ const QList<Token> tokens = tokenize(blockText, BackwardsScanner::previousBlockState(cursor.block()));
bool recognizedQtMethod = false;
for (int i = 0; i < tokens.size(); ++i) {
- const SimpleToken &tk = tokens.at(i);
+ const Token &tk = tokens.at(i);
- if (column >= tk.begin() && column <= tk.end()) {
+ if (((unsigned) column) >= tk.begin() && ((unsigned) column) <= tk.end()) {
if (i >= 2 && tokens.at(i).is(T_IDENTIFIER) && tokens.at(i - 1).is(T_LPAREN)
&& (tokens.at(i - 2).is(T_SIGNAL) || tokens.at(i - 2).is(T_SLOT))) {
@@ -1269,7 +1269,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
if (! recognizedQtMethod) {
const QTextBlock block = tc.block();
- const SimpleToken tk = SimpleLexer::tokenAt(block.text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(block), true);
+ const Token tk = SimpleLexer::tokenAt(block.text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(block), true);
beginOfToken = block.position() + tk.begin();
endOfToken = block.position() + tk.end();
@@ -1431,11 +1431,11 @@ bool CPPEditor::contextAllowsAutoParentheses(const QTextCursor &cursor,
bool CPPEditor::contextAllowsElectricCharacters(const QTextCursor &cursor) const
{
- const SimpleToken tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
+ const Token tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
// XXX Duplicated from CPPEditor::isInComment to avoid tokenizing twice
if (tk.isComment()) {
- const int pos = cursor.selectionEnd() - cursor.block().position();
+ const unsigned pos = cursor.selectionEnd() - cursor.block().position();
if (pos == tk.end()) {
if (tk.is(T_CPP_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))
@@ -1452,7 +1452,7 @@ bool CPPEditor::contextAllowsElectricCharacters(const QTextCursor &cursor) const
else if (tk.is(T_STRING_LITERAL) || tk.is(T_WIDE_STRING_LITERAL)
|| tk.is(T_CHAR_LITERAL) || tk.is(T_WIDE_CHAR_LITERAL)) {
- const int pos = cursor.selectionEnd() - cursor.block().position();
+ const unsigned pos = cursor.selectionEnd() - cursor.block().position();
if (pos <= tk.end())
return false;
}
@@ -1462,10 +1462,10 @@ bool CPPEditor::contextAllowsElectricCharacters(const QTextCursor &cursor) const
bool CPPEditor::isInComment(const QTextCursor &cursor) const
{
- const SimpleToken tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
+ const Token tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
if (tk.isComment()) {
- const int pos = cursor.selectionEnd() - cursor.block().position();
+ const unsigned pos = cursor.selectionEnd() - cursor.block().position();
if (pos == tk.end()) {
if (tk.is(T_CPP_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))
@@ -1520,7 +1520,7 @@ void CPPEditor::indentBlock(QTextDocument *doc, QTextBlock block, QChar typedCha
const int tokenCount = tk.startToken();
if (tokenCount != 0) {
- const SimpleToken firstToken = tk[0];
+ const Token firstToken = tk[0];
if (firstToken.is(T_COLON)) {
const int previousLineIndent = indentationColumn(ts, tk, -1);
@@ -1556,8 +1556,8 @@ void CPPEditor::indentBlock(QTextDocument *doc, QTextBlock block, QChar typedCha
}
const QString spell = tk.text(tokenIndex);
- if (tk[tokenIndex].followsNewline() && (spell.startsWith(QLatin1String("QT_")) ||
- spell.startsWith(QLatin1String("Q_")))) {
+ if (tk[tokenIndex].newline() && (spell.startsWith(QLatin1String("QT_")) ||
+ spell.startsWith(QLatin1String("Q_")))) {
const int indent = indentationColumn(ts, tk, tokenIndex);
ts.indentLine(block, indent);
return;
diff --git a/src/plugins/cppeditor/cpphighlighter.cpp b/src/plugins/cppeditor/cpphighlighter.cpp
index 9637a72c233..321025058e3 100644
--- a/src/plugins/cppeditor/cpphighlighter.cpp
+++ b/src/plugins/cppeditor/cpphighlighter.cpp
@@ -62,7 +62,7 @@ void CppHighlighter::highlightBlock(const QString &text)
tokenize.setObjCEnabled(false);
int initialState = state;
- const QList<SimpleToken> tokens = tokenize(text, initialState);
+ const QList<Token> tokens = tokenize(text, initialState);
state = tokenize.state(); // refresh the state
int foldingIndent = initialBraceDepth;
@@ -81,7 +81,7 @@ void CppHighlighter::highlightBlock(const QString &text)
return;
}
- const int firstNonSpace = tokens.first().position();
+ const unsigned firstNonSpace = tokens.first().begin();
Parentheses parentheses;
parentheses.reserve(20); // assume wizard level ;-)
@@ -89,36 +89,36 @@ void CppHighlighter::highlightBlock(const QString &text)
bool highlightAsPreprocessor = false;
for (int i = 0; i < tokens.size(); ++i) {
- const SimpleToken &tk = tokens.at(i);
+ const Token &tk = tokens.at(i);
- int previousTokenEnd = 0;
+ unsigned previousTokenEnd = 0;
if (i != 0) {
// mark the whitespaces
- previousTokenEnd = tokens.at(i - 1).position() +
+ previousTokenEnd = tokens.at(i - 1).begin() +
tokens.at(i - 1).length();
}
- if (previousTokenEnd != tk.position()) {
- setFormat(previousTokenEnd, tk.position() - previousTokenEnd,
+ if (previousTokenEnd != tk.begin()) {
+ setFormat(previousTokenEnd, tk.begin() - previousTokenEnd,
m_formats[CppVisualWhitespace]);
}
if (tk.is(T_LPAREN) || tk.is(T_LBRACE) || tk.is(T_LBRACKET)) {
- const QChar c = text.at(tk.position());
- parentheses.append(Parenthesis(Parenthesis::Opened, c, tk.position()));
+ const QChar c = text.at(tk.begin());
+ parentheses.append(Parenthesis(Parenthesis::Opened, c, tk.begin()));
if (tk.is(T_LBRACE)) {
++braceDepth;
// if a folding block opens at the beginning of a line, treat the entire line
// as if it were inside the folding block
- if (tk.position() == firstNonSpace) {
+ if (tk.begin() == firstNonSpace) {
++foldingIndent;
BaseTextDocumentLayout::userData(currentBlock())->setFoldingStartIncluded(true);
}
}
} else if (tk.is(T_RPAREN) || tk.is(T_RBRACE) || tk.is(T_RBRACKET)) {
- const QChar c = text.at(tk.position());
- parentheses.append(Parenthesis(Parenthesis::Closed, c, tk.position()));
+ const QChar c = text.at(tk.begin());
+ parentheses.append(Parenthesis(Parenthesis::Closed, c, tk.begin()));
if (tk.is(T_RBRACE)) {
--braceDepth;
if (braceDepth < foldingIndent) {
@@ -137,30 +137,30 @@ void CppHighlighter::highlightBlock(const QString &text)
highlightAsPreprocessor = false;
if (i == 0 && tk.is(T_POUND)) {
- highlightLine(text, tk.position(), tk.length(), m_formats[CppPreprocessorFormat]);
+ highlightLine(text, tk.begin(), tk.length(), m_formats[CppPreprocessorFormat]);
highlightAsPreprocessor = true;
} else if (highlightCurrentWordAsPreprocessor &&
- (tk.isKeyword() || tk.is(T_IDENTIFIER)) && isPPKeyword(text.midRef(tk.position(), tk.length())))
- setFormat(tk.position(), tk.length(), m_formats[CppPreprocessorFormat]);
+ (tk.isKeyword() || tk.is(T_IDENTIFIER)) && isPPKeyword(text.midRef(tk.begin(), tk.length())))
+ setFormat(tk.begin(), tk.length(), m_formats[CppPreprocessorFormat]);
else if (tk.is(T_NUMERIC_LITERAL))
- setFormat(tk.position(), tk.length(), m_formats[CppNumberFormat]);
+ setFormat(tk.begin(), tk.length(), m_formats[CppNumberFormat]);
else if (tk.is(T_STRING_LITERAL) || tk.is(T_CHAR_LITERAL) || tk.is(T_ANGLE_STRING_LITERAL) ||
tk.is(T_AT_STRING_LITERAL))
- highlightLine(text, tk.position(), tk.length(), m_formats[CppStringFormat]);
+ highlightLine(text, tk.begin(), tk.length(), m_formats[CppStringFormat]);
else if (tk.is(T_WIDE_STRING_LITERAL) || tk.is(T_WIDE_CHAR_LITERAL))
- highlightLine(text, tk.position(), tk.length(), m_formats[CppStringFormat]);
+ highlightLine(text, tk.begin(), tk.length(), m_formats[CppStringFormat]);
else if (tk.isComment()) {
if (tk.is(T_COMMENT) || tk.is(T_CPP_COMMENT))
- highlightLine(text, tk.position(), tk.length(), m_formats[CppCommentFormat]);
+ highlightLine(text, tk.begin(), tk.length(), m_formats[CppCommentFormat]);
else // a doxygen comment
- highlightDoxygenComment(text, tk.position(), tk.length());
+ highlightDoxygenComment(text, tk.begin(), tk.length());
// we need to insert a close comment parenthesis, if
// - the line starts in a C Comment (initalState != 0)
@@ -173,38 +173,38 @@ void CppHighlighter::highlightBlock(const QString &text)
BaseTextDocumentLayout::userData(currentBlock())->setFoldingEndIncluded(true);
else
foldingIndent = qMin(braceDepth, foldingIndent);
- const int tokenEnd = tk.position() + tk.length() - 1;
+ const int tokenEnd = tk.begin() + tk.length() - 1;
parentheses.append(Parenthesis(Parenthesis::Closed, QLatin1Char('-'), tokenEnd));
// clear the initial state.
initialState = 0;
}
- } else if (tk.isKeyword() || isQtKeyword(text.midRef(tk.position(), tk.length())) || tk.isObjCAtKeyword() || tk.isObjCTypeQualifier())
- setFormat(tk.position(), tk.length(), m_formats[CppKeywordFormat]);
+ } else if (tk.isKeyword() || isQtKeyword(text.midRef(tk.begin(), tk.length())) || tk.isObjCAtKeyword())
+ setFormat(tk.begin(), tk.length(), m_formats[CppKeywordFormat]);
else if (tk.isOperator())
- setFormat(tk.position(), tk.length(), m_formats[CppOperatorFormat]);
+ setFormat(tk.begin(), tk.length(), m_formats[CppOperatorFormat]);
else if (i == 0 && tokens.size() > 1 && tk.is(T_IDENTIFIER) && tokens.at(1).is(T_COLON))
- setFormat(tk.position(), tk.length(), m_formats[CppLabelFormat]);
+ setFormat(tk.begin(), tk.length(), m_formats[CppLabelFormat]);
else if (tk.is(T_IDENTIFIER))
- highlightWord(text.midRef(tk.position(), tk.length()), tk.position(), tk.length());
+ highlightWord(text.midRef(tk.begin(), tk.length()), tk.begin(), tk.length());
}
// mark the trailing white spaces
{
- const SimpleToken tk = tokens.last();
- const int lastTokenEnd = tk.position() + tk.length();
+ const Token tk = tokens.last();
+ const int lastTokenEnd = tk.begin() + tk.length();
if (text.length() > lastTokenEnd)
highlightLine(text, lastTokenEnd, text.length() - lastTokenEnd, QTextCharFormat());
}
if (! initialState && state && ! tokens.isEmpty()) {
parentheses.append(Parenthesis(Parenthesis::Opened, QLatin1Char('+'),
- tokens.last().position()));
+ tokens.last().begin()));
++braceDepth;
}
diff --git a/src/plugins/cpptools/cppcodecompletion.cpp b/src/plugins/cpptools/cppcodecompletion.cpp
index c3a1b345fcd..ab45c47369d 100644
--- a/src/plugins/cpptools/cppcodecompletion.cpp
+++ b/src/plugins/cpptools/cppcodecompletion.cpp
@@ -319,9 +319,9 @@ void FunctionArgumentWidget::updateArgumentHighlight()
int argnr = 0;
int parcount = 0;
SimpleLexer tokenize;
- QList<SimpleToken> tokens = tokenize(str);
+ QList<Token> tokens = tokenize(str);
for (int i = 0; i < tokens.count(); ++i) {
- const SimpleToken &tk = tokens.at(i);
+ const Token &tk = tokens.at(i);
if (tk.is(T_LPAREN))
++parcount;
else if (tk.is(T_RPAREN))
@@ -555,9 +555,9 @@ static int startOfOperator(TextEditor::ITextEditable *editor,
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(true);
tokenize.setSkipComments(false);
- const QList<SimpleToken> &tokens = tokenize(tc.block().text());
+ const QList<Token> &tokens = tokenize(tc.block().text());
const int tokenIdx = SimpleLexer::tokenAt(tokens, tc.positionInBlock());
- const SimpleToken &tk = (tokenIdx == -1) ? SimpleToken() : tokens.at(tokenIdx);
+ const Token &tk = (tokenIdx == -1) ? Token() : tokens.at(tokenIdx);
if (completionKind == T_DOXY_COMMENT && !(tk.is(T_DOXY_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))) {
completionKind = T_EOF_SYMBOL;
@@ -579,11 +579,11 @@ static int startOfOperator(TextEditor::ITextEditable *editor,
else if (completionKind == T_LPAREN) {
int i = 0;
for (; i < tokens.size(); ++i) {
- const SimpleToken &token = tokens.at(i);
- if (token.position() == tk.position()) {
+ const Token &token = tokens.at(i);
+ if (token.begin() == tk.begin()) {
if (i == 0) // no token on the left, but might be on a previous line
break;
- const SimpleToken &previousToken = tokens.at(i - 1);
+ const Token &previousToken = tokens.at(i - 1);
if (previousToken.is(T_IDENTIFIER) || previousToken.is(T_GREATER)
|| previousToken.is(T_SIGNAL) || previousToken.is(T_SLOT))
break;
@@ -601,8 +601,8 @@ static int startOfOperator(TextEditor::ITextEditable *editor,
if (tokens.size() >= 3) {
if (tokens.at(0).is(T_POUND) && tokens.at(1).is(T_IDENTIFIER) && (tokens.at(2).is(T_STRING_LITERAL) ||
tokens.at(2).is(T_ANGLE_STRING_LITERAL))) {
- const SimpleToken &directiveToken = tokens.at(1);
- QString directive = tc.block().text().mid(directiveToken.position(),
+ const Token &directiveToken = tokens.at(1);
+ QString directive = tc.block().text().mid(directiveToken.begin(),
directiveToken.length());
if (directive == QLatin1String("include") ||
directive == QLatin1String("include_next") ||
--
GitLab