Commit 8e4fb678 authored by Erik Verbruggen's avatar Erik Verbruggen
Browse files

Removing SimpleToken

parent e3e8b1a5
......@@ -52,19 +52,19 @@ BackwardsScanner::BackwardsScanner(const QTextCursor &cursor, int maxBlockCount,
_startToken = _tokens.size();
}
SimpleToken BackwardsScanner::LA(int index) const
Token BackwardsScanner::LA(int index) const
{ return const_cast<BackwardsScanner *>(this)->fetchToken(_startToken - index); }
SimpleToken BackwardsScanner::operator[](int index) const
Token BackwardsScanner::operator[](int index) const
{ return const_cast<BackwardsScanner *>(this)->fetchToken(index); }
const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
const Token &BackwardsScanner::fetchToken(int tokenIndex)
{
while (_offset + tokenIndex < 0) {
_block = _block.previous();
if (_blocksTokenized == _maxBlockCount || !_block.isValid()) {
++_offset;
_tokens.prepend(SimpleToken()); // sentinel
_tokens.prepend(Token()); // sentinel
break;
} else {
++_blocksTokenized;
......@@ -73,10 +73,10 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
_text.prepend(QLatin1Char('\n'));
_text.prepend(blockText);
QList<SimpleToken> adaptedTokens;
QList<Token> adaptedTokens;
for (int i = 0; i < _tokens.size(); ++i) {
SimpleToken t = _tokens.at(i);
t.setPosition(t.position() + blockText.length() + 1);
Token t = _tokens.at(i);
t.offset += + blockText.length() + 1;
adaptedTokens.append(t);
}
......@@ -100,19 +100,19 @@ QString BackwardsScanner::text() const
QString BackwardsScanner::mid(int index) const
{
const SimpleToken &firstToken = _tokens.at(index + _offset);
const Token &firstToken = _tokens.at(index + _offset);
return _text.mid(firstToken.begin());
}
QString BackwardsScanner::text(int index) const
{
const SimpleToken &firstToken = _tokens.at(index + _offset);
const Token &firstToken = _tokens.at(index + _offset);
return _text.mid(firstToken.begin(), firstToken.length());
}
QStringRef BackwardsScanner::textRef(int index) const
{
const SimpleToken &firstToken = _tokens.at(index + _offset);
const Token &firstToken = _tokens.at(index + _offset);
return _text.midRef(firstToken.begin(), firstToken.length());
}
......@@ -181,11 +181,11 @@ int BackwardsScanner::startOfLine(int index) const
const BackwardsScanner tk(*this);
forever {
const SimpleToken &tok = tk[index - 1];
const Token &tok = tk[index - 1];
if (tok.is(T_EOF_SYMBOL))
break;
else if (tok.followsNewline())
else if (tok.newline())
return index - 1;
--index;
......@@ -201,7 +201,7 @@ int BackwardsScanner::startOfBlock(int index) const
const int start = index;
forever {
SimpleToken token = tk[index - 1];
Token token = tk[index - 1];
if (token.is(T_EOF_SYMBOL)) {
break;
......@@ -234,9 +234,10 @@ int BackwardsScanner::startOfBlock(int index) const
QString BackwardsScanner::indentationString(int index) const
{
const SimpleToken tokenAfterNewline = operator[](startOfLine(index + 1));
const int newlinePos = qMax(0, _text.lastIndexOf(QLatin1Char('\n'), tokenAfterNewline.position()));
return _text.mid(newlinePos, tokenAfterNewline.position() - newlinePos);
const Token tokenAfterNewline = operator[](startOfLine(index + 1));
const int newlinePos = qMax(0, _text.lastIndexOf(QLatin1Char('\n'),
tokenAfterNewline.begin()));
return _text.mid(newlinePos, tokenAfterNewline.begin() - newlinePos);
}
......
......@@ -55,10 +55,10 @@ public:
QString text(int index) const;
QStringRef textRef(int index) const;
// 1-based
SimpleToken LA(int index) const;
Token LA(int index) const;
// n-la token is [startToken - n]
SimpleToken operator[](int index) const; // ### deprecate
Token operator[](int index) const; // ### deprecate
QString indentationString(int index) const;
......@@ -71,10 +71,10 @@ public:
static int previousBlockState(const QTextBlock &block);
private:
const SimpleToken &fetchToken(int tokenIndex);
const Token &fetchToken(int tokenIndex);
private:
QList<SimpleToken> _tokens;
QList<Token> _tokens;
int _offset;
int _blocksTokenized;
QTextBlock _block;
......
......@@ -56,7 +56,7 @@ int ExpressionUnderCursor::startOfExpression(BackwardsScanner &tk, int index)
index = startOfExpression_helper(tk, index);
if (_jumpedComma) {
const SimpleToken &tok = tk[index - 1];
const Token &tok = tk[index - 1];
switch (tok.kind()) {
case T_COMMA:
......@@ -204,7 +204,7 @@ int ExpressionUnderCursor::startOfExpression_helper(BackwardsScanner &tk, int in
return index;
}
bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk)
bool ExpressionUnderCursor::isAccessToken(const Token &tk)
{
switch (tk.kind()) {
case T_COLON_COLON:
......@@ -237,12 +237,12 @@ int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
int index = scanner.startToken();
forever {
const SimpleToken &tk = scanner[index - 1];
const Token &tk = scanner[index - 1];
if (tk.is(T_EOF_SYMBOL))
break;
else if (tk.is(T_LPAREN))
return scanner.startPosition() + tk.position();
return scanner.startPosition() + tk.begin();
else if (tk.is(T_RPAREN)) {
int matchingBrace = scanner.startOfMatchingBrace(index);
......
......@@ -42,7 +42,6 @@ QT_END_NAMESPACE
namespace CPlusPlus {
class BackwardsScanner;
class SimpleToken;
class CPLUSPLUS_EXPORT ExpressionUnderCursor
{
......@@ -56,7 +55,7 @@ public:
private:
int startOfExpression(BackwardsScanner &tk, int index);
int startOfExpression_helper(BackwardsScanner &tk, int index);
bool isAccessToken(const SimpleToken &tk);
bool isAccessToken(const Token &tk);
private:
bool _jumpedComma;
......
......@@ -152,7 +152,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri
const int startToken = tk.startToken();
int index = startToken;
const SimpleToken &token = tk[index - 1];
const Token &token = tk[index - 1];
if (text.at(0) == QLatin1Char('"') && (token.is(T_STRING_LITERAL) || token.is(T_WIDE_STRING_LITERAL))) {
if (text.length() != 1)
......@@ -220,7 +220,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
--index; // consume the `{'
const SimpleToken &token = tk[index - 1];
const Token &token = tk[index - 1];
if (token.is(T_STRING_LITERAL) && tk[index - 2].is(T_EXTERN)) {
// recognized extern "C"
......@@ -230,7 +230,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
int i = index - 1;
forever {
const SimpleToken &current = tk[i - 1];
const Token &current = tk[i - 1];
if (current.is(T_EOF_SYMBOL))
break;
......@@ -290,7 +290,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
}
// look at the token before the matched brace
const SimpleToken &tokenBeforeBrace = tk[lparenIndex - 1];
const Token &tokenBeforeBrace = tk[lparenIndex - 1];
if (tokenBeforeBrace.is(T_IF)) {
// recognized an if statement
......
......@@ -37,47 +37,6 @@
using namespace CPlusPlus;
SimpleToken::SimpleToken(const Token &token)
: _kind(token.f.kind)
, _flags(0)
, _position(token.begin())
, _length(token.f.length)
{
f._whitespace = token.f.whitespace;
f._newline = token.f.newline;
}
bool SimpleToken::isLiteral() const
{
return _kind >= T_FIRST_LITERAL && _kind <= T_LAST_LITERAL;
}
bool SimpleToken::isOperator() const
{
return _kind >= T_FIRST_OPERATOR && _kind <= T_LAST_OPERATOR;
}
bool SimpleToken::isKeyword() const
{
return _kind >= T_FIRST_KEYWORD && _kind < T_FIRST_QT_KEYWORD;
}
bool SimpleToken::isComment() const
{
return _kind == T_COMMENT || _kind == T_DOXY_COMMENT ||
_kind == T_CPP_COMMENT || _kind == T_CPP_DOXY_COMMENT;
}
bool SimpleToken::isObjCAtKeyword() const
{
return _kind >= T_FIRST_OBJC_AT_KEYWORD && _kind <= T_LAST_OBJC_AT_KEYWORD;
}
const char *SimpleToken::name() const
{
return Token::name(_kind);
}
SimpleLexer::SimpleLexer()
: _lastState(0),
_skipComments(false),
......@@ -119,9 +78,9 @@ void SimpleLexer::setSkipComments(bool skipComments)
_skipComments = skipComments;
}
QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
QList<Token> SimpleLexer::operator()(const QString &text, int state)
{
QList<SimpleToken> tokens;
QList<Token> tokens;
const QByteArray bytes = text.toLatin1();
const char *firstChar = bytes.constData();
......@@ -131,6 +90,7 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
lex.setQtMocRunEnabled(_qtMocRunEnabled);
lex.setObjCEnabled(_objCEnabled);
lex.setStartWithNewline(true);
lex.setObjCEnabled(_objCEnabled);
if (! _skipComments)
lex.setScanCommentTokens(true);
......@@ -147,57 +107,53 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
break;
QStringRef spell = text.midRef(lex.tokenOffset(), lex.tokenLength());
SimpleToken simpleTk(tk);
lex.setScanAngleStringLiteralTokens(false);
if (tk.f.newline && tk.is(T_POUND))
inPreproc = true;
else if (inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
else if (inPreproc && tokens.size() == 1 && tk.is(T_IDENTIFIER) &&
spell == QLatin1String("include"))
lex.setScanAngleStringLiteralTokens(true);
else if (_objCEnabled
&& inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
&& inPreproc && tokens.size() == 1 && tk.is(T_IDENTIFIER) &&
spell == QLatin1String("import"))
lex.setScanAngleStringLiteralTokens(true);
if (_objCEnabled && tk.is(T_IDENTIFIER))
simpleTk.f._objcTypeQualifier = (classifyObjectiveCContextKeyword(firstChar + tk.offset, tk.f.length) != Token_identifier);
tokens.append(simpleTk);
tokens.append(tk);
}
_lastState = lex.state();
return tokens;
}
int SimpleLexer::tokenAt(const QList<SimpleToken> &tokens, int offset)
int SimpleLexer::tokenAt(const QList<Token> &tokens, unsigned offset)
{
for (int index = tokens.size() - 1; index >= 0; --index) {
const SimpleToken &tk = tokens.at(index);
if (tk.position() <= offset && tk.end() >= offset)
const Token &tk = tokens.at(index);
if (tk.begin() <= offset && tk.end() >= offset)
return index;
}
return -1;
}
SimpleToken SimpleLexer::tokenAt(const QString &text,
int offset,
int state,
bool qtMocRunEnabled)
Token SimpleLexer::tokenAt(const QString &text,
unsigned offset,
int state,
bool qtMocRunEnabled)
{
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(qtMocRunEnabled);
const QList<SimpleToken> tokens = tokenize(text, state);
const QList<Token> tokens = tokenize(text, state);
const int tokenIdx = tokenAt(tokens, offset);
return (tokenIdx == -1) ? SimpleToken() : tokens.at(tokenIdx);
return (tokenIdx == -1) ? Token() : tokens.at(tokenIdx);
}
int SimpleLexer::tokenBefore(const QList<SimpleToken> &tokens, int offset)
int SimpleLexer::tokenBefore(const QList<Token> &tokens, unsigned offset)
{
for (int index = tokens.size() - 1; index >= 0; --index) {
const SimpleToken &tk = tokens.at(index);
if (tk.position() <= offset)
const Token &tk = tokens.at(index);
if (tk.begin() <= offset)
return index;
}
......
......@@ -39,73 +39,6 @@ namespace CPlusPlus {
class SimpleLexer;
class Token;
class CPLUSPLUS_EXPORT SimpleToken
{
public:
SimpleToken(const Token &token);
SimpleToken()
: _kind(0)
, _flags(0)
, _position(0)
, _length(0)
{ }
inline int kind() const
{ return _kind; }
inline int position() const
{ return _position; }
inline int length() const
{ return _length; }
inline int begin() const
{ return _position; }
inline int end() const
{ return _position + _length; }
inline bool followsNewline() const
{ return f._newline; }
inline bool followsWhitespace() const
{ return f._whitespace; }
inline bool is(int k) const { return _kind == k; }
inline bool isNot(int k) const { return _kind != k; }
bool isLiteral() const;
bool isOperator() const;
bool isKeyword() const;
bool isComment() const;
bool isObjCAtKeyword() const;
bool isObjCTypeQualifier() const { return f._objcTypeQualifier; }
const char *name() const;
// internal
inline void setPosition(int position)
{ _position = position; }
public:
short _kind;
union {
short _flags;
struct {
unsigned _newline: 1;
unsigned _whitespace: 1;
unsigned _objcTypeQualifier: 1;
} f;
};
int _position;
int _length;
friend class SimpleLexer;
};
class CPLUSPLUS_EXPORT SimpleLexer
{
public:
......@@ -121,15 +54,18 @@ public:
bool objCEnabled() const;
void setObjCEnabled(bool onoff);
QList<SimpleToken> operator()(const QString &text, int state = 0);
QList<Token> operator()(const QString &text, int state = 0);
int state() const
{ return _lastState; }
static int tokenAt(const QList<SimpleToken> &tokens, int offset);
static SimpleToken tokenAt(const QString &text, int offset, int state, bool qtMocRunEnabled = false);
static int tokenAt(const QList<Token> &tokens, unsigned offset);
static Token tokenAt(const QString &text,
unsigned offset,
int state,
bool qtMocRunEnabled = false);
static int tokenBefore(const QList<SimpleToken> &tokens, int offset);
static int tokenBefore(const QList<Token> &tokens, unsigned offset);
private:
int _lastState;
......
......@@ -1221,14 +1221,14 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(true);
const QString blockText = cursor.block().text();
const QList<SimpleToken> tokens = tokenize(blockText, BackwardsScanner::previousBlockState(cursor.block()));
const QList<Token> tokens = tokenize(blockText, BackwardsScanner::previousBlockState(cursor.block()));
bool recognizedQtMethod = false;
for (int i = 0; i < tokens.size(); ++i) {
const SimpleToken &tk = tokens.at(i);
const Token &tk = tokens.at(i);
if (column >= tk.begin() && column <= tk.end()) {
if (((unsigned) column) >= tk.begin() && ((unsigned) column) <= tk.end()) {
if (i >= 2 && tokens.at(i).is(T_IDENTIFIER) && tokens.at(i - 1).is(T_LPAREN)
&& (tokens.at(i - 2).is(T_SIGNAL) || tokens.at(i - 2).is(T_SLOT))) {
......@@ -1269,7 +1269,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
if (! recognizedQtMethod) {
const QTextBlock block = tc.block();
const SimpleToken tk = SimpleLexer::tokenAt(block.text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(block), true);
const Token tk = SimpleLexer::tokenAt(block.text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(block), true);
beginOfToken = block.position() + tk.begin();
endOfToken = block.position() + tk.end();
......@@ -1431,11 +1431,11 @@ bool CPPEditor::contextAllowsAutoParentheses(const QTextCursor &cursor,
bool CPPEditor::contextAllowsElectricCharacters(const QTextCursor &cursor) const
{
const SimpleToken tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
const Token tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
// XXX Duplicated from CPPEditor::isInComment to avoid tokenizing twice
if (tk.isComment()) {
const int pos = cursor.selectionEnd() - cursor.block().position();
const unsigned pos = cursor.selectionEnd() - cursor.block().position();
if (pos == tk.end()) {
if (tk.is(T_CPP_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))
......@@ -1452,7 +1452,7 @@ bool CPPEditor::contextAllowsElectricCharacters(const QTextCursor &cursor) const
else if (tk.is(T_STRING_LITERAL) || tk.is(T_WIDE_STRING_LITERAL)
|| tk.is(T_CHAR_LITERAL) || tk.is(T_WIDE_CHAR_LITERAL)) {
const int pos = cursor.selectionEnd() - cursor.block().position();
const unsigned pos = cursor.selectionEnd() - cursor.block().position();
if (pos <= tk.end())
return false;
}
......@@ -1462,10 +1462,10 @@ bool CPPEditor::contextAllowsElectricCharacters(const QTextCursor &cursor) const
bool CPPEditor::isInComment(const QTextCursor &cursor) const
{
const SimpleToken tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
const Token tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
if (tk.isComment()) {
const int pos = cursor.selectionEnd() - cursor.block().position();
const unsigned pos = cursor.selectionEnd() - cursor.block().position();
if (pos == tk.end()) {
if (tk.is(T_CPP_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))
......@@ -1520,7 +1520,7 @@ void CPPEditor::indentBlock(QTextDocument *doc, QTextBlock block, QChar typedCha
const int tokenCount = tk.startToken();
if (tokenCount != 0) {
const SimpleToken firstToken = tk[0];
const Token firstToken = tk[0];
if (firstToken.is(T_COLON)) {
const int previousLineIndent = indentationColumn(ts, tk, -1);
......@@ -1556,8 +1556,8 @@ void CPPEditor::indentBlock(QTextDocument *doc, QTextBlock block, QChar typedCha
}
const QString spell = tk.text(tokenIndex);
if (tk[tokenIndex].followsNewline() && (spell.startsWith(QLatin1String("QT_")) ||
spell.startsWith(QLatin1String("Q_")))) {
if (tk[tokenIndex].newline() && (spell.startsWith(QLatin1String("QT_")) ||
spell.startsWith(QLatin1String("Q_")))) {
const int indent = indentationColumn(ts, tk, tokenIndex);
ts.indentLine(block, indent);
return;
......
......@@ -62,7 +62,7 @@ void CppHighlighter::highlightBlock(const QString &text)
tokenize.setObjCEnabled(false);
int initialState = state;
const QList<SimpleToken> tokens = tokenize(text, initialState);
const QList<Token> tokens = tokenize(text, initialState);
state = tokenize.state(); // refresh the state
int foldingIndent = initialBraceDepth;
......@@ -81,7 +81,7 @@ void CppHighlighter::highlightBlock(const QString &text)
return;
}
const int firstNonSpace = tokens.first().position();
const unsigned firstNonSpace = tokens.first().begin();
Parentheses parentheses;
parentheses.reserve(20); // assume wizard level ;-)
......@@ -89,36 +89,36 @@ void CppHighlighter::highlightBlock(const QString &text)
bool highlightAsPreprocessor = false;
for (int i = 0; i < tokens.size(); ++i) {
const SimpleToken &tk = tokens.at(i);
const Token &tk = tokens.at(i);
int previousTokenEnd = 0;
unsigned previousTokenEnd = 0;
if (i != 0) {
// mark the whitespaces
previousTokenEnd = tokens.at(i - 1).position() +
previousTokenEnd = tokens.at(i - 1).begin() +
tokens.at(i - 1).length();
}
if (previousTokenEnd != tk.position()) {
setFormat(previousTokenEnd, tk.position() - previousTokenEnd,
if (previousTokenEnd != tk.begin()) {
setFormat(previousTokenEnd, tk.begin() - previousTokenEnd,
m_formats[CppVisualWhitespace]);
}
if (tk.is(T_LPAREN) || tk.is(T_LBRACE) || tk.is(T_LBRACKET)) {
const QChar c = text.at(tk.position());
parentheses.append(Parenthesis(Parenthesis::Opened, c, tk.position()));
const QChar c = text.at(tk.begin());
parentheses.append(Parenthesis(Parenthesis::Opened, c, tk.begin()));
if (tk.is(T_LBRACE)) {
++braceDepth;
// if a folding block opens at the beginning of a line, treat the entire line
// as if it were inside the folding block
if (tk.position() == firstNonSpace) {
if (tk.begin() == firstNonSpace) {
++foldingIndent;
BaseTextDocumentLayout::userData(currentBlock())->setFoldingStartIncluded(true);
}
}
} else if (tk.is(T_RPAREN) || tk.is(T_RBRACE) || tk.is(T_RBRACKET)) {
const QChar c = text.at(tk.position());
parentheses.append(Parenthesis(Parenthesis::Closed, c, tk.position()));
const QChar c = text.at(tk.begin());
parentheses.append(Parenthesis(Parenthesis::Closed, c, tk.begin()));
if (tk.is(T_RBRACE)) {
--braceDepth;
if (braceDepth < foldingIndent) {
......@@ -137,30 +137,30 @@ void CppHighlighter::highlightBlock(const QString &text)
highlightAsPreprocessor = false;
if (i == 0 && tk.is(T_POUND)) {