C++: Use Token::utf16chars{Begin,End} where appropriate

...especially in CppTools/CppEditor where the offsets are used with a
QString/QTextDocument.

Change-Id: Ic6d18fbc01fb9cc899a9bd2d7424cd2edae487f1
Reviewed-by: Erik Verbruggen <erik.verbruggen@digia.com>

src/libs/cplusplus/BackwardsScanner.cpp

@@ -87,7 +87,7 @@ const Token &BackwardsScanner::fetchToken(int tokenIndex)
             QList<Token> adaptedTokens;
             for (int i = 0; i < _tokens.size(); ++i) {
                 Token t = _tokens.at(i);
-                t.byteOffset += + blockText.length() + 1;
+                t.utf16charOffset += blockText.length() + 1;
                 adaptedTokens.append(t);
             }
 
@@ -112,19 +112,19 @@ QString BackwardsScanner::text() const
 QString BackwardsScanner::mid(int index) const
 {
     const Token &firstToken = _tokens.at(index + _offset);
-    return _text.mid(firstToken.bytesBegin());
+    return _text.mid(firstToken.utf16charsBegin());
 }
 
 QString BackwardsScanner::text(int index) const
 {
     const Token &firstToken = _tokens.at(index + _offset);
-    return _text.mid(firstToken.bytesBegin(), firstToken.bytes());
+    return _text.mid(firstToken.utf16charsBegin(), firstToken.utf16chars());
 }
 
 QStringRef BackwardsScanner::textRef(int index) const
 {
     const Token &firstToken = _tokens.at(index + _offset);
-    return _text.midRef(firstToken.bytesBegin(), firstToken.bytes());
+    return _text.midRef(firstToken.utf16charsBegin(), firstToken.utf16chars());
 }
 
 int BackwardsScanner::size() const
@@ -247,8 +247,8 @@ QString BackwardsScanner::indentationString(int index) const
 {
     const Token tokenAfterNewline = operator[](startOfLine(index + 1));
     const int newlinePos = qMax(0, _text.lastIndexOf(QLatin1Char('\n'),
-                                                     tokenAfterNewline.bytesBegin()));
-    return _text.mid(newlinePos, tokenAfterNewline.bytesBegin() - newlinePos);
+                                                     tokenAfterNewline.utf16charsBegin()));
+    return _text.mid(newlinePos, tokenAfterNewline.utf16charsBegin() - newlinePos);
 }
 
 

src/libs/cplusplus/ExpressionUnderCursor.cpp

@@ -266,7 +266,7 @@ int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
         if (tk.is(T_EOF_SYMBOL)) {
             break;
         } else if (tk.is(T_LPAREN)) {
-            return scanner.startPosition() + tk.bytesBegin();
+            return scanner.startPosition() + tk.utf16charsBegin();
         } else if (tk.is(T_RPAREN)) {
             int matchingBrace = scanner.startOfMatchingBrace(index);
 

src/libs/cplusplus/FindUsages.cpp

@@ -178,7 +178,7 @@ void FindUsages::reportResult(unsigned tokenIndex)
     if (col)
         --col;  // adjust the column position.
 
-    const int len = tk.bytes();
+    const int len = tk.utf16chars();
 
     const Usage u(_doc->fileName(), lineText, line, col, len);
     _usages.append(u);

src/libs/cplusplus/SimpleLexer.cpp

@@ -112,11 +112,11 @@ QList<Token> SimpleLexer::operator()(const QString &text, int state)
     return tokens;
 }
 
-int SimpleLexer::tokenAt(const QList<Token> &tokens, unsigned offset)
+int SimpleLexer::tokenAt(const QList<Token> &tokens, unsigned utf16charsOffset)
 {
     for (int index = tokens.size() - 1; index >= 0; --index) {
         const Token &tk = tokens.at(index);
-        if (tk.bytesBegin() <= offset && tk.bytesEnd() >= offset)
+        if (tk.utf16charsBegin() <= utf16charsOffset && tk.utf16charsEnd() >= utf16charsOffset)
             return index;
     }
 
@@ -124,7 +124,7 @@ int SimpleLexer::tokenAt(const QList<Token> &tokens, unsigned offset)
 }
 
 Token SimpleLexer::tokenAt(const QString &text,
-                           unsigned offset,
+                           unsigned utf16charsOffset,
                            int state,
                            bool qtMocRunEnabled)
 {
@@ -138,15 +138,15 @@ Token SimpleLexer::tokenAt(const QString &text,
     SimpleLexer tokenize;
     tokenize.setLanguageFeatures(features);
     const QList<Token> tokens = tokenize(text, state);
-    const int tokenIdx = tokenAt(tokens, offset);
+    const int tokenIdx = tokenAt(tokens, utf16charsOffset);
     return (tokenIdx == -1) ? Token() : tokens.at(tokenIdx);
 }
 
-int SimpleLexer::tokenBefore(const QList<Token> &tokens, unsigned offset)
+int SimpleLexer::tokenBefore(const QList<Token> &tokens, unsigned utf16charsOffset)
 {
     for (int index = tokens.size() - 1; index >= 0; --index) {
         const Token &tk = tokens.at(index);
-        if (tk.bytesBegin() <= offset)
+        if (tk.utf16charsBegin() <= utf16charsOffset)
             return index;
     }
 

src/libs/cplusplus/SimpleLexer.h

@@ -59,13 +59,13 @@ public:
     int state() const
     { return _lastState; }
 
-    static int tokenAt(const QList<Token> &tokens, unsigned offset);
+    static int tokenAt(const QList<Token> &tokens, unsigned utf16charsOffset);
     static Token tokenAt(const QString &text,
-                         unsigned offset,
+                         unsigned utf16charsOffset,
                          int state,
                          bool qtMocRunEnabled = false);
 
-    static int tokenBefore(const QList<Token> &tokens, unsigned offset);
+    static int tokenBefore(const QList<Token> &tokens, unsigned utf16charsOffset);
 
 private:
     int _lastState;

src/libs/qmljs/qmljsfindexportedcpptypes.cpp

@@ -225,8 +225,8 @@ protected:
             // go through comments backwards to find the annotation closest to the call
             for (unsigned i = _doc->translationUnit()->commentCount(); i-- > 0; ) {
                 const Token commentToken = _doc->translationUnit()->commentAt(i);
-                if (commentToken.bytesBegin() >= end.bytesBegin()
-                        || commentToken.bytesEnd() <= begin.bytesBegin()) {
+                if (commentToken.utf16charsBegin() >= end.utf16charsBegin()
+                        || commentToken.utf16charsEnd() <= begin.utf16charsBegin()) {
                     continue;
                 }
                 const QString comment = stringOf(commentToken);

src/plugins/cppeditor/cppautocompleter.cpp

@@ -72,7 +72,7 @@ bool CppAutoCompleter::contextAllowsElectricCharacters(const QTextCursor &cursor
 
     if (token.isStringLiteral() || token.isCharLiteral()) {
         const unsigned pos = cursor.selectionEnd() - cursor.block().position();
-        if (pos <= token.bytesEnd())
+        if (pos <= token.utf16charsEnd())
             return false;
     }
 
@@ -115,10 +115,10 @@ bool CppAutoCompleter::isInCommentHelper(const QTextCursor &cursor, Token *retTo
 
     const unsigned pos = cursor.selectionEnd() - cursor.block().position();
 
-    if (tokens.isEmpty() || pos < tokens.first().bytesBegin())
+    if (tokens.isEmpty() || pos < tokens.first().utf16charsBegin())
         return prevState > 0;
 
-    if (pos >= tokens.last().bytesEnd()) {
+    if (pos >= tokens.last().utf16charsEnd()) {
         const Token tk = tokens.last();
         if (tk.is(T_CPP_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))
             return true;
@@ -137,7 +137,7 @@ const Token CppAutoCompleter::tokenAtPosition(const QList<Token> &tokens, const
 {
     for (int i = tokens.size() - 1; i >= 0; --i) {
         const Token tk = tokens.at(i);
-        if (pos >= tk.bytesBegin() && pos < tk.bytesEnd())
+        if (pos >= tk.utf16charsBegin() && pos < tk.utf16charsEnd())
             return tk;
     }
     return Token();

src/plugins/cppeditor/cppcodemodelinspectordialog.cpp

@@ -821,7 +821,7 @@ public:
     void clear();
 
     enum Columns { SpelledColumn, KindColumn, IndexColumn, OffsetColumn, LineColumnNumberColumn,
-                   LengthInBytesColumn, GeneratedColumn, ExpandedColumn, WhiteSpaceColumn,
+                   BytesAndCodePointsColumn, GeneratedColumn, ExpandedColumn, WhiteSpaceColumn,
                    NewlineColumn, ColumnCount };
 
     int rowCount(const QModelIndex &parent = QModelIndex()) const;
@@ -890,10 +890,11 @@ QVariant TokensModel::data(const QModelIndex &index, int role) const
         else if (column == OffsetColumn)
             return token.bytesBegin();
         else if (column == LineColumnNumberColumn)
-            return QString::fromLatin1("%1:%2")
-                    .arg(CMI::Utils::toString(info.line), CMI::Utils::toString(info.column));
-        else if (column == LengthInBytesColumn)
-            return CMI::Utils::toString(token.bytes());
+            return QString::fromLatin1("%1:%2").arg(CMI::Utils::toString(info.line),
+                                                    CMI::Utils::toString(info.column));
+        else if (column == BytesAndCodePointsColumn)
+            return QString::fromLatin1("%1/%2").arg(CMI::Utils::toString(token.bytes()),
+                                                    CMI::Utils::toString(token.utf16chars()));
         else if (column == GeneratedColumn)
             return CMI::Utils::toString(token.generated());
         else if (column == ExpandedColumn)
@@ -922,8 +923,8 @@ QVariant TokensModel::headerData(int section, Qt::Orientation orientation, int r
             return QLatin1String("Offset");
         case LineColumnNumberColumn:
             return QLatin1String("Line:Column");
-        case LengthInBytesColumn:
-            return QLatin1String("Bytes");
+        case BytesAndCodePointsColumn:
+            return QLatin1String("Bytes/Codepoints");
         case GeneratedColumn:
             return QLatin1String("Generated");
         case ExpandedColumn:

src/plugins/cppeditor/cppeditor.cpp

@@ -795,7 +795,7 @@ void CPPEditorWidget::markSymbolsNow()
             if (column)
                 --column;  // adjust the column position.
 
-            const int len = unit->tokenAt(index).bytes();
+            const int len = unit->tokenAt(index).utf16chars();
 
             QTextCursor cursor(document()->findBlockByNumber(line - 1));
             cursor.setPosition(cursor.position() + column);

src/plugins/cppeditor/cppfollowsymbolundercursor.cpp

@@ -481,8 +481,8 @@ BaseTextEditorWidget::Link FollowSymbolUnderCursor::findLink(const QTextCursor &
     for (int i = 0; i < tokens.size(); ++i) {
         const Token &tk = tokens.at(i);
 
-        if (((unsigned) positionInBlock) >= tk.bytesBegin()
-                && ((unsigned) positionInBlock) < tk.bytesEnd()) {
+        if (((unsigned) positionInBlock) >= tk.utf16charsBegin()
+                && ((unsigned) positionInBlock) < tk.utf16charsEnd()) {
             int closingParenthesisPos = tokens.size();
             if (i >= 2 && tokens.at(i).is(T_IDENTIFIER) && tokens.at(i - 1).is(T_LPAREN)
                 && (tokens.at(i - 2).is(T_SIGNAL) || tokens.at(i - 2).is(T_SLOT))) {
@@ -506,10 +506,10 @@ BaseTextEditorWidget::Link FollowSymbolUnderCursor::findLink(const QTextCursor &
             if (closingParenthesisPos < tokens.size()) {
                 QTextBlock block = cursor.block();
 
-                beginOfToken = block.position() + tokens.at(i).bytesBegin();
-                endOfToken = block.position() + tokens.at(i).bytesEnd();
+                beginOfToken = block.position() + tokens.at(i).utf16charsBegin();
+                endOfToken = block.position() + tokens.at(i).utf16charsEnd();
 
-                tc.setPosition(block.position() + tokens.at(closingParenthesisPos).bytesEnd());
+                tc.setPosition(block.position() + tokens.at(closingParenthesisPos).utf16charsEnd());
                 recognizedQtMethod = true;
             }
             break;
@@ -524,8 +524,8 @@ BaseTextEditorWidget::Link FollowSymbolUnderCursor::findLink(const QTextCursor &
 
             // In this case we want to look at one token before the current position to recognize
             // an operator if the cursor is inside the actual operator: operator[$]
-            if (unsigned(positionInBlock) >= tk.bytesBegin()
-                    && unsigned(positionInBlock) <= tk.bytesEnd()) {
+            if (unsigned(positionInBlock) >= tk.utf16charsBegin()
+                    && unsigned(positionInBlock) <= tk.utf16charsEnd()) {
                 cursorRegionReached = true;
                 if (tk.is(T_OPERATOR)) {
                     link = attemptFuncDeclDef(cursor, m_widget, theSnapshot,
@@ -535,7 +535,7 @@ BaseTextEditorWidget::Link FollowSymbolUnderCursor::findLink(const QTextCursor &
                 } else if (tk.isOperator() && i > 0 && tokens.at(i - 1).is(T_OPERATOR)) {
                     QTextCursor c = cursor;
                     c.movePosition(QTextCursor::Left, QTextCursor::MoveAnchor,
-                                   positionInBlock - tokens.at(i - 1).bytesBegin());
+                                   positionInBlock - tokens.at(i - 1).utf16charsBegin());
                     link = attemptFuncDeclDef(c, m_widget, theSnapshot, documentFromSemanticInfo,
                                               symbolFinder);
                     if (link.hasValidLinkText())
@@ -564,8 +564,8 @@ BaseTextEditorWidget::Link FollowSymbolUnderCursor::findLink(const QTextCursor &
         const Token tk = SimpleLexer::tokenAt(block.text(), pos,
                                               BackwardsScanner::previousBlockState(block), true);
 
-        beginOfToken = block.position() + tk.bytesBegin();
-        endOfToken = block.position() + tk.bytesEnd();
+        beginOfToken = block.position() + tk.utf16charsBegin();
+        endOfToken = block.position() + tk.utf16charsEnd();
 
         // Handle include directives
         if (tk.is(T_STRING_LITERAL) || tk.is(T_ANGLE_STRING_LITERAL)) {

src/plugins/cppeditor/cppfunctiondecldeflink.cpp

@@ -414,17 +414,17 @@ static bool hasCommentedName(
     // maybe in a comment but in the right spot?
     int nameStart = 0;
     if (param->declarator)
-        nameStart = unit->tokenAt(param->declarator->lastToken() - 1).bytesEnd();
+        nameStart = unit->tokenAt(param->declarator->lastToken() - 1).utf16charsEnd();
     else if (param->type_specifier_list)
-        nameStart = unit->tokenAt(param->type_specifier_list->lastToken() - 1).bytesEnd();
+        nameStart = unit->tokenAt(param->type_specifier_list->lastToken() - 1).utf16charsEnd();
     else
-        nameStart = unit->tokenAt(param->firstToken()).bytesBegin();
+        nameStart = unit->tokenAt(param->firstToken()).utf16charsBegin();
 
     int nameEnd = 0;
     if (param->equal_token)
-        nameEnd = unit->tokenAt(param->equal_token).bytesBegin();
+        nameEnd = unit->tokenAt(param->equal_token).utf16charsBegin();
     else
-        nameEnd = unit->tokenAt(param->lastToken()).bytesBegin(); // one token after
+        nameEnd = unit->tokenAt(param->lastToken()).utf16charsBegin(); // one token after
 
     QString text = source.mid(nameStart, nameEnd - nameStart);
 

src/plugins/cppeditor/cpphighlighter.cpp

@@ -108,7 +108,7 @@ void CppHighlighter::highlightBlock(const QString &text)
         return;
     }
 
-    const unsigned firstNonSpace = tokens.first().bytesBegin();
+    const unsigned firstNonSpace = tokens.first().utf16charsBegin();
 
     Parentheses parentheses;
     parentheses.reserve(20); // assume wizard level ;-)
@@ -122,32 +122,32 @@ void CppHighlighter::highlightBlock(const QString &text)
         unsigned previousTokenEnd = 0;
         if (i != 0) {
             // mark the whitespaces
-            previousTokenEnd = tokens.at(i - 1).bytesBegin() +
-                               tokens.at(i - 1).bytes();
+            previousTokenEnd = tokens.at(i - 1).utf16charsBegin() +
+                               tokens.at(i - 1).utf16chars();
         }
 
-        if (previousTokenEnd != tk.bytesBegin()) {
+        if (previousTokenEnd != tk.utf16charsBegin()) {
             setFormat(previousTokenEnd,
-                      tk.bytesBegin() - previousTokenEnd,
+                      tk.utf16charsBegin() - previousTokenEnd,
                       formatForCategory(CppVisualWhitespace));
         }
 
         if (tk.is(T_LPAREN) || tk.is(T_LBRACE) || tk.is(T_LBRACKET)) {
-            const QChar c = text.at(tk.bytesBegin());
-            parentheses.append(Parenthesis(Parenthesis::Opened, c, tk.bytesBegin()));
+            const QChar c = text.at(tk.utf16charsBegin());
+            parentheses.append(Parenthesis(Parenthesis::Opened, c, tk.utf16charsBegin()));
             if (tk.is(T_LBRACE)) {
                 ++braceDepth;
 
                 // if a folding block opens at the beginning of a line, treat the entire line
                 // as if it were inside the folding block
-                if (tk.bytesBegin() == firstNonSpace) {
+                if (tk.utf16charsBegin() == firstNonSpace) {
                     ++foldingIndent;
                     BaseTextDocumentLayout::userData(currentBlock())->setFoldingStartIncluded(true);
                 }
             }
         } else if (tk.is(T_RPAREN) || tk.is(T_RBRACE) || tk.is(T_RBRACKET)) {
-            const QChar c = text.at(tk.bytesBegin());
-            parentheses.append(Parenthesis(Parenthesis::Closed, c, tk.bytesBegin()));
+            const QChar c = text.at(tk.utf16charsBegin());
+            parentheses.append(Parenthesis(Parenthesis::Closed, c, tk.utf16charsBegin()));
             if (tk.is(T_RBRACE)) {
                 --braceDepth;
                 if (braceDepth < foldingIndent) {
@@ -169,14 +169,14 @@ void CppHighlighter::highlightBlock(const QString &text)
             continue;
 
         if (i == 0 && tk.is(T_POUND)) {
-            highlightLine(text, tk.bytesBegin(), tk.bytes(),
+            highlightLine(text, tk.utf16charsBegin(), tk.utf16chars(),
                           formatForCategory(CppPreprocessorFormat));
             expectPreprocessorKeyword = true;
         } else if (highlightCurrentWordAsPreprocessor
                    && (tk.isKeyword() || tk.is(T_IDENTIFIER))
-                   && isPPKeyword(text.midRef(tk.bytesBegin(), tk.bytes()))) {
-            setFormat(tk.bytesBegin(), tk.bytes(), formatForCategory(CppPreprocessorFormat));
-            const QStringRef ppKeyword = text.midRef(tk.bytesBegin(), tk.bytes());
+                   && isPPKeyword(text.midRef(tk.utf16charsBegin(), tk.utf16chars()))) {
+            setFormat(tk.utf16charsBegin(), tk.utf16chars(), formatForCategory(CppPreprocessorFormat));
+            const QStringRef ppKeyword = text.midRef(tk.utf16charsBegin(), tk.utf16chars());
             if (ppKeyword == QLatin1String("error")
                     || ppKeyword == QLatin1String("warning")
                     || ppKeyword == QLatin1String("pragma")) {
@@ -184,18 +184,18 @@ void CppHighlighter::highlightBlock(const QString &text)
             }
 
         } else if (tk.is(T_NUMERIC_LITERAL)) {
-            setFormat(tk.bytesBegin(), tk.bytes(), formatForCategory(CppNumberFormat));
+            setFormat(tk.utf16charsBegin(), tk.utf16chars(), formatForCategory(CppNumberFormat));
         } else if (tk.isStringLiteral() || tk.isCharLiteral()) {
-            highlightLine(text, tk.bytesBegin(), tk.bytes(), formatForCategory(CppStringFormat));
+            highlightLine(text, tk.utf16charsBegin(), tk.utf16chars(), formatForCategory(CppStringFormat));
         } else if (tk.isComment()) {
-            const int startPosition = initialLexerState ? previousTokenEnd : tk.bytesBegin();
+            const int startPosition = initialLexerState ? previousTokenEnd : tk.utf16charsBegin();
             if (tk.is(T_COMMENT) || tk.is(T_CPP_COMMENT)) {
-                highlightLine(text, startPosition, tk.bytesEnd() - startPosition,
+                highlightLine(text, startPosition, tk.utf16charsEnd() - startPosition,
                               formatForCategory(CppCommentFormat));
             }
 
             else // a doxygen comment
-                highlightDoxygenComment(text, startPosition, tk.bytesEnd() - startPosition);
+                highlightDoxygenComment(text, startPosition, tk.utf16charsEnd() - startPosition);
 
             // we need to insert a close comment parenthesis, if
             //  - the line starts in a C Comment (initalState != 0)
@@ -208,7 +208,7 @@ void CppHighlighter::highlightBlock(const QString &text)
                     BaseTextDocumentLayout::userData(currentBlock())->setFoldingEndIncluded(true);
                 else
                     foldingIndent = qMin(braceDepth, foldingIndent);
-                const int tokenEnd = tk.bytesBegin() + tk.bytes() - 1;
+                const int tokenEnd = tk.utf16charsBegin() + tk.utf16chars() - 1;
                 parentheses.append(Parenthesis(Parenthesis::Closed, QLatin1Char('-'), tokenEnd));
 
                 // clear the initial state.
@@ -216,20 +216,21 @@ void CppHighlighter::highlightBlock(const QString &text)
             }
 
         } else if (tk.isKeyword()
-                   || CppTools::isQtKeyword(text.midRef(tk.bytesBegin(), tk.bytes()))
+                   || CppTools::isQtKeyword(text.midRef(tk.utf16charsBegin(), tk.utf16chars()))
                    || tk.isObjCAtKeyword()) {
-            setFormat(tk.bytesBegin(), tk.bytes(), formatForCategory(CppKeywordFormat));
+            setFormat(tk.utf16charsBegin(), tk.utf16chars(), formatForCategory(CppKeywordFormat));
         } else if (tk.isOperator()) {
-            setFormat(tk.bytesBegin(), tk.bytes(), formatForCategory(CppOperatorFormat));
+            setFormat(tk.utf16charsBegin(), tk.utf16chars(), formatForCategory(CppOperatorFormat));
         } else if (i == 0 && tokens.size() > 1 && tk.is(T_IDENTIFIER) && tokens.at(1).is(T_COLON)) {
-            setFormat(tk.bytesBegin(), tk.bytes(), formatForCategory(CppLabelFormat));
+            setFormat(tk.utf16charsBegin(), tk.utf16chars(), formatForCategory(CppLabelFormat));
         } else if (tk.is(T_IDENTIFIER)) {
-            highlightWord(text.midRef(tk.bytesBegin(), tk.bytes()), tk.bytesBegin(), tk.bytes());
+            highlightWord(text.midRef(tk.utf16charsBegin(), tk.utf16chars()), tk.utf16charsBegin(),
+                          tk.utf16chars());
         }
     }
 
     // mark the trailing white spaces
-    const int lastTokenEnd = tokens.last().bytesEnd();
+    const int lastTokenEnd = tokens.last().utf16charsEnd();
     if (text.length() > lastTokenEnd)
         highlightLine(text, lastTokenEnd, text.length() - lastTokenEnd, formatForCategory(CppVisualWhitespace));
 
@@ -237,7 +238,7 @@ void CppHighlighter::highlightBlock(const QString &text)
         const Token &lastToken = tokens.last();
         if (lastToken.is(T_COMMENT) || lastToken.is(T_DOXY_COMMENT)) {
             parentheses.append(Parenthesis(Parenthesis::Opened, QLatin1Char('+'),
-                                           lastToken.bytesBegin()));
+                                           lastToken.utf16charsBegin()));
             ++braceDepth;
         }
     }

src/plugins/cpptools/cppchecksymbols.cpp

@@ -356,7 +356,7 @@ bool CheckSymbols::warning(AST *ast, const QString &text)
     const Token &firstToken = tokenAt(ast->firstToken());
     const Token &lastToken = tokenAt(ast->lastToken() - 1);
 
-    const unsigned length = lastToken.bytesEnd() - firstToken.bytesBegin();
+    const unsigned length = lastToken.utf16charsEnd() - firstToken.utf16charsBegin();
     unsigned line = 1, column = 1;
     getTokenStartPosition(ast->firstToken(), &line, &column);
 
@@ -476,7 +476,7 @@ bool CheckSymbols::visit(NamespaceAST *ast)
         if (!tok.generated()) {
             unsigned line, column;
             getTokenStartPosition(ast->identifier_token, &line, &column);
-            Result use(line, column, tok.bytes(), CppHighlightingSupport::TypeUse);
+            Result use(line, column, tok.utf16chars(), CppHighlightingSupport::TypeUse);
             addUse(use);
         }
     }
@@ -726,8 +726,8 @@ void CheckSymbols::checkNamespace(NameAST *name)
         }
     }
 
-    const unsigned length = tokenAt(name->lastToken() - 1).bytesEnd()
-            - tokenAt(name->firstToken()).bytesBegin();
+    const unsigned length = tokenAt(name->lastToken() - 1).utf16charsEnd()
+            - tokenAt(name->firstToken()).utf16charsBegin();
     warning(line, column, QCoreApplication::translate("CPlusPlus::CheckSymbols",
                                                       "Expected a namespace-name"), length);
 }
@@ -1116,7 +1116,7 @@ void CheckSymbols::addUse(unsigned tokenIndex, Kind kind)
 
     unsigned line, column;
     getTokenStartPosition(tokenIndex, &line, &column);
-    const unsigned length = tok.bytes();
+    const unsigned length = tok.utf16chars();
 
     const Result use(line, column, length, kind);
     addUse(use);
@@ -1153,7 +1153,7 @@ void CheckSymbols::addType(ClassOrNamespace *b, NameAST *ast)
 
     unsigned line, column;
     getTokenStartPosition(startToken, &line, &column);
-    const unsigned length = tok.bytes();
+    const unsigned length = tok.utf16chars();
     const Result use(line, column, length, CppHighlightingSupport::TypeUse);
     addUse(use);
 }
@@ -1194,7 +1194,7 @@ bool CheckSymbols::maybeAddTypeOrStatic(const QList<LookupItem> &candidates, Nam
 
             unsigned line, column;
             getTokenStartPosition(startToken, &line, &column);
-            const unsigned length = tok.bytes();
+            const unsigned length = tok.utf16chars();
 
             Kind kind = CppHighlightingSupport::TypeUse;
             if (c->enclosingEnum() != 0)
@@ -1236,7 +1236,7 @@ bool CheckSymbols::maybeAddField(const QList<LookupItem> &candidates, NameAST *a
 
         unsigned line, column;
         getTokenStartPosition(startToken, &line, &column);
-        const unsigned length = tok.bytes();
+        const unsigned length = tok.utf16chars();
 
         const Result use(line, column, length, CppHighlightingSupport::FieldUse);
         addUse(use);
@@ -1320,7 +1320,7 @@ bool CheckSymbols::maybeAddFunction(const QList<LookupItem> &candidates, NameAST
 
         unsigned line, column;
         getTokenStartPosition(startToken, &line, &column);
-        const unsigned length = tok.bytes();
+        const unsigned length = tok.utf16chars();
 
         // Add a diagnostic message if argument count does not match
         if (matchType == Match_TooFewArgs)

src/plugins/cpptools/cppcodeformatter.cpp

@@ -1012,7 +1012,7 @@ int CodeFormatter::column(int index) const
 
 QStringRef CodeFormatter::currentTokenText() const
 {
-    return m_currentLine.midRef(m_currentToken.bytesBegin(), m_currentToken.bytes());
+    return m_currentLine.midRef(m_currentToken.utf16charsBegin(), m_currentToken.utf16chars());
 }
 
 void CodeFormatter::turnInto(int newState)
@@ -1189,10 +1189,10 @@ void QtStyleCodeFormatter::onEnter(int newState, int *indentDepth, int *savedInd
     const Token &tk = currentToken();
     const bool firstToken = (tokenIndex() == 0);
     const bool lastToken = (tokenIndex() == tokenCount() - 1);
-    const int tokenPosition = column(tk.bytesBegin());
-    const int nextTokenPosition = lastToken ? tokenPosition + tk.bytes()
-                                            : column(tokenAt(tokenIndex() + 1).bytesBegin());
-    const int spaceOrNextTokenPosition = lastToken ? tokenPosition + tk.bytes() + 1
+    const int tokenPosition = column(tk.utf16charsBegin());
+    const int nextTokenPosition = lastToken ? tokenPosition + tk.utf16chars()
+                                            : column(tokenAt(tokenIndex() + 1).utf16charsBegin());
+    const int spaceOrNextTokenPosition = lastToken ? tokenPosition + tk.utf16chars() + 1
                                                    : nextTokenPosition;
 
     if (shouldClearPaddingOnEnter(newState))
@@ -1474,7 +1474,7 @@ void QtStyleCodeFormatter::adjustIndent(const QList<CPlusPlus::Token> &tokens, i
     case multiline_comment_start:
     case multiline_comment_cont:
         if (!tokens.isEmpty()) {
-            *indentDepth = column(tokens.at(0).bytesBegin());
+            *indentDepth = column(tokens.at(0).utf16charsBegin());
             return;
         }
         break;

src/plugins/cpptools/cppcompletionassist.cpp

@@ -703,8 +703,8 @@ bool CppCompletionAssistProcessor::accepts() const
                         const QString &line = tc.block().text();
                         const Token &idToken = tokens.at(1);
                         const QStringRef &identifier =
-                                line.midRef(idToken.bytesBegin(),
-                                            idToken.bytesEnd() - idToken.bytesBegin());
+                                line.midRef(idToken.utf16charsBegin(),
+                                            idToken.utf16charsEnd() - idToken.utf16charsBegin());
                         if (identifier == QLatin1String("include")
                                 || identifier == QLatin1String("include_next")
                                 || (m_languageFeatures.objCEnabled && identifier == QLatin1String("import"))) {
@@ -838,8 +838,8 @@ int CppCompletionAssistProcessor::startOfOperator(int pos,
                 if (tokens.at(0).is(T_POUND) && tokens.at(1).is(T_IDENTIFIER) && (tokens.at(2).is(T_STRING_LITERAL) ||
                                                                                   tokens.at(2).is(T_ANGLE_STRING_LITERAL))) {
                     const Token &directiveToken = tokens.at(1);
-                    QString directive = tc.block().text().mid(directiveToken.bytesBegin(),
-                                                              directiveToken.bytes());
+                    QString directive = tc.block().text().mid(directiveToken.utf16charsBegin(),
+                                                              directiveToken.utf16chars());
                     if (directive == QLatin1String("include") ||
                             directive == QLatin1String("include_next") ||
                             directive == QLatin1String("import")) {

src/plugins/cpptools/cpprefactoringchanges.cpp

@@ -201,7 +201,7 @@ ChangeSet::Range CppRefactoringFile::range(unsigned tokenIndex) const
     unsigned line, column;
     cppDocument()->translationUnit()->getPosition(token.utf16charsBegin(), &line, &column);
     const int start = document()->findBlockByNumber(line - 1).position() + column - 1;
-    return ChangeSet::Range(start, start + token.bytes());
+    return ChangeSet::Range(start, start + token.utf16chars());
 }
 
 ChangeSet::Range CppRefactoringFile::range(AST *ast) const
@@ -241,7 +241,7 @@ void CppRefactoringFile::startAndEndOf(unsigned index, int *start, int *end) con
     Token token(tokenAt(index));
     cppDocument()->translationUnit()->getPosition(token.utf16charsBegin(), &line, &column);
     *start = document()->findBlockByNumber(line - 1).position() + column - 1;
-    *end = *start + token.bytes();
+    *end = *start + token.utf16chars();
 }
 
 QString CppRefactoringFile::textOf(const AST *ast) const

src/plugins/cpptools/doxygengenerator.cpp

@@ -83,7 +83,7 @@ QString DoxygenGenerator::generate(QTextCursor cursor)
         foreach (const Token &tk, tks) {
             if (tk.is(T_SEMICOLON) || tk.is(T_LBRACE)) {
                 // No need to continue beyond this, we might already have something meaningful.
-                cursor.setPosition(block.position() + tk.bytesEnd(), QTextCursor::KeepAnchor);
+                cursor.setPosition(block.position() + tk.utf16charsEnd(), QTextCursor::KeepAnchor);
                 break;
             }
         }

src/plugins/glsleditor/glslautocompleter.cpp

@@ -74,7 +74,7 @@ bool GLSLCompleter::contextAllowsElectricCharacters(const QTextCursor &cursor) c
     if (tk.isComment()) {
         const unsigned pos = cursor.selectionEnd() - cursor.block().position();
 
-        if (pos == tk.bytesEnd()) {
+        if (pos == tk.utf16charsEnd()) {
             if (tk.is(T_CPP_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))
                 return false;
 
@@ -83,11 +83,11 @@ bool GLSLCompleter::contextAllowsElectricCharacters(const QTextCursor &cursor) c
                 return false;