Commit 587eb49c authored by Nikolai Kosjar's avatar Nikolai Kosjar
Browse files

C++: TranslationUnit::getPosition takes utf16char offsets



...and not byte offsets anymore. This is necessary in order to calculate
the line and column numbers correctly with respect to unicode code
points.

Change-Id: I5d79857b3eaefeb8d563b4f1e3938a64debc5e08
Reviewed-by: default avatarErik Verbruggen <erik.verbruggen@digia.com>
parent 70122b30
......@@ -86,7 +86,7 @@ void ASTVisitor::getTokenPosition(unsigned index,
{ translationUnit()->getTokenPosition(index, line, column, fileName); }
void ASTVisitor::getTokenStartPosition(unsigned index, unsigned *line, unsigned *column) const
{ getPosition(tokenAt(index).bytesBegin(), line, column); }
{ getPosition(tokenAt(index).utf16charsBegin(), line, column); }
void ASTVisitor::getTokenEndPosition(unsigned index, unsigned *line, unsigned *column) const
{ getPosition(tokenAt(index).bytesEnd(), line, column); }
{ getPosition(tokenAt(index).utf16charsEnd(), line, column); }
......@@ -1094,8 +1094,8 @@ void Bind::lambdaDeclarator(LambdaDeclaratorAST *ast)
Function *fun = control()->newFunction(0, 0);
fun->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
fun->setStartOffset(tokenAt(ast->firstToken()).utf16charsBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (ast->trailing_return_type)
_type = this->trailingReturnType(ast->trailing_return_type, _type);
fun->setReturnType(_type);
......@@ -1192,8 +1192,8 @@ bool Bind::visit(CompoundStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
unsigned startScopeToken = ast->lbrace_token ? ast->lbrace_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
ast->symbol = block;
_scope->addMember(block);
Scope *previousScope = switchScope(block);
......@@ -1235,8 +1235,8 @@ bool Bind::visit(ForeachStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
......@@ -1285,8 +1285,8 @@ bool Bind::visit(RangeBasedForStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
......@@ -1334,8 +1334,8 @@ bool Bind::visit(ForStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
......@@ -1354,8 +1354,8 @@ bool Bind::visit(IfStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
......@@ -1410,8 +1410,8 @@ bool Bind::visit(SwitchStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
......@@ -1436,8 +1436,8 @@ bool Bind::visit(CatchClauseAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
......@@ -1453,8 +1453,8 @@ bool Bind::visit(WhileStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
......@@ -1469,8 +1469,8 @@ bool Bind::visit(ObjCFastEnumerationAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
......@@ -2139,7 +2139,7 @@ bool Bind::visit(FunctionDefinitionAST *ast)
if (fun) {
setDeclSpecifiers(fun, declSpecifiers);
fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (_scope->isClass()) {
fun->setVisibility(_visibility);
......@@ -2201,8 +2201,8 @@ bool Bind::visit(NamespaceAST *ast)
}
Namespace *ns = control()->newNamespace(sourceLocation, namespaceName);
ns->setStartOffset(tokenAt(sourceLocation).bytesEnd()); // the scope starts after the namespace or the identifier token.
ns->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
ns->setStartOffset(tokenAt(sourceLocation).utf16charsEnd()); // the scope starts after the namespace or the identifier token.
ns->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
ns->setInline(ast->inline_token != 0);
ast->symbol = ns;
_scope->addMember(ns);
......@@ -2261,8 +2261,8 @@ bool Bind::visit(ParameterDeclarationAST *ast)
bool Bind::visit(TemplateDeclarationAST *ast)
{
Template *templ = control()->newTemplate(ast->firstToken(), 0);
templ->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
templ->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
templ->setStartOffset(tokenAt(ast->firstToken()).utf16charsBegin());
templ->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
ast->symbol = templ;
Scope *previousScope = switchScope(templ);
......@@ -2375,34 +2375,34 @@ unsigned Bind::calculateScopeStart(ObjCClassDeclarationAST *ast) const
{
if (ast->inst_vars_decl)
if (unsigned pos = ast->inst_vars_decl->lbrace_token)
return tokenAt(pos).bytesEnd();
return tokenAt(pos).utf16charsEnd();
if (ast->protocol_refs)
if (unsigned pos = ast->protocol_refs->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
if (ast->superclass)
if (unsigned pos = ast->superclass->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
if (ast->colon_token)
return tokenAt(ast->colon_token).bytesEnd();
return tokenAt(ast->colon_token).utf16charsEnd();
if (ast->rparen_token)
return tokenAt(ast->rparen_token).bytesEnd();
return tokenAt(ast->rparen_token).utf16charsEnd();
if (ast->category_name)
if (unsigned pos = ast->category_name->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
if (ast->lparen_token)
return tokenAt(ast->lparen_token).bytesEnd();
return tokenAt(ast->lparen_token).utf16charsEnd();
if (ast->class_name)
if (unsigned pos = ast->class_name->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
return tokenAt(ast->firstToken()).bytesBegin();
return tokenAt(ast->firstToken()).utf16charsBegin();
}
bool Bind::visit(ObjCClassDeclarationAST *ast)
......@@ -2420,7 +2420,7 @@ bool Bind::visit(ObjCClassDeclarationAST *ast)
_scope->addMember(klass);
klass->setStartOffset(calculateScopeStart(ast));
klass->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
klass->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (ast->interface_token)
klass->setInterface(true);
......@@ -2479,12 +2479,12 @@ unsigned Bind::calculateScopeStart(ObjCProtocolDeclarationAST *ast) const
{
if (ast->protocol_refs)
if (unsigned pos = ast->protocol_refs->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
if (ast->name)
if (unsigned pos = ast->name->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
return tokenAt(ast->firstToken()).bytesBegin();
return tokenAt(ast->firstToken()).utf16charsBegin();
}
bool Bind::visit(ObjCProtocolDeclarationAST *ast)
......@@ -2499,7 +2499,7 @@ bool Bind::visit(ObjCProtocolDeclarationAST *ast)
const unsigned sourceLocation = location(ast->name, ast->firstToken());
ObjCProtocol *protocol = control()->newObjCProtocol(sourceLocation, name);
protocol->setStartOffset(calculateScopeStart(ast));
protocol->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
protocol->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
ast->symbol = protocol;
_scope->addMember(protocol);
......@@ -2920,7 +2920,7 @@ bool Bind::visit(ClassSpecifierAST *ast)
{
// unsigned classkey_token = ast->classkey_token;
unsigned sourceLocation = ast->firstToken();
unsigned startScopeOffset = tokenAt(sourceLocation).bytesEnd(); // at the end of the class key
unsigned startScopeOffset = tokenAt(sourceLocation).utf16charsEnd(); // at the end of the class key
for (SpecifierListAST *it = ast->attribute_list; it; it = it->next) {
_type = this->specifier(it->value, _type);
......@@ -2930,12 +2930,12 @@ bool Bind::visit(ClassSpecifierAST *ast)
if (ast->name && ! ast->name->asAnonymousName()) {
sourceLocation = location(ast->name, sourceLocation);
startScopeOffset = tokenAt(sourceLocation).bytesEnd(); // at the end of the class name
startScopeOffset = tokenAt(sourceLocation).utf16charsEnd(); // at the end of the class name
if (QualifiedNameAST *q = ast->name->asQualifiedName()) {
if (q->unqualified_name) {
sourceLocation = q->unqualified_name->firstToken();
startScopeOffset = tokenAt(q->unqualified_name->lastToken() - 1).bytesEnd(); // at the end of the unqualified name
startScopeOffset = tokenAt(q->unqualified_name->lastToken() - 1).utf16charsEnd(); // at the end of the unqualified name
}
}
......@@ -2944,7 +2944,7 @@ bool Bind::visit(ClassSpecifierAST *ast)
Class *klass = control()->newClass(sourceLocation, className);
klass->setStartOffset(startScopeOffset);
klass->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
klass->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
_scope->addMember(klass);
if (_scope->isClass())
......@@ -3003,8 +3003,8 @@ bool Bind::visit(EnumSpecifierAST *ast)
const Name *enumName = this->name(ast->name);
Enum *e = control()->newEnum(sourceLocation, enumName);
e->setStartOffset(tokenAt(sourceLocation).bytesEnd()); // at the end of the enum or identifier token.
e->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
e->setStartOffset(tokenAt(sourceLocation).utf16charsEnd()); // at the end of the enum or identifier token.
e->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (ast->key_token)
e->setScoped(true);
ast->symbol = e;
......@@ -3126,8 +3126,8 @@ bool Bind::visit(NestedDeclaratorAST *ast)
bool Bind::visit(FunctionDeclaratorAST *ast)
{
Function *fun = control()->newFunction(0, 0);
fun->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
fun->setStartOffset(tokenAt(ast->firstToken()).utf16charsBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (ast->trailing_return_type)
_type = this->trailingReturnType(ast->trailing_return_type, _type);
fun->setReturnType(_type);
......
......@@ -109,7 +109,7 @@ void Lexer::pushLineStartOffset()
++_currentLine;
if (_translationUnit)
_translationUnit->pushLineOffset(_currentChar - _firstChar);
_translationUnit->pushLineOffset(_currentCharUtf16);
}
void Lexer::scan(Token *tok)
......
......@@ -166,7 +166,7 @@ void Symbol::setSourceLocation(unsigned sourceLocation, TranslationUnit *transla
if (translationUnit) {
const Token &tk = translationUnit->tokenAt(sourceLocation);
_isGenerated = tk.generated();
translationUnit->getPosition(tk.bytesBegin(), &_line, &_column, &_fileId);
translationUnit->getPosition(tk.utf16charsBegin(), &_line, &_column, &_fileId);
} else {
_isGenerated = false;
_line = 0;
......
......@@ -348,10 +348,10 @@ void TranslationUnit::pushPreprocessorLine(unsigned offset,
const StringLiteral *fileName)
{ _ppLines.push_back(PPLine(offset, line, fileName)); }
unsigned TranslationUnit::findLineNumber(unsigned offset) const
unsigned TranslationUnit::findLineNumber(unsigned utf16charOffset) const
{
std::vector<unsigned>::const_iterator it =
std::lower_bound(_lineOffsets.begin(), _lineOffsets.end(), offset);
std::lower_bound(_lineOffsets.begin(), _lineOffsets.end(), utf16charOffset);
if (it != _lineOffsets.begin())
--it;
......@@ -370,31 +370,31 @@ TranslationUnit::PPLine TranslationUnit::findPreprocessorLine(unsigned offset) c
return *it;
}
unsigned TranslationUnit::findColumnNumber(unsigned offset, unsigned lineNumber) const
unsigned TranslationUnit::findColumnNumber(unsigned utf16CharOffset, unsigned lineNumber) const
{
if (! offset)
if (! utf16CharOffset)
return 0;
return offset - _lineOffsets[lineNumber];
return utf16CharOffset - _lineOffsets[lineNumber];
}
void TranslationUnit::getTokenPosition(unsigned index,
unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).bytesBegin(), line, column, fileName); }
{ return getPosition(tokenAt(index).utf16charsBegin(), line, column, fileName); }
void TranslationUnit::getTokenStartPosition(unsigned index, unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).bytesBegin(), line, column, fileName); }
{ return getPosition(tokenAt(index).utf16charsBegin(), line, column, fileName); }
void TranslationUnit::getTokenEndPosition(unsigned index, unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).bytesEnd(), line, column, fileName); }
{ return getPosition(tokenAt(index).utf16charsEnd(), line, column, fileName); }
void TranslationUnit::getPosition(unsigned tokenOffset,
void TranslationUnit::getPosition(unsigned utf16charOffset,
unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
......@@ -405,20 +405,20 @@ void TranslationUnit::getPosition(unsigned tokenOffset,
// If this token is expanded we already have the information directly from the expansion
// section header. Otherwise, we need to calculate it.
TokenLineColumn::const_iterator it = _expandedLineColumn.find(tokenOffset);
TokenLineColumn::const_iterator it = _expandedLineColumn.find(utf16charOffset);
if (it != _expandedLineColumn.end()) {
lineNumber = it->second.first;
columnNumber = it->second.second + 1;
file = _fileId;
} else {
// Identify line within the entire translation unit.
lineNumber = findLineNumber(tokenOffset);
lineNumber = findLineNumber(utf16charOffset);
// Identify column.
columnNumber = findColumnNumber(tokenOffset, lineNumber);
columnNumber = findColumnNumber(utf16charOffset, lineNumber);
// Adjust the line in regards to the preprocessing markers.
const PPLine ppLine = findPreprocessorLine(tokenOffset);
const PPLine ppLine = findPreprocessorLine(utf16charOffset);
lineNumber -= findLineNumber(ppLine.offset) + 1;
lineNumber += ppLine.line;
......@@ -508,7 +508,7 @@ void TranslationUnit::fatal(unsigned index, const char *format, ...)
unsigned TranslationUnit::findPreviousLineOffset(unsigned tokenIndex) const
{
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(tokenIndex).bytesBegin())];
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(tokenIndex).utf16charsBegin())];
return lineOffset;
}
......@@ -522,13 +522,16 @@ bool TranslationUnit::maybeSplitGreaterGreaterToken(unsigned tokenIndex)
tok.f.kind = T_GREATER;
tok.f.bytes = 1;
tok.f.utf16chars = 1;
Token newGreater;
newGreater.f.kind = T_GREATER;
newGreater.f.expanded = tok.expanded();
newGreater.f.generated = tok.generated();
newGreater.f.bytes = 1;
newGreater.f.utf16chars = 1;
newGreater.byteOffset = tok.byteOffset + 1;
newGreater.utf16charOffset = tok.utf16charOffset + 1;
_tokens->insert(_tokens->begin() + tokenIndex + 1, newGreater);
......@@ -551,7 +554,7 @@ void TranslationUnit::releaseTokensAndComments()
void TranslationUnit::showErrorLine(unsigned index, unsigned column, FILE *out)
{
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(index).bytesBegin())];
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(index).utf16charsBegin())];
for (const char *cp = _firstSourceChar + lineOffset + 1; *cp && *cp != '\n'; ++cp) {
fputc(*cp, out);
}
......
......@@ -127,7 +127,7 @@ public:
unsigned *column = 0,
const StringLiteral **fileName = 0) const;
void getPosition(unsigned offset,
void getPosition(unsigned utf16charOffset,
unsigned *line,
unsigned *column = 0,
const StringLiteral **fileName = 0) const;
......@@ -172,8 +172,8 @@ private:
};
void releaseTokensAndComments();
unsigned findLineNumber(unsigned offset) const;
unsigned findColumnNumber(unsigned offset, unsigned lineNumber) const;
unsigned findLineNumber(unsigned utf16charOffset) const;
unsigned findColumnNumber(unsigned utf16CharOffset, unsigned lineNumber) const;
PPLine findPreprocessorLine(unsigned offset) const;
void showErrorLine(unsigned index, unsigned column, FILE *out);
......
......@@ -852,7 +852,7 @@ void TokensModel::configure(CPlusPlus::TranslationUnit *translationUnit)
for (int i = 0, total = translationUnit->tokenCount(); i < total; ++i) {
TokenInfo info;
info.token = translationUnit->tokenAt(i);
translationUnit->getPosition(info.token.bytesBegin(), &info.line, &info.column);
translationUnit->getPosition(info.token.utf16charsBegin(), &info.line, &info.column);
m_tokenInfos.append(info);
}
emit layoutChanged();
......
......@@ -238,7 +238,7 @@ TestActionsTestCase::TestActionsTestCase(const Actions &tokenActions, const Acti
} else {
// Position the cursor on the token
unsigned line, column;
translationUnit->getPosition(token.bytesBegin(), &line, &column);
translationUnit->getPosition(token.utf16charsBegin(), &line, &column);
editor->gotoLine(line, column - 1);
QApplication::processEvents();
......@@ -293,7 +293,7 @@ void TestActionsTestCase::moveWordCamelCaseToToken(TranslationUnit *translationU
QVERIFY(editorWidget);
unsigned line, column;
translationUnit->getPosition(token.bytesBegin(), &line, &column);
translationUnit->getPosition(token.utf16charsBegin(), &line, &column);
while (editor->currentLine() < (int) line
|| (editor->currentLine() == (int) line
......
......@@ -199,7 +199,7 @@ ChangeSet::Range CppRefactoringFile::range(unsigned tokenIndex) const
{
const Token &token = tokenAt(tokenIndex);
unsigned line, column;
cppDocument()->translationUnit()->getPosition(token.bytesBegin(), &line, &column);
cppDocument()->translationUnit()->getPosition(token.utf16charsBegin(), &line, &column);
const int start = document()->findBlockByNumber(line - 1).position() + column - 1;
return ChangeSet::Range(start, start + token.bytes());
}
......@@ -212,7 +212,7 @@ ChangeSet::Range CppRefactoringFile::range(AST *ast) const
int CppRefactoringFile::startOf(unsigned index) const
{
unsigned line, column;
cppDocument()->translationUnit()->getPosition(tokenAt(index).bytesBegin(), &line, &column);
cppDocument()->translationUnit()->getPosition(tokenAt(index).utf16charsBegin(), &line, &column);
return document()->findBlockByNumber(line - 1).position() + column - 1;
}
......@@ -224,7 +224,7 @@ int CppRefactoringFile::startOf(const AST *ast) const
int CppRefactoringFile::endOf(unsigned index) const
{
unsigned line, column;
cppDocument()->translationUnit()->getPosition(tokenAt(index).bytesEnd(), &line, &column);
cppDocument()->translationUnit()->getPosition(tokenAt(index).utf16charsEnd(), &line, &column);
return document()->findBlockByNumber(line - 1).position() + column - 1;
}
......@@ -239,7 +239,7 @@ void CppRefactoringFile::startAndEndOf(unsigned index, int *start, int *end) con
{
unsigned line, column;
Token token(tokenAt(index));
cppDocument()->translationUnit()->getPosition(token.bytesBegin(), &line, &column);
cppDocument()->translationUnit()->getPosition(token.utf16charsBegin(), &line, &column);
*start = document()->findBlockByNumber(line - 1).position() + column - 1;
*end = *start + token.bytes();
}
......
......@@ -98,7 +98,7 @@ void CppTodoItemsScanner::processDocument(CPlusPlus::Document::Ptr doc)
const QStringList commentLines =
QString::fromUtf8(source).split(QLatin1Char('\n'), QString::SkipEmptyParts);
unsigned lineNumber = 0;
translationUnit->getPosition(token.bytesBegin(), &lineNumber);
translationUnit->getPosition(token.utf16charsBegin(), &lineNumber);
for (int j = 0; j < commentLines.count(); ++j) {
const QString &commentLine = commentLines.at(j);
processCommentLine(doc->fileName(), commentLine, lineNumber + j, itemList);
......
......@@ -36,6 +36,15 @@
#include <QtTest>
#include <QDebug>
struct LineColumn
{
LineColumn(unsigned line = 0, unsigned column = 0) : line(line), column(column) {}
unsigned line;
unsigned column;
};
typedef QList<LineColumn> LineColumnList;
Q_DECLARE_METATYPE(LineColumnList)
//TESTED_COMPONENT=src/libs/cplusplus
using namespace CPlusPlus;
......@@ -58,6 +67,9 @@ private slots:
void unicodeStringLiteral();
void unicodeStringLiteral_data();
void locationOfUtf16CharOffset();
void locationOfUtf16CharOffset_data();
private:
class Document
{
......@@ -132,8 +144,49 @@ private:
}
} m_diagnosticClient;
};
class TokenGetter
{
public:
typedef QSharedPointer<TokenGetter> Ptr;
public:
TokenGetter(TranslationUnit *translationUnit) : m_translationUnit(translationUnit) {}
virtual ~TokenGetter() {}
virtual unsigned tokenCount() { return m_translationUnit->tokenCount(); }
virtual Token tokenAt(unsigned index) { return m_translationUnit->tokenAt(index); }
protected:
TranslationUnit *m_translationUnit;
};
class CommentTokenGetter : public TokenGetter
{
public:
CommentTokenGetter(TranslationUnit *translationUnit) : TokenGetter(translationUnit) {}
unsigned tokenCount() { return m_translationUnit->commentCount(); }
Token tokenAt(unsigned index) { return m_translationUnit->commentAt(index); }
};
static void compareTokenLocations(TranslationUnit *translationUnit,
tst_TranslationUnit::TokenGetter *tokenGetter,
const LineColumnList &expectedLinesColumns);
};
void tst_TranslationUnit::compareTokenLocations(TranslationUnit *translationUnit,
tst_TranslationUnit::TokenGetter *tokenGetter,
const LineColumnList &expectedLinesColumns)
{
QCOMPARE(tokenGetter->tokenCount(), (unsigned) expectedLinesColumns.count());
for (unsigned i = 0, tokenCount = tokenGetter->tokenCount(); i < tokenCount; ++i) {
const LineColumn expected = expectedLinesColumns.at(i);
const unsigned utf16CharOffset = tokenGetter->tokenAt(i).utf16charsBegin();
unsigned line, column;
translationUnit->getPosition(utf16CharOffset, &line, &column);
// qDebug("%d: LineColumn(%u, %u)", i, line, column);
QCOMPARE(line, expected.line);
QCOMPARE(column, expected.column);
}
}
void tst_TranslationUnit::unicodeIdentifier()
{
QFETCH(QByteArray, identifierText);
......@@ -221,5 +274,231 @@ void tst_TranslationUnit::unicodeStringLiteral_data()
QTest::newRow("non-latin1 literal 10") << _("U\"\U00010302\u00FC\u4E8C\"");
}