Commit 587eb49c authored by Nikolai Kosjar's avatar Nikolai Kosjar

C++: TranslationUnit::getPosition takes utf16char offsets

...and not byte offsets anymore. This is necessary in order to calculate
the line and column numbers correctly with respect to unicode code
points.

Change-Id: I5d79857b3eaefeb8d563b4f1e3938a64debc5e08
Reviewed-by: default avatarErik Verbruggen <erik.verbruggen@digia.com>
parent 70122b30
......@@ -86,7 +86,7 @@ void ASTVisitor::getTokenPosition(unsigned index,
{ translationUnit()->getTokenPosition(index, line, column, fileName); }
void ASTVisitor::getTokenStartPosition(unsigned index, unsigned *line, unsigned *column) const
{ getPosition(tokenAt(index).bytesBegin(), line, column); }
{ getPosition(tokenAt(index).utf16charsBegin(), line, column); }
void ASTVisitor::getTokenEndPosition(unsigned index, unsigned *line, unsigned *column) const
{ getPosition(tokenAt(index).bytesEnd(), line, column); }
{ getPosition(tokenAt(index).utf16charsEnd(), line, column); }
This diff is collapsed.
......@@ -109,7 +109,7 @@ void Lexer::pushLineStartOffset()
++_currentLine;
if (_translationUnit)
_translationUnit->pushLineOffset(_currentChar - _firstChar);
_translationUnit->pushLineOffset(_currentCharUtf16);
}
void Lexer::scan(Token *tok)
......
......@@ -166,7 +166,7 @@ void Symbol::setSourceLocation(unsigned sourceLocation, TranslationUnit *transla
if (translationUnit) {
const Token &tk = translationUnit->tokenAt(sourceLocation);
_isGenerated = tk.generated();
translationUnit->getPosition(tk.bytesBegin(), &_line, &_column, &_fileId);
translationUnit->getPosition(tk.utf16charsBegin(), &_line, &_column, &_fileId);
} else {
_isGenerated = false;
_line = 0;
......
......@@ -348,10 +348,10 @@ void TranslationUnit::pushPreprocessorLine(unsigned offset,
const StringLiteral *fileName)
{ _ppLines.push_back(PPLine(offset, line, fileName)); }
unsigned TranslationUnit::findLineNumber(unsigned offset) const
unsigned TranslationUnit::findLineNumber(unsigned utf16charOffset) const
{
std::vector<unsigned>::const_iterator it =
std::lower_bound(_lineOffsets.begin(), _lineOffsets.end(), offset);
std::lower_bound(_lineOffsets.begin(), _lineOffsets.end(), utf16charOffset);
if (it != _lineOffsets.begin())
--it;
......@@ -370,31 +370,31 @@ TranslationUnit::PPLine TranslationUnit::findPreprocessorLine(unsigned offset) c
return *it;
}
unsigned TranslationUnit::findColumnNumber(unsigned offset, unsigned lineNumber) const
unsigned TranslationUnit::findColumnNumber(unsigned utf16CharOffset, unsigned lineNumber) const
{
if (! offset)
if (! utf16CharOffset)
return 0;
return offset - _lineOffsets[lineNumber];
return utf16CharOffset - _lineOffsets[lineNumber];
}
void TranslationUnit::getTokenPosition(unsigned index,
unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).bytesBegin(), line, column, fileName); }
{ return getPosition(tokenAt(index).utf16charsBegin(), line, column, fileName); }
void TranslationUnit::getTokenStartPosition(unsigned index, unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).bytesBegin(), line, column, fileName); }
{ return getPosition(tokenAt(index).utf16charsBegin(), line, column, fileName); }
void TranslationUnit::getTokenEndPosition(unsigned index, unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).bytesEnd(), line, column, fileName); }
{ return getPosition(tokenAt(index).utf16charsEnd(), line, column, fileName); }
void TranslationUnit::getPosition(unsigned tokenOffset,
void TranslationUnit::getPosition(unsigned utf16charOffset,
unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
......@@ -405,20 +405,20 @@ void TranslationUnit::getPosition(unsigned tokenOffset,
// If this token is expanded we already have the information directly from the expansion
// section header. Otherwise, we need to calculate it.
TokenLineColumn::const_iterator it = _expandedLineColumn.find(tokenOffset);
TokenLineColumn::const_iterator it = _expandedLineColumn.find(utf16charOffset);
if (it != _expandedLineColumn.end()) {
lineNumber = it->second.first;
columnNumber = it->second.second + 1;
file = _fileId;
} else {
// Identify line within the entire translation unit.
lineNumber = findLineNumber(tokenOffset);
lineNumber = findLineNumber(utf16charOffset);
// Identify column.
columnNumber = findColumnNumber(tokenOffset, lineNumber);
columnNumber = findColumnNumber(utf16charOffset, lineNumber);
// Adjust the line in regards to the preprocessing markers.
const PPLine ppLine = findPreprocessorLine(tokenOffset);
const PPLine ppLine = findPreprocessorLine(utf16charOffset);
lineNumber -= findLineNumber(ppLine.offset) + 1;
lineNumber += ppLine.line;
......@@ -508,7 +508,7 @@ void TranslationUnit::fatal(unsigned index, const char *format, ...)
unsigned TranslationUnit::findPreviousLineOffset(unsigned tokenIndex) const
{
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(tokenIndex).bytesBegin())];
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(tokenIndex).utf16charsBegin())];
return lineOffset;
}
......@@ -522,13 +522,16 @@ bool TranslationUnit::maybeSplitGreaterGreaterToken(unsigned tokenIndex)
tok.f.kind = T_GREATER;
tok.f.bytes = 1;
tok.f.utf16chars = 1;
Token newGreater;
newGreater.f.kind = T_GREATER;
newGreater.f.expanded = tok.expanded();
newGreater.f.generated = tok.generated();
newGreater.f.bytes = 1;
newGreater.f.utf16chars = 1;
newGreater.byteOffset = tok.byteOffset + 1;
newGreater.utf16charOffset = tok.utf16charOffset + 1;
_tokens->insert(_tokens->begin() + tokenIndex + 1, newGreater);
......@@ -551,7 +554,7 @@ void TranslationUnit::releaseTokensAndComments()
void TranslationUnit::showErrorLine(unsigned index, unsigned column, FILE *out)
{
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(index).bytesBegin())];
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(index).utf16charsBegin())];
for (const char *cp = _firstSourceChar + lineOffset + 1; *cp && *cp != '\n'; ++cp) {
fputc(*cp, out);
}
......
......@@ -127,7 +127,7 @@ public:
unsigned *column = 0,
const StringLiteral **fileName = 0) const;
void getPosition(unsigned offset,
void getPosition(unsigned utf16charOffset,
unsigned *line,
unsigned *column = 0,
const StringLiteral **fileName = 0) const;
......@@ -172,8 +172,8 @@ private:
};
void releaseTokensAndComments();
unsigned findLineNumber(unsigned offset) const;
unsigned findColumnNumber(unsigned offset, unsigned lineNumber) const;
unsigned findLineNumber(unsigned utf16charOffset) const;
unsigned findColumnNumber(unsigned utf16CharOffset, unsigned lineNumber) const;
PPLine findPreprocessorLine(unsigned offset) const;
void showErrorLine(unsigned index, unsigned column, FILE *out);
......
......@@ -852,7 +852,7 @@ void TokensModel::configure(CPlusPlus::TranslationUnit *translationUnit)
for (int i = 0, total = translationUnit->tokenCount(); i < total; ++i) {
TokenInfo info;
info.token = translationUnit->tokenAt(i);
translationUnit->getPosition(info.token.bytesBegin(), &info.line, &info.column);
translationUnit->getPosition(info.token.utf16charsBegin(), &info.line, &info.column);
m_tokenInfos.append(info);
}
emit layoutChanged();
......
......@@ -238,7 +238,7 @@ TestActionsTestCase::TestActionsTestCase(const Actions &tokenActions, const Acti
} else {
// Position the cursor on the token
unsigned line, column;
translationUnit->getPosition(token.bytesBegin(), &line, &column);
translationUnit->getPosition(token.utf16charsBegin(), &line, &column);
editor->gotoLine(line, column - 1);
QApplication::processEvents();
......@@ -293,7 +293,7 @@ void TestActionsTestCase::moveWordCamelCaseToToken(TranslationUnit *translationU
QVERIFY(editorWidget);
unsigned line, column;
translationUnit->getPosition(token.bytesBegin(), &line, &column);
translationUnit->getPosition(token.utf16charsBegin(), &line, &column);
while (editor->currentLine() < (int) line
|| (editor->currentLine() == (int) line
......
......@@ -199,7 +199,7 @@ ChangeSet::Range CppRefactoringFile::range(unsigned tokenIndex) const
{
const Token &token = tokenAt(tokenIndex);
unsigned line, column;
cppDocument()->translationUnit()->getPosition(token.bytesBegin(), &line, &column);
cppDocument()->translationUnit()->getPosition(token.utf16charsBegin(), &line, &column);
const int start = document()->findBlockByNumber(line - 1).position() + column - 1;
return ChangeSet::Range(start, start + token.bytes());
}
......@@ -212,7 +212,7 @@ ChangeSet::Range CppRefactoringFile::range(AST *ast) const
int CppRefactoringFile::startOf(unsigned index) const
{
unsigned line, column;
cppDocument()->translationUnit()->getPosition(tokenAt(index).bytesBegin(), &line, &column);
cppDocument()->translationUnit()->getPosition(tokenAt(index).utf16charsBegin(), &line, &column);
return document()->findBlockByNumber(line - 1).position() + column - 1;
}
......@@ -224,7 +224,7 @@ int CppRefactoringFile::startOf(const AST *ast) const
int CppRefactoringFile::endOf(unsigned index) const
{
unsigned line, column;
cppDocument()->translationUnit()->getPosition(tokenAt(index).bytesEnd(), &line, &column);
cppDocument()->translationUnit()->getPosition(tokenAt(index).utf16charsEnd(), &line, &column);
return document()->findBlockByNumber(line - 1).position() + column - 1;
}
......@@ -239,7 +239,7 @@ void CppRefactoringFile::startAndEndOf(unsigned index, int *start, int *end) con
{
unsigned line, column;
Token token(tokenAt(index));
cppDocument()->translationUnit()->getPosition(token.bytesBegin(), &line, &column);
cppDocument()->translationUnit()->getPosition(token.utf16charsBegin(), &line, &column);
*start = document()->findBlockByNumber(line - 1).position() + column - 1;
*end = *start + token.bytes();
}
......
......@@ -98,7 +98,7 @@ void CppTodoItemsScanner::processDocument(CPlusPlus::Document::Ptr doc)
const QStringList commentLines =
QString::fromUtf8(source).split(QLatin1Char('\n'), QString::SkipEmptyParts);
unsigned lineNumber = 0;
translationUnit->getPosition(token.bytesBegin(), &lineNumber);
translationUnit->getPosition(token.utf16charsBegin(), &lineNumber);
for (int j = 0; j < commentLines.count(); ++j) {
const QString &commentLine = commentLines.at(j);
processCommentLine(doc->fileName(), commentLine, lineNumber + j, itemList);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment