Cpp{Tools,Editor}: Expect UTF-8 encoded literals

Change-Id: I9843c4163aad3fa3f1bfa33060c76328fc2dc25a
Reviewed-by: Erik Verbruggen <erik.verbruggen@digia.com>

src/libs/cplusplus/NamePrettyPrinter.cpp

@@ -67,7 +67,7 @@ void NamePrettyPrinter::visit(const Identifier *name)
 {
     const Identifier *id = name->identifier();
     if (id)
-        _name = QString::fromLatin1(id->chars(), id->size());
+        _name = QString::fromUtf8(id->chars(), id->size());
     else
         _name = QLatin1String("anonymous");
 }
@@ -76,7 +76,7 @@ void NamePrettyPrinter::visit(const TemplateNameId *name)
 {
     const Identifier *id = name->identifier();
     if (id)
-        _name = QString::fromLatin1(id->chars(), id->size());
+        _name = QString::fromUtf8(id->chars(), id->size());
     else
         _name = QLatin1String("anonymous");
     _name += QLatin1Char('<');
@@ -100,7 +100,7 @@ void NamePrettyPrinter::visit(const DestructorNameId *name)
 {
     const Identifier *id = name->identifier();
     _name += QLatin1Char('~');
-    _name += QString::fromLatin1(id->chars(), id->size());
+    _name += QString::fromUtf8(id->chars(), id->size());
 }
 
 void NamePrettyPrinter::visit(const OperatorNameId *name)
@@ -263,7 +263,7 @@ void NamePrettyPrinter::visit(const SelectorNameId *name)
             continue;
 
         if (const Identifier *id = n->identifier()) {
-            _name += QString::fromLatin1(id->chars(), id->size());
+            _name += QString::fromUtf8(id->chars(), id->size());
 
             if (name->hasArguments() || name->nameCount() > 1)
                 _name += QLatin1Char(':');

src/plugins/cppeditor/cppeditorplugin.h

@@ -143,6 +143,7 @@ private slots:
     void test_quickfix_InsertDefFromDecl_erroneousStatementAtEndOfFile();
     void test_quickfix_InsertDefFromDecl_rvalueReference();
     void test_quickfix_InsertDefFromDecl_findImplementationFile();
+    void test_quickfix_InsertDefFromDecl_unicodeIdentifier();
 
     void test_quickfix_InsertDeclFromDef();
 

src/plugins/cppeditor/cppfunctiondecldeflink.cpp

@@ -802,7 +802,7 @@ Utils::ChangeSet FunctionDeclDefLink::changes(const Snapshot &snapshot, int targ
 
                 // don't change the name if it's in a comment
                 if (hasCommentedName(targetFile->cppDocument()->translationUnit(),
-                                     QLatin1String(targetFile->cppDocument()->utf8Source()),
+                                     QString::fromUtf8(targetFile->cppDocument()->utf8Source()),
                                      targetFunctionDeclarator, existingParamIndex))
                     replacementName = 0;
 

src/plugins/cppeditor/cppquickfix_test.cpp

@@ -1777,6 +1777,32 @@ void CppEditorPlugin::test_quickfix_InsertDefFromDecl_findImplementationFile()
     QuickFixTestCase(testFiles, &factory);
 }
 
+void CppEditorPlugin::test_quickfix_InsertDefFromDecl_unicodeIdentifier()
+{
+    QList<QuickFixTestDocument::Ptr> testFiles;
+
+    QByteArray original;
+    QByteArray expected;
+
+    original =
+            "class Foo {\n"
+            "    void @\u00FC\u4E8C\U00010302();\n"
+            "};\n";
+            ;
+    expected = original;
+    expected +=
+            "\n"
+            "\n"
+            "void Foo::\u00FC\u4E8C\U00010302()\n"
+            "{\n"
+            "\n"
+            "}\n";
+    testFiles << QuickFixTestDocument::create("file.cpp", original, expected);
+
+    InsertDefFromDecl factory;
+    QuickFixTestCase(testFiles, &factory);
+}
+
 // Function for one of InsertDeclDef section cases
 void insertToSectionDeclFromDef(const QByteArray &section, int sectionIndex)
 {

src/plugins/cppeditor/cppquickfixes.cpp

@@ -2823,7 +2823,7 @@ public:
             m_offerQuickFix = false;
             return;
         }
-        m_variableString = QString::fromLatin1(variableId->chars(), variableId->size());
+        m_variableString = QString::fromUtf8(variableId->chars(), variableId->size());
 
         m_baseName = m_variableString;
         if (m_baseName.startsWith(QLatin1String("m_")))
@@ -2850,7 +2850,7 @@ public:
                 Symbol *symbol = klass->memberAt(i);
                 if (const Name *symbolName = symbol->name()) {
                     if (const Identifier *id = symbolName->identifier()) {
-                        const QString memberName = QString::fromLatin1(id->chars(), id->size());
+                        const QString memberName = QString::fromUtf8(id->chars(), id->size());
                         if (memberName == m_getterName || memberName == m_setterName) {
                             m_offerQuickFix = false;
                             return;
@@ -2886,7 +2886,7 @@ public:
             Symbol *s = symbols->value;
             if (const Name *name = s->name()) {
                 if (const Identifier *id = name->identifier()) {
-                    const QString symbolName = QString::fromLatin1(id->chars(), id->size());
+                    const QString symbolName = QString::fromUtf8(id->chars(), id->size());
                     if (symbolName == m_variableString) {
                         symbol = s;
                         break;
@@ -2911,7 +2911,7 @@ public:
         QTC_ASSERT(className, return);
         const Identifier *classId = className->identifier();
         QTC_ASSERT(classId, return);
-        QString classString = QString::fromLatin1(classId->chars(), classId->size());
+        QString classString = QString::fromUtf8(classId->chars(), classId->size());
 
         bool wasHeader = true;
         QString declFileName = currentFile->fileName();

src/plugins/cppeditor/followsymbol_switchmethoddecldef_test.cpp

@@ -499,6 +499,11 @@ void CppEditorPlugin::test_SwitchMethodDeclarationDefinition_data()
         "    void foo();\n"
         "};\n"
     ) << _();
+
+    QTest::newRow("unicodeIdentifier") << _(
+        "class Foo { void $\u00FC\u4E8C\U00010302(); };\n"
+        "void Foo::@\u00FC\u4E8C\U00010302() {}\n"
+    ) << _();
 }
 
 void CppEditorPlugin::test_SwitchMethodDeclarationDefinition()

src/plugins/cpptools/cppcompletionassist.cpp

@@ -530,7 +530,7 @@ protected:
     void visit(const TemplateNameId *name)
     {
         _item = newCompletionItem(name);
-        _item->setText(QLatin1String(name->identifier()->chars()));
+        _item->setText(QString::fromUtf8(name->identifier()->chars(), name->identifier()->size()));
     }
 
     void visit(const DestructorNameId *name)
@@ -1682,9 +1682,9 @@ bool CppCompletionAssistProcessor::completeQtMethod(const QList<CPlusPlus::Looku
                     signature += QLatin1Char(')');
 
                     const QByteArray normalized =
-                            QMetaObject::normalizedSignature(signature.toLatin1());
+                            QMetaObject::normalizedSignature(signature.toUtf8());
 
-                    signature = QString::fromLatin1(normalized, normalized.size());
+                    signature = QString::fromUtf8(normalized, normalized.size());
 
                     if (!signatures.contains(signature)) {
                         BasicProposalItem *ci = toCompletionItem(fun);
@@ -1884,7 +1884,7 @@ bool CppCompletionAssistProcessor::completeConstructorOrFunction(const QList<CPl
             QString possibleDecl = bs.mid(lineStartToken).trimmed().append(QLatin1String("();"));
 
             Document::Ptr doc = Document::create(QLatin1String("<completion>"));
-            doc->setUtf8Source(possibleDecl.toLatin1());
+            doc->setUtf8Source(possibleDecl.toUtf8());
             if (doc->parse(Document::ParseDeclaration)) {
                 doc->check();
                 if (SimpleDeclarationAST *sd = doc->translationUnit()->ast()->asSimpleDeclaration()) {

src/plugins/cpptools/cppfindreferences.cpp

@@ -95,7 +95,7 @@ static QByteArray typeId(Symbol *symbol)
     } else if (symbol->asDeclaration()) {
         QByteArray temp("d,");
         Overview pretty;
-        temp.append(pretty.prettyType(symbol->type()).toLatin1());
+        temp.append(pretty.prettyType(symbol->type()).toUtf8());
         return temp;
     } else if (symbol->asArgument()) {
         return QByteArray("a");

src/plugins/cpptools/cpplocalsymbols.cpp

@@ -86,11 +86,11 @@ protected:
                     continue;
                 if (!member->isGenerated() && (member->isDeclaration() || member->isArgument())) {
                     if (member->name() && member->name()->isNameId()) {
-                        const Identifier *id = member->identifier();
+                        const Token token = tokenAt(member->sourceLocation());
                         unsigned line, column;
-                        getTokenStartPosition(member->sourceLocation(), &line, &column);
+                        getPosition(token.utf16charsBegin(), &line, &column);
                         localUses[member].append(
-                                    HighlightingResult(line, column, id->size(),
+                                    HighlightingResult(line, column, token.utf16chars(),
                                                        CppHighlightingSupport::LocalUse));
                     }
                 }
@@ -101,7 +101,8 @@ protected:
     bool checkLocalUse(NameAST *nameAst, unsigned firstToken)
     {
         if (SimpleNameAST *simpleName = nameAst->asSimpleName()) {
-            if (tokenAt(simpleName->identifier_token).generated())
+            const Token token = tokenAt(simpleName->identifier_token);
+            if (token.generated())
                 return false;
             const Identifier *id = identifier(simpleName->identifier_token);
             for (int i = _scopeStack.size() - 1; i != -1; --i) {
@@ -113,7 +114,7 @@ protected:
                         unsigned line, column;
                         getTokenStartPosition(simpleName->identifier_token, &line, &column);
                         localUses[member].append(
-                                    HighlightingResult(line, column, id->size(),
+                                    HighlightingResult(line, column, token.utf16chars(),
                                                        CppHighlightingSupport::LocalUse));
                         return false;
                     }

src/plugins/cpptools/cppmodelmanager_test.cpp

@@ -784,7 +784,7 @@ QString nameOfFirstDeclaration(const Document::Ptr &doc)
             if (CPlusPlus::Declaration *decl = s->asDeclaration()) {
                 if (const CPlusPlus::Name *name = decl->name()) {
                     if (const CPlusPlus::Identifier *identifier = name->identifier())
-                        return QString::fromLatin1(identifier->chars(), identifier->size());
+                        return QString::fromUtf8(identifier->chars(), identifier->size());
                 }
             }
         }

src/plugins/cpptools/cpppointerdeclarationformatter.cpp

@@ -429,8 +429,8 @@ void PointerDeclarationFormatter::checkAndRewrite(DeclaratorAST *declarator,
             "No pointer or references in rewritten declaration");
 
     if (DEBUG_OUTPUT) {
-        qDebug("==> Rewritten: \"%s\" --> \"%s\"", originalDeclaration.toLatin1().constData(),
-               rewrittenDeclaration.toLatin1().constData());
+        qDebug("==> Rewritten: \"%s\" --> \"%s\"", originalDeclaration.toUtf8().constData(),
+               rewrittenDeclaration.toUtf8().constData());
     }
 
     // Creating the replacement in the changeset may fail due to operations

src/plugins/cpptools/cpppointerdeclarationformatter_test.cpp

@@ -89,7 +89,7 @@ public:
         // Write source to temprorary file
         const QString filePath = QDir::tempPath() + QLatin1String("/file.h");
         Document::Ptr document = Document::create(filePath);
-        QVERIFY(writeFile(document->fileName(), sourceWithoutCursorMarker.toLatin1()));
+        QVERIFY(writeFile(document->fileName(), sourceWithoutCursorMarker.toUtf8()));
 
         // Preprocess source
         Environment env;
@@ -145,7 +145,7 @@ void CppToolsPlugin::test_format_pointerdeclaration_in_simpledeclarations()
     QFETCH(QString, source);
     QFETCH(QString, reformattedSource);
 
-    PointerDeclarationFormatterTestCase(source.toLatin1(),
+    PointerDeclarationFormatterTestCase(source.toUtf8(),
                                         reformattedSource,
                                         Document::ParseDeclaration,
                                         PointerDeclarationFormatter::RespectCursor);
@@ -369,7 +369,7 @@ void CppToolsPlugin::test_format_pointerdeclaration_in_controlflowstatements()
     QFETCH(QString, source);
     QFETCH(QString, reformattedSource);
 
-    PointerDeclarationFormatterTestCase(source.toLatin1(),
+    PointerDeclarationFormatterTestCase(source.toUtf8(),
                                         reformattedSource,
                                         Document::ParseStatement,
                                         PointerDeclarationFormatter::RespectCursor);
@@ -444,7 +444,7 @@ void CppToolsPlugin::test_format_pointerdeclaration_multiple_declarators()
     QFETCH(QString, source);
     QFETCH(QString, reformattedSource);
 
-    PointerDeclarationFormatterTestCase(source.toLatin1(),
+    PointerDeclarationFormatterTestCase(source.toUtf8(),
                                         reformattedSource,
                                         Document::ParseDeclaration,
                                         PointerDeclarationFormatter::RespectCursor);
@@ -499,7 +499,7 @@ void CppToolsPlugin::test_format_pointerdeclaration_multiple_matches()
     QFETCH(QString, source);
     QFETCH(QString, reformattedSource);
 
-    PointerDeclarationFormatterTestCase(source.toLatin1(),
+    PointerDeclarationFormatterTestCase(source.toUtf8(),
                                         reformattedSource,
                                         Document::ParseTranlationUnit,
                                         PointerDeclarationFormatter::IgnoreCursor);
@@ -582,7 +582,7 @@ void CppToolsPlugin::test_format_pointerdeclaration_macros()
     QFETCH(QString, source);
     QFETCH(QString, reformattedSource);
 
-    PointerDeclarationFormatterTestCase(source.toLatin1(),
+    PointerDeclarationFormatterTestCase(source.toUtf8(),
                                         reformattedSource,
                                         Document::ParseTranlationUnit,
                                         PointerDeclarationFormatter::RespectCursor);

tests/auto/cplusplus/checksymbols/tst_checksymbols.cpp

@@ -120,7 +120,8 @@ public:
         // Preprocess source
         Environment env;
         Preprocessor preprocess(0, &env);
-        const QByteArray preprocessedSource = preprocess.run(filePath, QLatin1String(source));
+        const QByteArray preprocessedSource = preprocess.run(filePath, source);
+
         document->setUtf8Source(preprocessedSource);
         QVERIFY(document->parse(parseMode));
         document->check();
@@ -207,6 +208,17 @@ private slots:
     void test_completion_enum_inside_block_inside_function_QTCREATORBUG5456();
     void test_completion_enum_inside_function_QTCREATORBUG5456();
     void test_completion_using_inside_different_namespace_QTCREATORBUG7978();
+
+    //
+    // The following "non-latin1" code points are used in the next tests:
+    //
+    //   U+00FC  - 2 code units in UTF8, 1 in UTF16 - LATIN SMALL LETTER U WITH DIAERESIS
+    //   U+4E8C  - 3 code units in UTF8, 1 in UTF16 - CJK UNIFIED IDEOGRAPH-4E8C
+    //   U+10302 - 4 code units in UTF8, 2 in UTF16 - OLD ITALIC LETTER KE
+    //
+
+    void test_checksymbols_unicodeIdentifier1();
+    void test_checksymbols_unicodeIdentifier2();
 };
 
 void tst_CheckSymbols::test_checksymbols_TypeUse()
@@ -1913,5 +1925,55 @@ void tst_CheckSymbols::test_completion_using_inside_different_namespace_QTCREATO
     TestData::check(source, expectedUses);
 }
 
+void tst_CheckSymbols::test_checksymbols_unicodeIdentifier1()
+{
+    const QByteArray source =
+            "class My\u00FC\u4E8C\U00010302Type { int \u00FC\u4E8C\U00010302Member; };\n"
+            "void f(My\u00FC\u4E8C\U00010302Type var\u00FC\u4E8C\U00010302)\n"
+            "{ var\u00FC\u4E8C\U00010302.\u00FC\u4E8C\U00010302Member = 0; }\n";
+            ;
+
+    const QList<Use> expectedUses = QList<Use>()
+            << Use(1, 7, 10, CppHighlightingSupport::TypeUse)
+            << Use(1, 24, 10, CppHighlightingSupport::FieldUse)
+            << Use(2, 6, 1, CppHighlightingSupport::FunctionUse)
+            << Use(2, 8, 10, CppHighlightingSupport::TypeUse)
+            << Use(2, 19, 7, CppHighlightingSupport::LocalUse)
+            << Use(3, 3, 7, CppHighlightingSupport::LocalUse)
+            << Use(3, 11, 10, CppHighlightingSupport::FieldUse)
+            ;
+
+    TestData::check(source, expectedUses);
+}
+
+void tst_CheckSymbols::test_checksymbols_unicodeIdentifier2()
+{
+    const QByteArray source =
+            "class v\u00FC\u4E8C\U00010302\n"
+            "{\n"
+            "public:\n"
+            "    v\u00FC\u4E8C\U00010302();\n"
+            "    ~v\u00FC\u4E8C\U00010302();\n"
+            "};\n"
+            "\n"
+            "v\u00FC\u4E8C\U00010302::v\u00FC\u4E8C\U00010302() {}\n"
+            "v\u00FC\u4E8C\U00010302::~v\u00FC\u4E8C\U00010302() {}\n"
+            ;
+
+    const QList<Use> expectedUses = QList<Use>()
+            << Use(1, 7, 5, CppHighlightingSupport::TypeUse)
+            << Use(4, 5, 5, CppHighlightingSupport::TypeUse)
+            << Use(5, 6, 5, CppHighlightingSupport::TypeUse)
+            << Use(5, 6, 5, CppHighlightingSupport::TypeUse)
+            << Use(8, 1, 5, CppHighlightingSupport::TypeUse)
+            << Use(8, 8, 5, CppHighlightingSupport::FunctionUse)
+            << Use(9, 1, 5, CppHighlightingSupport::TypeUse)
+            << Use(9, 1, 5, CppHighlightingSupport::TypeUse)
+            << Use(9, 9, 5, CppHighlightingSupport::TypeUse)
+            ;
+
+    TestData::check(source, expectedUses);
+}
+
 QTEST_APPLESS_MAIN(tst_CheckSymbols)
 #include "tst_checksymbols.moc"

tests/auto/cplusplus/findusages/tst_findusages.cpp

@@ -111,6 +111,8 @@ private Q_SLOTS:
     void templatedFunction_QTCREATORBUG9749();
 
     void usingInDifferentNamespace_QTCREATORBUG7978();
+
+    void unicodeIdentifier();
 };
 
 void tst_FindUsages::dump(const QList<Usage> &usages) const
@@ -951,5 +953,42 @@ void tst_FindUsages::usingInDifferentNamespace_QTCREATORBUG7978()
     QCOMPARE(findUsages.usages().size(), 3);
 }
 
+void tst_FindUsages::unicodeIdentifier()
+{
+    //
+    // The following "non-latin1" code points are used:
+    //
+    //   U+00FC  - 2 code units in UTF8, 1 in UTF16 - LATIN SMALL LETTER U WITH DIAERESIS
+    //   U+4E8C  - 3 code units in UTF8, 1 in UTF16 - CJK UNIFIED IDEOGRAPH-4E8C
+    //   U+10302 - 4 code units in UTF8, 2 in UTF16 - OLD ITALIC LETTER KE
+    //
+
+    const QByteArray src = "\n"
+            "int var\u00FC\u4E8C\U00010302;\n"
+            "void f() { var\u00FC\u4E8C\U00010302 = 1; }\n";
+            ;
+
+    Document::Ptr doc = Document::create("u");
+    doc->setUtf8Source(src);
+    doc->parse();
+    doc->check();
+
+    QVERIFY(doc->diagnosticMessages().isEmpty());
+    QCOMPARE(doc->globalSymbolCount(), 2U);
+
+    Snapshot snapshot;
+    snapshot.insert(doc);
+
+    Declaration *declaration = doc->globalSymbolAt(0)->asDeclaration();
+    QVERIFY(declaration);
+
+    FindUsages findUsages(src, doc, snapshot);
+    findUsages(declaration);
+    const QList<Usage> usages = findUsages.usages();
+    QCOMPARE(usages.size(), 2);
+    QCOMPARE(usages.at(0).len, 7);
+    QCOMPARE(usages.at(1).len, 7);
+}
+
 QTEST_APPLESS_MAIN(tst_FindUsages)
 #include "tst_findusages.moc"