Clang: Extract highlighting information

Prepare the move of the semantic highlighting to the clang back end. We
have it under tests too so it should be quite easy to make changes or
corrections.

Change-Id: I5706a8a06fde5a9ba2eba3a8ba62782102ac0bd3
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@theqtcompany.com>

src/libs/clangbackendipc/clangbackendipc_global.h

@@ -53,5 +53,28 @@ enum class DiagnosticSeverity // one to one mapping of the clang enum numbers
     Error = 3,
     Fatal = 4
 };
+
+enum class HighlightingType
+{
+    Invalid,
+    Keyword,
+    StringLiteral,
+    NumberLiteral,
+    Comment,
+    Function,
+    VirtualFunction,
+    Type,
+    LocalVariable,
+    Field,
+    GlobalVariable,
+    Enumeration,
+    Operator,
+    Preprocessor,
+    PreprocessorDefinition,
+    PreprocessorExpansion,
+    Label,
+    OutputArgument
+};
+
 }
 #endif // CLANGBACKENDIPC_GLOBAL_H

src/libs/sqlite/utf8string.cpp

@@ -158,6 +158,11 @@ bool Utf8String::endsWith(const Utf8String &text) const
     return byteArray.endsWith(text.byteArray);
 }
 
+bool Utf8String::isNull() const
+{
+    return byteArray.isNull();
+}
+
 bool Utf8String::isEmpty() const
 {
     return byteArray.isEmpty();

src/libs/sqlite/utf8string.h

@@ -86,6 +86,7 @@ public:
     bool startsWith(const char *text) const;
     bool startsWith(char character) const;
     bool endsWith(const Utf8String &text) const;
+    bool isNull() const;
     bool isEmpty() const;
     bool hasContent() const;
 

src/tools/clangbackend/ipcsource/clangbackendclangipc-source.pri

@@ -25,7 +25,13 @@ HEADERS += $$PWD/clangipcserver.h \
     $$PWD/diagnosticsetiterator.h \
     $$PWD/clangfilesystemwatcher.h \
     $$PWD/translationunitalreadyexistsexception.h \
-    $$PWD/commandlinearguments.h
+    $$PWD/commandlinearguments.h \
+    $$PWD/cursor.h \
+    $$PWD/type.h \
+    $$PWD/highlightinginformations.h \
+    $$PWD/highlightinginformation.h \
+    $$PWD/highlightinginformationsiterator.h \
+    $$PWD/skippedsourceranges.h
 
 SOURCES += $$PWD/clangipcserver.cpp \
     $$PWD/codecompleter.cpp \
@@ -51,4 +57,9 @@ SOURCES += $$PWD/clangipcserver.cpp \
     $$PWD/fixit.cpp \
     $$PWD/clangfilesystemwatcher.cpp \
     $$PWD/translationunitalreadyexistsexception.cpp \
-    $$PWD/commandlinearguments.cpp
+    $$PWD/commandlinearguments.cpp \
+    $$PWD/cursor.cpp \
+    $$PWD/type.cpp \
+    $$PWD/highlightinginformations.cpp \
+    $$PWD/highlightinginformation.cpp \
+    $$PWD/skippedsourceranges.cpp

src/tools/clangbackend/ipcsource/clangstring.cpp

@@ -61,6 +61,11 @@ ClangString &ClangString::operator=(ClangString &&other)
     return *this;
 }
 
+const char *ClangString::cString() const
+{
+    return clang_getCString(cxString);
+}
+
 ClangString::ClangString(ClangString &&other)
     : cxString(std::move(other.cxString))
 {
@@ -70,7 +75,7 @@ ClangString::ClangString(ClangString &&other)
 
 ClangString::operator Utf8String() const
 {
-    return Utf8String(clang_getCString(cxString), -1);
+    return Utf8String(cString(), -1);
 }
 
 } // namespace ClangBackEnd

src/tools/clangbackend/ipcsource/clangstring.h

@@ -51,6 +51,8 @@ public:
 
     operator Utf8String() const;
 
+    const char *cString() const;
+
     bool isNull() const;
 
 private:

src/tools/clangbackend/ipcsource/cursor.cpp

+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+
+#include "cursor.h"
+
+#include "clangstring.h"
+#include "sourcelocation.h"
+#include "sourcerange.h"
+
+#include <ostream>
+
+namespace ClangBackEnd {
+
+Cursor::Cursor()
+    : cxCursor(clang_getNullCursor())
+{
+}
+
+Cursor::Cursor(CXCursor cxCursor)
+    : cxCursor(cxCursor)
+{
+}
+
+bool Cursor::isNull() const
+{
+    return clang_Cursor_isNull(cxCursor);
+}
+
+bool Cursor::isValid() const
+{
+    return !clang_isInvalid(kind());
+}
+
+bool Cursor::isTranslationUnit() const
+{
+    return clang_isTranslationUnit(kind());
+}
+
+bool Cursor::isDefinition() const
+{
+    return clang_isCursorDefinition(cxCursor);
+}
+
+bool Cursor::isDynamicCall() const
+{
+    return clang_Cursor_isDynamicCall(cxCursor);
+}
+
+bool Cursor::isVirtualMethod() const
+{
+    return clang_CXXMethod_isVirtual(cxCursor);
+}
+
+bool Cursor::isPureVirtualMethod() const
+{
+    return clang_CXXMethod_isPureVirtual(cxCursor);
+}
+
+bool Cursor::isConstantMethod() const
+{
+    return clang_CXXMethod_isConst(cxCursor);
+}
+
+bool Cursor::isStaticMethod() const
+{
+    return clang_CXXMethod_isStatic(cxCursor);
+}
+
+bool Cursor::isCompoundType() const
+{
+    switch (kind()) {
+        case CXCursor_ClassDecl:
+        case CXCursor_StructDecl:
+        case CXCursor_UnionDecl: return true;
+        default: return false;
+    }
+}
+
+bool Cursor::isDeclaration() const
+{
+    return clang_isDeclaration(kind());
+}
+
+bool Cursor::isLocalVariable() const
+{
+    switch (semanticParent().kind()) {
+        case CXCursor_FunctionDecl:
+        case CXCursor_CXXMethod:
+        case CXCursor_Constructor:
+        case CXCursor_Destructor:
+        case CXCursor_ConversionFunction:
+        case CXCursor_FunctionTemplate:
+        case CXCursor_ObjCInstanceMethodDecl: return true;
+        default:
+            return false;
+    }
+}
+
+bool Cursor::hasFinalFunctionAttribute() const
+{
+    bool hasFinal = false;
+
+    visit([&] (Cursor cursor, Cursor /*parent*/) {
+        if (cursor.kind() == CXCursor_CXXFinalAttr) {
+            hasFinal = true;
+            return CXChildVisit_Break;
+        } else {
+            return CXChildVisit_Recurse;
+        }
+    });
+
+    return hasFinal;
+}
+
+bool Cursor::hasFinalClassAttribute() const
+{
+      bool hasFinal = false;
+
+      visit([&] (Cursor cursor, Cursor /*parent*/) {
+          switch (cursor.kind()) {
+              case CXCursor_CXXFinalAttr:
+                  hasFinal = true;
+                  return CXChildVisit_Break;
+              case CXCursor_CXXMethod:
+                  return CXChildVisit_Break;
+              default:
+                  return CXChildVisit_Recurse;
+          }
+      });
+
+      return hasFinal;
+}
+
+bool Cursor::isUnexposed() const
+{
+    return clang_isUnexposed(kind());
+}
+
+Utf8String Cursor::unifiedSymbolResolution() const
+{
+    return ClangString(clang_getCursorUSR(cxCursor));
+}
+
+Utf8String Cursor::mangling() const
+{
+    return ClangString(clang_Cursor_getMangling(cxCursor));
+}
+
+ClangString Cursor::spelling() const
+{
+    return ClangString(clang_getCursorSpelling(cxCursor));
+}
+
+Utf8String Cursor::displayName() const
+{
+    return ClangString(clang_getCursorDisplayName(cxCursor));
+}
+
+Utf8String Cursor::briefComment() const
+{
+    return ClangString(clang_Cursor_getBriefCommentText(cxCursor));
+}
+
+Utf8String Cursor::rawComment() const
+{
+    return ClangString(clang_Cursor_getRawCommentText(cxCursor));
+}
+
+int Cursor::argumentCount() const
+{
+    return clang_Cursor_getNumArguments(cxCursor);
+}
+
+Type Cursor::type() const
+{
+    return clang_getCursorType(cxCursor);
+}
+
+Type Cursor::nonPointerTupe() const
+{
+    auto typeResult = type();
+
+    if (typeResult.isPointer())
+        typeResult = typeResult.pointeeType();
+
+    return typeResult;
+}
+
+SourceLocation Cursor::sourceLocation() const
+{
+    return clang_getCursorLocation(cxCursor);
+}
+
+SourceRange Cursor::sourceRange() const
+{
+    return clang_getCursorExtent(cxCursor);
+}
+
+SourceRange Cursor::commentRange() const
+{
+    return clang_Cursor_getCommentRange(cxCursor);
+}
+
+Cursor Cursor::definition() const
+{
+    return clang_getCursorDefinition(cxCursor);
+}
+
+Cursor Cursor::canonical() const
+{
+    return clang_getCanonicalCursor(cxCursor);
+}
+
+Cursor Cursor::referenced() const
+{
+    return clang_getCursorReferenced(cxCursor);
+}
+
+Cursor Cursor::semanticParent() const
+{
+    return clang_getCursorSemanticParent(cxCursor);
+}
+
+Cursor Cursor::lexicalParent() const
+{
+    return clang_getCursorLexicalParent(cxCursor);
+}
+
+Cursor Cursor::functionBaseDeclaration() const
+{
+    auto functionBaseCursor = functionBase();
+
+    if (functionBaseCursor.isValid())
+        return functionBaseCursor.nonPointerTupe().canonical().declaration();
+    else
+        return semanticParent().semanticParent();
+}
+
+Cursor Cursor::functionBase() const
+{
+    Cursor functionBaseCursor;
+
+    visit([&] (Cursor cursor, Cursor /*parentCursor*/) {
+        switch (cursor.kind()) {
+            case CXCursor_DeclRefExpr:
+                functionBaseCursor = cursor;                                                       ;
+                return CXChildVisit_Break;
+            default:
+                 return CXChildVisit_Recurse;
+        }
+    });
+
+    return functionBaseCursor;
+}
+
+Cursor Cursor::argument(int index) const
+{
+    return clang_Cursor_getArgument(cxCursor, index);
+}
+namespace {
+void collectOutputArguments(const Cursor &callExpression,
+                            std::vector<Cursor> &outputArguments)
+{
+    auto callExpressionType = callExpression.referenced().type();
+    auto argumentCount = callExpression.argumentCount();
+    outputArguments.reserve(argumentCount);
+
+    for (int argumentIndex = 0; argumentIndex < argumentCount; ++argumentIndex) {
+        auto argument = callExpression.argument(argumentIndex);
+        auto argumentType = callExpressionType.argument(argumentIndex);
+
+        if (!argument.isUnexposed() && argumentType.isOutputParameter())
+            outputArguments.push_back(callExpression.argument(argumentIndex));
+    }
+}
+}
+
+std::vector<Cursor> Cursor::outputArguments() const
+{
+    std::vector<Cursor> outputArguments;
+
+    if (kind() == CXCursor_CallExpr)
+        collectOutputArguments(*this, outputArguments);
+
+    return outputArguments;
+}
+
+CXCursorKind Cursor::kind() const
+{
+    return clang_getCursorKind(cxCursor);
+}
+
+bool operator==(const Cursor &first, const Cursor &second)
+{
+    return clang_equalCursors(first.cxCursor, second.cxCursor);
+}
+
+void PrintTo(CXCursorKind cursorKind, ::std::ostream *os)
+{
+    ClangString cursorKindSpelling(clang_getCursorKindSpelling(cursorKind));
+    *os << cursorKindSpelling.cString();
+}
+
+void PrintTo(const Cursor &cursor, ::std::ostream*os)
+{
+    if (cursor.isValid()) {
+        ClangString cursorKindSpelling(clang_getCursorKindSpelling(cursor.kind()));
+        *os << cursorKindSpelling.cString() << " ";
+
+        auto identifier = cursor.displayName();
+        if (identifier.hasContent()) {
+            *os  << "\""
+                 << identifier.constData()
+                 << "\": ";
+        }
+
+        PrintTo(cursor.sourceLocation(), os);
+    } else {
+        *os << "Invalid cursor!";
+    }
+}
+
+} // namespace ClangBackEnd
+

src/tools/clangbackend/ipcsource/cursor.h

+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+
+#ifndef CLANGBACKEND_CURSOR_H
+#define CLANGBACKEND_CURSOR_H
+
+#include "type.h"
+
+#include <clang-c/Index.h>
+
+#include <iosfwd>
+
+#include <vector>
+
+class Utf8String;
+
+namespace ClangBackEnd {
+
+class SourceLocation;
+class SourceRange;
+class ClangString;
+
+class Cursor
+{
+    friend class Type;
+    friend bool operator==(const Cursor &first, const Cursor &second);
+public:
+    Cursor();
+    Cursor(CXCursor cxCursor);
+
+    bool isNull() const;
+    bool isValid() const;
+
+    bool isTranslationUnit() const;
+    bool isDefinition() const;
+    bool isDynamicCall() const;
+    bool isVirtualMethod() const;
+    bool isPureVirtualMethod() const;
+    bool isConstantMethod() const;
+    bool isStaticMethod() const;
+    bool isCompoundType() const;
+    bool isDeclaration() const;
+    bool isLocalVariable() const;
+    bool hasFinalFunctionAttribute() const;
+    bool hasFinalClassAttribute() const;
+    bool isUnexposed() const;
+
+    Utf8String unifiedSymbolResolution() const;
+    Utf8String mangling() const;
+    ClangString spelling() const;
+    Utf8String displayName() const;
+    Utf8String briefComment() const;
+    Utf8String rawComment() const;
+    int argumentCount() const;
+
+    Type type() const;
+    Type nonPointerTupe() const;
+
+    SourceLocation sourceLocation() const;
+    SourceRange sourceRange() const;
+    SourceRange commentRange() const;
+
+    Cursor definition() const;
+    Cursor canonical() const;
+    Cursor alias() const;
+    Cursor referenced() const;
+    Cursor semanticParent() const;
+    Cursor lexicalParent() const;
+    Cursor functionBaseDeclaration() const;
+    Cursor functionBase() const;
+    Cursor argument(int index) const;
+    std::vector<Cursor> outputArguments() const;
+
+    CXCursorKind kind() const;
+
+    template <class VisitorCallback>
+    void visit(VisitorCallback visitorCallback) const;
+
+private:
+    CXCursor cxCursor;
+};
+
+template <class VisitorCallback>
+void Cursor::visit(VisitorCallback visitorCallback) const
+{
+    auto visitor = [] (CXCursor cursor, CXCursor parent, CXClientData lambda) -> CXChildVisitResult {
+        auto &visitorCallback = *static_cast<VisitorCallback*>(lambda);
+
+        return visitorCallback(cursor, parent);
+    };
+
+    clang_visitChildren(cxCursor, visitor, &visitorCallback);
+}
+
+bool operator==(const Cursor &first, const Cursor &second);
+
+void PrintTo(CXCursorKind cursorKind, ::std::ostream *os);
+void PrintTo(const Cursor &cursor, ::std::ostream* os);
+} // namespace ClangBackEnd
+
+
+#endif // CLANGBACKEND_CURSOR_H

src/tools/clangbackend/ipcsource/highlightinginformation.cpp

+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+
+#include "clangstring.h"
+#include "cursor.h"
+#include "highlightinginformation.h"
+#include "sourcelocation.h"
+#include "sourcerange.h"
+
+#include <cstring>
+#include <ostream>