Clang: Add symbol indexer

Change-Id: I8ff879631ae022bc870431628be002903360369c
Reviewed-by: Tim Jenssen <tim.jenssen@qt.io>

src/tools/clangrefactoringbackend/source/clangrefactoringbackend-source.pri

@@ -4,12 +4,15 @@ HEADERS += \
     $$PWD/clangrefactoringbackend_global.h \
     $$PWD/sourcerangefilter.h \
     $$PWD/symbolscollector.h \
+    $$PWD/symbolindexer.h \
     $$PWD/symbolentry.h \
     $$PWD/collectsymbolsconsumer.h \
     $$PWD/collectsymbolsaction.h \
     $$PWD/collectmacrossourcefilecallbacks.h \
     $$PWD/collectsymbolsastvisitor.h \
-    $$PWD/sourcelocationentry.h
+    $$PWD/sourcelocationentry.h \
+    $$PWD/symbolscollectorinterface.h \
+    $$PWD/symbolstorageinterface.h
 
 !isEmpty(LIBTOOLING_LIBS) {
 SOURCES += \
@@ -45,6 +48,7 @@ HEADERS += \
 SOURCES += \
     $$PWD/sourcerangefilter.cpp \
     $$PWD/symbolscollector.cpp \
+    $$PWD/symbolindexer.cpp \
     $$PWD/collectsymbolsaction.cpp \
     $$PWD/collectmacrossourcefilecallbacks.cpp \
     $$PWD/symbolentry.cpp \

src/tools/clangrefactoringbackend/source/clangtool.cpp

@@ -43,9 +43,9 @@ String toNativePath(String &&path)
 }
 
 void ClangTool::addFile(std::string &&directory,
-                           std::string &&fileName,
-                           std::string &&content,
-                           std::vector<std::string> &&commandLine)
+                        std::string &&fileName,
+                        std::string &&content,
+                        std::vector<std::string> &&commandLine)
 {
     m_fileContents.emplace_back(toNativePath(std::move(directory)),
                               std::move(fileName),

src/tools/clangrefactoringbackend/source/refactoringcompilationdatabase.h

@@ -54,8 +54,8 @@ public:
     std::vector<clang::tooling::CompileCommand> getAllCompileCommands() const override;
 
     void addFile(const std::string &directory,
-                            const std::string &fileName,
-                            const std::vector<std::string> &commandLine);
+                 const std::string &fileName,
+                 const std::vector<std::string> &commandLine);
 
 private:
     std::vector<clang::tooling::CompileCommand> compileCommands;

src/tools/clangrefactoringbackend/source/sourcelocationentry.h

@@ -70,6 +70,15 @@ public:
     uint line =  0;
     uint column = 0;
     SymbolType symbolType;
+
+    friend bool operator==(const SourceLocationEntry &first, const SourceLocationEntry &second)
+    {
+        return first.symbolId == second.symbolId
+            && first.fileId == second.fileId
+            && first.line == second.line
+            && first.column == second.column
+            && first.symbolType == second.symbolType;
+    }
 };
 
 using SourceLocationEntries = std::vector<SourceLocationEntry>;

src/tools/clangrefactoringbackend/source/symbolentry.h

@@ -45,8 +45,19 @@ public:
           symbolName(name.data(), name.size())
     {}
 
+    SymbolEntry(Utils::PathString &&usr,
+                Utils::SmallString &&symbolName)
+        : usr(std::move(usr)),
+          symbolName(std::move(symbolName))
+    {}
+
     Utils::PathString usr;
     Utils::SmallString symbolName;
+
+    friend bool operator==(const SymbolEntry &first, const SymbolEntry &second)
+    {
+        return first.usr == second.usr && first.symbolName == second.symbolName;
+    }
 };
 
 using SymbolEntries = std::unordered_map<uint, SymbolEntry>;

src/tools/clangrefactoringbackend/source/symbolindexer.cpp

+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "symbolindexer.h"
+
+namespace ClangBackEnd {
+
+SymbolIndexer::SymbolIndexer(SymbolsCollectorInterface &symbolsCollector, SymbolStorageInterface &symbolStorage)
+    : m_symbolsCollector(symbolsCollector),
+      m_symbolStorage(symbolStorage)
+{
+}
+
+void SymbolIndexer::updateProjectParts(V2::ProjectPartContainers &&projectParts)
+{
+    for (const V2::ProjectPartContainer &projectPart : projectParts)
+        m_symbolsCollector.addFiles(projectPart.sourcePaths(), projectPart.arguments());
+
+    m_symbolsCollector.collectSymbols();
+
+    m_symbolStorage.addSymbolsAndSourceLocations(m_symbolsCollector.symbols(),
+                                                 m_symbolsCollector.sourceLocations());
+}
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/symbolindexer.h

+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "symbolscollectorinterface.h"
+#include "symbolstorageinterface.h"
+
+#include <projectpartcontainerv2.h>
+
+namespace ClangBackEnd {
+
+class SymbolIndexer
+{
+public:
+    SymbolIndexer(SymbolsCollectorInterface &symbolsCollector,
+                  SymbolStorageInterface &symbolStorage);
+
+    void updateProjectParts(V2::ProjectPartContainers &&projectParts);
+
+private:
+    SymbolsCollectorInterface &m_symbolsCollector;
+    SymbolStorageInterface &m_symbolStorage;
+};
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/symbolscollector.cpp

@@ -32,6 +32,11 @@ SymbolsCollector::SymbolsCollector(FilePathCache<> &filePathCache)
 {
 }
 
+void SymbolsCollector::addFiles(const Utils::PathStringVector &filePaths, const Utils::SmallStringVector &arguments)
+{
+    ClangTool::addFiles(filePaths, arguments);
+}
+
 void SymbolsCollector::collectSymbols()
 {
     auto tool = createTool();

src/tools/clangrefactoringbackend/source/symbolscollector.h

@@ -28,20 +28,24 @@
 #include "clangtool.h"
 #include "collectmacrossourcefilecallbacks.h"
 #include "collectsymbolsaction.h"
+#include "symbolscollectorinterface.h"
 #include "symbolentry.h"
 #include "stringcache.h"
 
 namespace ClangBackEnd {
 
-class SymbolsCollector: public ClangTool
+class SymbolsCollector : public ClangTool, public SymbolsCollectorInterface
 {
 public:
     SymbolsCollector(FilePathCache<> &filePathCache);
 
-    void collectSymbols();
+    void addFiles(const Utils::PathStringVector &filePaths,
+                  const Utils::SmallStringVector &arguments) override;
 
-    const SymbolEntries &symbols() const;
-    const SourceLocationEntries &sourceLocations() const;
+    void collectSymbols() override;
+
+    const SymbolEntries &symbols() const override;
+    const SourceLocationEntries &sourceLocations() const override;
 
 private:
     CollectSymbolsAction m_collectSymbolsAction;

src/tools/clangrefactoringbackend/source/symbolscollectorinterface.h

+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "symbolentry.h"
+#include "sourcelocationentry.h"
+
+#include <utils/smallstringvector.h>
+
+#include <string>
+#include <vector>
+
+namespace ClangBackEnd {
+
+class SymbolsCollectorInterface
+{
+public:
+    virtual void addFiles(const Utils::PathStringVector &filePaths,
+                          const Utils::SmallStringVector &arguments) = 0;
+
+    virtual void collectSymbols() = 0;
+
+    virtual const SymbolEntries &symbols() const = 0;
+    virtual const SourceLocationEntries &sourceLocations() const = 0;
+};
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/symbolstorageinterface.h

+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "sourcelocationentry.h"
+#include "symbolentry.h"
+
+namespace ClangBackEnd {
+
+class SymbolStorageInterface
+{
+public:
+    virtual void addSymbolsAndSourceLocations(const SymbolEntries &symbolEentries,
+                                              const SourceLocationEntries &sourceLocations) = 0;
+};
+
+} // namespace ClangBackEnd

tests/unit/unittest/mocksymbolscollector.h

+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "googletest.h"
+
+#include <symbolscollectorinterface.h>
+
+class MockSymbolsCollector : public ClangBackEnd::SymbolsCollectorInterface
+{
+public:
+    MOCK_METHOD0(collectSymbols,
+                 void());
+
+    MOCK_METHOD2(addFiles,
+                 void(const Utils::PathStringVector &filePaths,
+                      const Utils::SmallStringVector &arguments));
+
+    MOCK_CONST_METHOD0(symbols,
+                       const ClangBackEnd::SymbolEntries &());
+
+    MOCK_CONST_METHOD0(sourceLocations,
+                       const ClangBackEnd::SourceLocationEntries &());
+};

tests/unit/unittest/mocksymbolstorage.h

+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "googletest.h"
+
+#include <symbolstorageinterface.h>
+
+class MockSymbolStorage : public ClangBackEnd::SymbolStorageInterface
+{
+public:
+   MOCK_METHOD2(addSymbolsAndSourceLocations,
+                void(const ClangBackEnd::SymbolEntries &symbolEentries,
+                     const ClangBackEnd::SourceLocationEntries &sourceLocations));
+};

tests/unit/unittest/symbolindexer-test.cpp

+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "googletest.h"
+#include "mocksymbolscollector.h"
+#include "mocksymbolstorage.h"
+
+#include <symbolindexer.h>
+#include <updatepchprojectpartsmessage.h>
+#include <projectpartcontainerv2.h>
+
+namespace {
+
+using testing::_;
+using testing::Contains;
+using testing::Field;
+using testing::IsEmpty;
+using testing::NiceMock;
+using testing::Property;
+using testing::Return;
+using testing::ReturnRef;
+using testing::Sequence;
+
+using Utils::PathString;
+using ClangBackEnd::V2::ProjectPartContainer;
+using ClangBackEnd::V2::ProjectPartContainers;
+using ClangBackEnd::SymbolEntries;
+using ClangBackEnd::SymbolEntry;
+using ClangBackEnd::SourceLocationEntries;
+using ClangBackEnd::SourceLocationEntry;
+using ClangBackEnd::SymbolType;
+
+class SymbolIndexer : public testing::Test
+{
+protected:
+    void SetUp();
+
+protected:
+    PathString main1Path = TESTDATA_DIR "/includecollector_main3.cpp";
+    PathString main2Path = TESTDATA_DIR "/includecollector_main2.cpp";
+    PathString header1Path = TESTDATA_DIR "/includecollector_header1.h";
+    PathString header2Path = TESTDATA_DIR "/includecollector_header2.h";
+    PathString generatedFileName = "includecollector_generated_file.h";
+    PathString generatedFilePath = TESTDATA_DIR "/includecollector_generated_file.h";
+    ProjectPartContainer projectPart1{"project1",
+                                      {"-I", TESTDATA_DIR, "-Wno-pragma-once-outside-header"},
+                                      {header1Path.clone()},
+                                      {main1Path.clone()}};
+    ProjectPartContainer projectPart2{"project2",
+                                      {"-I", TESTDATA_DIR, "-x", "c++-header", "-Wno-pragma-once-outside-header"},
+                                      {header2Path.clone()},
+                                      {main2Path.clone()}};
+    SymbolEntries symbolEntries{{1, {"function", "function"}}};
+    SourceLocationEntries sourceLocations{{1, 1, {42, 23}, SymbolType::Declaration}};
+    NiceMock<MockSymbolsCollector> mockCollector;
+    NiceMock<MockSymbolStorage> mockStorage;
+    ClangBackEnd::SymbolIndexer indexer{mockCollector, mockStorage};
+};
+
+TEST_F(SymbolIndexer, UpdateProjectPartsCallsAddFilesInCollector)
+{
+    EXPECT_CALL(mockCollector, addFiles(projectPart1.sourcePaths(), projectPart1.arguments()));
+
+    indexer.updateProjectParts({projectPart1});
+}
+
+TEST_F(SymbolIndexer, UpdateProjectPartsCallsAddFilesInCollectorForEveryProjectPart)
+{
+    EXPECT_CALL(mockCollector, addFiles(_, _))
+            .Times(2);
+
+    indexer.updateProjectParts({projectPart1, projectPart2});
+}
+
+TEST_F(SymbolIndexer, UpdateProjectPartsDoesNotCallAddFilesInCollectorForEmptyEveryProjectParts)
+{
+    EXPECT_CALL(mockCollector, addFiles(_, _))
+            .Times(0);
+
+    indexer.updateProjectParts({});
+}
+
+TEST_F(SymbolIndexer, UpdateProjectPartsCallscollectSymbolsInCollector)
+{
+    EXPECT_CALL(mockCollector, collectSymbols());
+
+    indexer.updateProjectParts({projectPart1});
+}
+
+TEST_F(SymbolIndexer, UpdateProjectPartsCallsSymbolsInCollector)
+{
+    EXPECT_CALL(mockCollector, symbols());
+
+    indexer.updateProjectParts({projectPart1});
+}
+
+TEST_F(SymbolIndexer, UpdateProjectPartsCallsSourceLocationsInCollector)
+{
+    EXPECT_CALL(mockCollector, sourceLocations());
+
+    indexer.updateProjectParts({projectPart1});
+}
+
+TEST_F(SymbolIndexer, UpdateProjectPartsCallsAddSymbolsAndSourceLocationsInStorage)
+{
+    EXPECT_CALL(mockStorage, addSymbolsAndSourceLocations(symbolEntries, sourceLocations));
+
+    indexer.updateProjectParts({projectPart1});
+}
+
+TEST_F(SymbolIndexer, UpdateProjectPartsCallsInOrder)
+{
+    Sequence sequence;
+
+    EXPECT_CALL(mockCollector, addFiles(_, _));
+    EXPECT_CALL(mockCollector, collectSymbols());
+    EXPECT_CALL(mockCollector, symbols());
+    EXPECT_CALL(mockCollector, sourceLocations());
+    EXPECT_CALL(mockStorage, addSymbolsAndSourceLocations(symbolEntries, sourceLocations));
+
+    indexer.updateProjectParts({projectPart1});
+}
+
+void SymbolIndexer::SetUp()
+{
+    ON_CALL(mockCollector, symbols()).WillByDefault(ReturnRef(symbolEntries));
+    ON_CALL(mockCollector, sourceLocations()).WillByDefault(ReturnRef(sourceLocations));
+}
+
+}
+

tests/unit/unittest/unittest.pro

@@ -66,6 +66,7 @@ SOURCES += \
     smallstring-test.cpp \
     sourcerangefilter-test.cpp \
     spydummy.cpp \
+    symbolindexer-test.cpp \
     stringcache-test.cpp \
     unittests-main.cpp \
     utf8-test.cpp
@@ -186,6 +187,8 @@ HEADERS += \
     sourcerangecontainer-matcher.h \
     spydummy.h \
     testenvironment.h \
+    mocksymbolscollector.h \
+    mocksymbolstorage.h
 
 !isEmpty(LIBCLANG_LIBS) {
 HEADERS += \