Clang: Add Symbol Indexing

It is a first step and now a database is generated if you start QtCreator.
Some code is now shared with the PchManager which can be improved in the
future.

Change-Id: Ic267fe7960f6c455d91832859a673ce98f269aa2
Reviewed-by: Tim Jenssen <tim.jenssen@qt.io>

src/libs/clangsupport/clangrefactoringservermessages.h

@@ -30,3 +30,5 @@
 #include "requestsourcelocationforrenamingmessage.h"
 #include "requestsourcerangesanddiagnosticsforquerymessage.h"
 #include "requestsourcerangesforquerymessage.h"
+#include "updatepchprojectpartsmessage.h"
+#include "removepchprojectpartsmessage.h"

src/libs/clangsupport/clangsupport-lib.pri

@@ -174,6 +174,7 @@ HEADERS += \
     $$PWD/ipcclientprovider.h \
     $$PWD/requestsourcerangesforquerymessage.h \
     $$PWD/stringcachefwd.h \
-    $$PWD/stringcachealgorithms.h
+    $$PWD/stringcachealgorithms.h \
+    $$PWD/projectmanagementserverinterface.h
 
 contains(QT_CONFIG, reduce_exports):CONFIG += hide_symbols

src/libs/clangsupport/pchmanagerserverinterface.h

@@ -25,7 +25,7 @@
 
 #pragma once
 
-#include "ipcserverinterface.h"
+#include "projectmanagementserverinterface.h"
 
 #include <memory>
 
@@ -35,15 +35,12 @@ class PchManagerClientInterface;
 class RemovePchProjectPartsMessage;
 class UpdatePchProjectPartsMessage;
 
-
-class CMBIPC_EXPORT PchManagerServerInterface : public IpcServerInterface
+class CMBIPC_EXPORT PchManagerServerInterface : public ProjectManagementServerInterface
 {
 public:
     void dispatch(const MessageEnvelop &messageEnvelop) override;
 
     virtual void end() = 0;
-    virtual void updatePchProjectParts(UpdatePchProjectPartsMessage &&message) = 0;
-    virtual void removePchProjectParts(RemovePchProjectPartsMessage &&message) = 0;
 };
 
 } // namespace ClangBackEnd

src/libs/clangsupport/projectmanagementserverinterface.h

+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "ipcserverinterface.h"
+
+namespace ClangBackEnd {
+
+class RemovePchProjectPartsMessage;
+class UpdatePchProjectPartsMessage;
+
+class CMBIPC_EXPORT ProjectManagementServerInterface : public IpcInterface
+{
+public:
+    virtual void updatePchProjectParts(UpdatePchProjectPartsMessage &&message) = 0;
+    virtual void removePchProjectParts(RemovePchProjectPartsMessage &&message) = 0;
+};
+
+} // namespace ClangBackEnd

src/libs/clangsupport/refactoringserverinterface.cpp

@@ -47,6 +47,12 @@ void RefactoringServerInterface::dispatch(const MessageEnvelop &messageEnvelop)
         case MessageType::RequestSourceRangesForQueryMessage:
             requestSourceRangesForQueryMessage(messageEnvelop.message<RequestSourceRangesForQueryMessage>());
             break;
+        case MessageType::UpdatePchProjectPartsMessage:
+            updatePchProjectParts(messageEnvelop.message<UpdatePchProjectPartsMessage>());
+            break;
+        case MessageType::RemovePchProjectPartsMessage:
+            removePchProjectParts(messageEnvelop.message<RemovePchProjectPartsMessage>());
+            break;
         case MessageType::CancelMessage:
             cancel();
             break;

src/libs/clangsupport/refactoringserverinterface.h

@@ -25,7 +25,7 @@
 
 #pragma once
 
-#include "ipcserverinterface.h"
+#include "projectmanagementserverinterface.h"
 
 #include <memory>
 
@@ -36,8 +36,10 @@ class RequestSourceLocationsForRenamingMessage;
 class RequestSourceRangesAndDiagnosticsForQueryMessage;
 class RequestSourceRangesForQueryMessage;
 class CancelMessage;
+class UpdatePchProjectPartsMessage;
+class RemovePchProjectPartsMessage;
 
-class CMBIPC_EXPORT RefactoringServerInterface : public IpcServerInterface
+class CMBIPC_EXPORT RefactoringServerInterface : public ProjectManagementServerInterface
 
 {
 public:

src/libs/clangsupport/refactoringserverproxy.cpp

@@ -62,6 +62,16 @@ void RefactoringServerProxy::requestSourceRangesForQueryMessage(RequestSourceRan
     writeMessageBlock.write(message);
 }
 
+void RefactoringServerProxy::updatePchProjectParts(UpdatePchProjectPartsMessage &&message)
+{
+    writeMessageBlock.write(message);
+}
+
+void RefactoringServerProxy::removePchProjectParts(RemovePchProjectPartsMessage &&message)
+{
+    writeMessageBlock.write(message);
+}
+
 void RefactoringServerProxy::cancel()
 {
     writeMessageBlock.write(CancelMessage());

src/libs/clangsupport/refactoringserverproxy.h

@@ -53,6 +53,8 @@ public:
     void requestSourceLocationsForRenamingMessage(RequestSourceLocationsForRenamingMessage &&message) override;
     void requestSourceRangesAndDiagnosticsForQueryMessage(RequestSourceRangesAndDiagnosticsForQueryMessage &&message) override;
     void requestSourceRangesForQueryMessage(RequestSourceRangesForQueryMessage &&message) override;
+    void updatePchProjectParts(UpdatePchProjectPartsMessage &&message) override;
+    void removePchProjectParts(RemovePchProjectPartsMessage &&message) override;
     void cancel() override;
 
     void readMessages();

src/libs/clangsupport/stringcache.h

@@ -125,12 +125,7 @@ public:
     {
         std::lock_guard<Mutex> lock(m_mutex);
 
-        Found found = find(stringView);
-
-        if (!found.wasFound)
-            return insertString(found.iterator, stringView);
-
-        return found.iterator->id;
+        return ungardedStringId(stringView);
     }
 
     template <typename Container>
@@ -144,7 +139,7 @@ public:
         std::transform(strings.begin(),
                        strings.end(),
                        std::back_inserter(ids),
-                       [&] (const auto &string) { return this->stringId(string); });
+                       [&] (const auto &string) { return this->ungardedStringId(string); });
 
         return ids;
     }
@@ -182,6 +177,16 @@ public:
     }
 
 private:
+    IndexType ungardedStringId(Utils::SmallStringView stringView)
+    {
+        Found found = find(stringView);
+
+        if (!found.wasFound)
+            return insertString(found.iterator, stringView);
+
+        return found.iterator->id;
+    }
+
     Found find(Utils::SmallStringView stringView)
     {
         return findInSorted(m_strings.cbegin(), m_strings.cend(), stringView, compare);

src/libs/sqlite/sqlite-lib.pri

@@ -43,7 +43,8 @@ HEADERS += \
     $$PWD/utf8stringvector.h \
     $$PWD/sqlitedatabase.h \
     $$PWD/sqlitetable.h \
-    $$PWD/sqlitecolumn.h
+    $$PWD/sqlitecolumn.h \
+    $$PWD/sqliteindex.h
 
 DEFINES += SQLITE_THREADSAFE=2 SQLITE_ENABLE_FTS4 SQLITE_ENABLE_FTS3_PARENTHESIS SQLITE_ENABLE_UNLOCK_NOTIFY SQLITE_ENABLE_COLUMN_METADATA
 

src/libs/sqlite/sqlitecolumn.h

@@ -29,6 +29,8 @@
 
 #include <utils/smallstring.h>
 
+#include <functional>
+
 namespace Sqlite {
 
 class SqliteColumn
@@ -108,5 +110,7 @@ private:
 };
 
 using SqliteColumns = std::vector<SqliteColumn>;
+using SqliteColumnConstReference = std::reference_wrapper<const SqliteColumn>;
+using SqliteColumnConstReferences = std::vector<SqliteColumnConstReference>;
 
 } // namespace Sqlite

src/libs/sqlite/sqliteindex.h

+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+
+#include "sqliteglobal.h"
+
+#include "sqliteexception.h"
+
+#include <utils/smallstringvector.h>
+
+namespace Sqlite {
+
+class SqliteIndex
+{
+public:
+    SqliteIndex(Utils::SmallString &&tableName, Utils::SmallStringVector &&columnNames)
+        : m_tableName(std::move(tableName)),
+          m_columnNames(std::move(columnNames))
+    {
+    }
+
+    Utils::SmallString sqlStatement() const
+    {
+        checkTableName();
+        checkColumns();
+
+        return {"CREATE INDEX IF NOT EXISTS index_",
+                m_tableName,
+                "_",
+                m_columnNames.join("_"),
+                " ON ",
+                m_tableName,
+                "(",
+                m_columnNames.join(", "),
+                ")"
+        };
+    }
+
+    void checkTableName() const
+    {
+        if (m_tableName.isEmpty())
+            throw SqliteException("SqliteIndex has not table name!");
+    }
+
+    void checkColumns() const
+    {
+        if (m_columnNames.empty())
+            throw SqliteException("SqliteIndex has no columns!");
+    }
+
+private:
+    Utils::SmallString m_tableName;
+    Utils::SmallStringVector m_columnNames;
+};
+
+using SqliteIndices = std::vector<SqliteIndex>;
+
+} //

src/libs/sqlite/sqlitetable.h

@@ -28,6 +28,7 @@
 #include "createtablesqlstatementbuilder.h"
 #include "sqliteglobal.h"
 #include "sqlitecolumn.h"
+#include "sqliteindex.h"
 #include "sqliteexception.h"
 
 namespace Sqlite {
@@ -37,6 +38,12 @@ class SqliteDatabase;
 class SqliteTable
 {
 public:
+    SqliteTable(std::size_t reserve = 10)
+    {
+        m_sqliteColumns.reserve(reserve);
+        m_sqliteIndices.reserve(reserve);
+    }
+
     void setName(Utils::SmallString &&name)
     {
         m_tableName = std::move(name);
@@ -76,6 +83,13 @@ public:
         return m_sqliteColumns.back();
     }
 
+    SqliteIndex &addIndex(const SqliteColumnConstReferences &columns)
+    {
+        m_sqliteIndices.emplace_back(m_tableName.clone(), sqliteColumnNames(columns));
+
+        return m_sqliteIndices.back();
+    }
+
     const SqliteColumns &columns() const
     {
         return m_sqliteColumns;
@@ -89,22 +103,25 @@ public:
     template <typename Database>
     void initialize(Database &database)
     {
-        try {
-            CreateTableSqlStatementBuilder builder;
+        CreateTableSqlStatementBuilder builder;
 
-            builder.setTableName(m_tableName.clone());
-            builder.setUseWithoutRowId(m_withoutRowId);
-            builder.setUseIfNotExists(m_useIfNotExists);
-            builder.setUseTemporaryTable(m_useTemporaryTable);
-            builder.setColumns(m_sqliteColumns);
+        builder.setTableName(m_tableName.clone());
+        builder.setUseWithoutRowId(m_withoutRowId);
+        builder.setUseIfNotExists(m_useIfNotExists);
+        builder.setUseTemporaryTable(m_useTemporaryTable);
+        builder.setColumns(m_sqliteColumns);
 
-            database.execute(builder.sqlStatement());
+        database.execute(builder.sqlStatement());
 
-            m_isReady = true;
+        initializeIndices(database);
 
-        } catch (const SqliteException &exception) {
-            exception.printWarning();
-        }
+        m_isReady = true;
+    }
+    template <typename Database>
+    void initializeIndices(Database &database)
+    {
+        for (const SqliteIndex &index : m_sqliteIndices)
+            database.execute(index.sqlStatement());
     }
 
     friend bool operator==(const SqliteTable &first, const SqliteTable &second)
@@ -116,9 +133,21 @@ public:
             && first.m_sqliteColumns == second.m_sqliteColumns;
     }
 
+private:
+    Utils::SmallStringVector sqliteColumnNames(const SqliteColumnConstReferences &columns)
+    {
+        Utils::SmallStringVector columnNames;
+
+        for (const SqliteColumn &column : columns)
+            columnNames.push_back(column.name());
+
+        return columnNames;
+    }
+
 private:
     Utils::SmallString m_tableName;
     SqliteColumns m_sqliteColumns;
+    SqliteIndices m_sqliteIndices;
     bool m_withoutRowId = false;
     bool m_useIfNotExists = false;
     bool m_useTemporaryTable = false;

src/libs/sqlite/sqlitetransaction.h

@@ -36,7 +36,7 @@ template <typename Database>
 class SqliteAbstractTransaction
 {
 public:
-    virtual ~SqliteAbstractTransaction()
+    ~SqliteAbstractTransaction()
     {
         if (!m_isAlreadyCommited)
             m_database.execute("ROLLBACK");
@@ -68,7 +68,6 @@ public:
     {
         database.execute("BEGIN");
     }
-
 };
 
 template <typename Database>
@@ -80,7 +79,6 @@ public:
     {
         database.execute("BEGIN IMMEDIATE");
     }
-
 };
 
 template <typename Database>
@@ -92,7 +90,6 @@ public:
     {
         database.execute("BEGIN EXCLUSIVE");
     }
-
 };
 
 } // namespace Sqlite

src/plugins/clangpchmanager/clangpchmanager-source.pri

@@ -11,12 +11,14 @@ HEADERS += \
     $$PWD/pchmanagernotifierinterface.h \
     $$PWD/pchmanagerconnectionclient.h \
     $$PWD/clangpchmanager_global.h \
-    $$PWD/projectupdater.h
+    $$PWD/projectupdater.h \
+    $$PWD/pchmanagerprojectupdater.h
 
 
 SOURCES += \
     $$PWD/pchmanagerclient.cpp \
     $$PWD/pchmanagernotifierinterface.cpp \
     $$PWD/pchmanagerconnectionclient.cpp \
-    $$PWD/projectupdater.cpp
+    $$PWD/projectupdater.cpp \
+    $$PWD/pchmanagerprojectupdater.cpp
 

src/plugins/clangpchmanager/clangpchmanagerplugin.cpp

@@ -52,7 +52,7 @@ class ClangPchManagerPluginData
 public:
     PchManagerClient pchManagerClient;
     PchManagerConnectionClient connectionClient{&pchManagerClient};
-    QtCreatorProjectUpdater projectUpdate{connectionClient.serverProxy(), pchManagerClient};
+    QtCreatorProjectUpdater<PchManagerProjectUpdater> projectUpdate{connectionClient.serverProxy(), pchManagerClient};
 };
 
 std::unique_ptr<ClangPchManagerPluginData> ClangPchManagerPlugin::d;

src/plugins/clangpchmanager/pchmanagerprojectupdater.cpp

+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "pchmanagerprojectupdater.h"
+
+#include "pchmanagerclient.h"
+
+namespace ClangPchManager {
+
+PchManagerProjectUpdater::PchManagerProjectUpdater(ClangBackEnd::ProjectManagementServerInterface &server,
+                                                   PchManagerClient &client)
+    : ProjectUpdater(server),
+      m_client(client)
+{
+}
+
+void PchManagerProjectUpdater::removeProjectParts(const QStringList &projectPartIds)
+{
+    ProjectUpdater::removeProjectParts(projectPartIds);
+
+    for (const QString &projectPartiId : projectPartIds)
+        m_client.precompiledHeaderRemoved(projectPartiId);
+}
+
+} // namespace ClangPchManager

src/plugins/clangpchmanager/pchmanagerprojectupdater.h

+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "projectupdater.h"
+
+namespace ClangPchManager {
+
+class PchManagerProjectUpdater : public ProjectUpdater
+{
+public:
+    PchManagerProjectUpdater(ClangBackEnd::ProjectManagementServerInterface &server,
+                             PchManagerClient &client);
+
+    void removeProjectParts(const QStringList &projectPartIds);
+
+private:
+    PchManagerClient &m_client;
+};
+
+} // namespace ClangPchManager

src/plugins/clangpchmanager/projectupdater.cpp

@@ -52,10 +52,8 @@ public:
     Utils::PathStringVector sources;
 };
 
-ProjectUpdater::ProjectUpdater(ClangBackEnd::PchManagerServerInterface &server,
-                               PchManagerClient &client)
-    : m_server(server),
-      m_client(client)
+ProjectUpdater::ProjectUpdater(ClangBackEnd::ProjectManagementServerInterface &server)
+    : m_server(server)
 {
 }
 
@@ -75,9 +73,6 @@ void ProjectUpdater::removeProjectParts(const QStringList &projectPartIds)
     ClangBackEnd::RemovePchProjectPartsMessage message{Utils::SmallStringVector(projectPartIds)};
 
     m_server.removePchProjectParts(std::move(message));
-
-    for (const QString &projectPartiId : projectPartIds)
-        m_client.precompiledHeaderRemoved(projectPartiId);
 }
 
 void ProjectUpdater::setExcludedPaths(Utils::PathStringVector &&excludedPaths)

src/plugins/clangpchmanager/projectupdater.h

@@ -35,7 +35,7 @@ class ProjectFile;
 }
 
 namespace ClangBackEnd {
-class PchManagerServerInterface;
+class ProjectManagementServerInterface;
 
 namespace V2 {
 class ProjectPartContainer;
@@ -51,11 +51,10 @@ namespace ClangPchManager {
 class HeaderAndSources;
 class PchManagerClient;
 
-class ProjectUpdater
+class CLANGPCHMANAGER_EXPORT ProjectUpdater
 {
 public:
-    ProjectUpdater(ClangBackEnd::PchManagerServerInterface &server,
-                   PchManagerClient &client);
+    ProjectUpdater(ClangBackEnd::ProjectManagementServerInterface &server);
 
     void updateProjectParts(const std::vector<CppTools::ProjectPart *> &projectParts,
                             ClangBackEnd::V2::FileContainers &&generatedFiles);
@@ -77,8 +76,7 @@ unittest_public:
 
 private:
     Utils::PathStringVector m_excludedPaths;
-    ClangBackEnd::PchManagerServerInterface &m_server;
-    PchManagerClient &m_client;
+    ClangBackEnd::ProjectManagementServerInterface &m_server;
 };
 
 } // namespace ClangPchManager

src/plugins/clangpchmanager/qtcreatorprojectupdater.cpp

@@ -26,26 +26,18 @@
 #include "qtcreatorprojectupdater.h"
 
 #include <cpptools/abstracteditorsupport.h>
-#include <cpptools/cppmodelmanager.h>
 
 #include <projectexplorer/project.h>
 
 namespace ClangPchManager {
 
-static CppTools::CppModelManager *cppModelManager()
-{
-    return CppTools::CppModelManager::instance();
-}
+namespace Internal {
 
-QtCreatorProjectUpdater::QtCreatorProjectUpdater(ClangBackEnd::PchManagerServerInterface &server,
-                                                 PchManagerClient &client)
-    : ProjectUpdater(server, client)
+CppTools::CppModelManager *cppModelManager()
 {
-    connectToCppModelManager();