Clang: Add symbol storage

Extend file path cache to 64 bit integer.

Change-Id: I5627f13d59a3214f389087038482cbcc8d0eb484
Reviewed-by: Tim Jenssen <tim.jenssen@qt.io>

src/libs/clangsupport/stringcache.h

@@ -25,6 +25,8 @@
 
 #pragma once
 
+#include "stringcachefwd.h"
+
 #include <utils/smallstringview.h>
 #include <utils/smallstringfwd.h>
 
@@ -52,11 +54,11 @@ public:
     void unlock() {}
 };
 
-template <typename StringType>
+template <typename StringType, typename IndexType>
 class StringCacheEntry
 {
 public:
-    StringCacheEntry(StringType &&string, uint id)
+    StringCacheEntry(StringType &&string, IndexType id)
         : string(std::move(string)),
           id(id)
     {}
@@ -77,22 +79,27 @@ public:
     }
 
     StringType string;
-    uint id;
+    IndexType id;
 };
 
-template <typename StringType>
-using StringCacheEntries = std::vector<StringCacheEntry<StringType>>;
+template <typename StringType, typename IndexType>
+using StringCacheEntries = std::vector<StringCacheEntry<StringType, IndexType>>;
+
+using FileCacheCacheEntry = StringCacheEntry<Utils::PathString, FilePathIndex>;
+using FileCacheCacheEntries = std::vector<FileCacheCacheEntry>;
 
 template <typename StringType,
-          typename Mutex = NonLockingMutex>
+          typename IndexType,
+          typename Mutex>
 class StringCache
 {
-    using const_iterator = typename StringCacheEntries<StringType>::const_iterator;
+    using CacheEnties = StringCacheEntries<StringType, IndexType>;
+    using const_iterator = typename CacheEnties::const_iterator;
 
     class Found
     {
     public:
-        typename StringCacheEntries<StringType>::const_iterator iterator;
+        typename CacheEnties::const_iterator iterator;
         bool wasFound;
     };
 
@@ -103,14 +110,14 @@ public:
         m_indices.reserve(1024);
     }
 
-    void populate(StringCacheEntries<StringType> &&entries)
+    void populate(CacheEnties &&entries)
     {
         uncheckedPopulate(std::move(entries));
 
         checkEntries();
     }
 
-    void uncheckedPopulate(StringCacheEntries<StringType> &&entries)
+    void uncheckedPopulate(CacheEnties &&entries)
     {
         std::sort(entries.begin(), entries.end());
 
@@ -123,7 +130,7 @@ public:
     }
 
 
-    uint stringId(Utils::SmallStringView stringView)
+    IndexType stringId(Utils::SmallStringView stringView)
     {
         std::lock_guard<Mutex> lock(m_mutex);
 
@@ -136,11 +143,11 @@ public:
     }
 
     template <typename Container>
-    std::vector<uint> stringIds(const Container &strings)
+    std::vector<IndexType> stringIds(const Container &strings)
     {
         std::lock_guard<Mutex> lock(m_mutex);
 
-        std::vector<uint> ids;
+        std::vector<IndexType> ids;
         ids.reserve(strings.size());
 
         std::transform(strings.begin(),
@@ -151,19 +158,19 @@ public:
         return ids;
     }
 
-    std::vector<uint> stringIds(std::initializer_list<StringType> strings)
+    std::vector<IndexType> stringIds(std::initializer_list<StringType> strings)
     {
         return stringIds<std::initializer_list<StringType>>(strings);
     }
 
-    Utils::SmallStringView string(uint id) const
+    Utils::SmallStringView string(IndexType id) const
     {
         std::lock_guard<Mutex> lock(m_mutex);
 
         return m_strings.at(m_indices.at(id)).string;
     }
 
-    std::vector<StringType> strings(const std::vector<uint> &ids) const
+    std::vector<StringType> strings(const std::vector<IndexType> &ids) const
     {
         std::lock_guard<Mutex> lock(m_mutex);
 
@@ -173,7 +180,7 @@ public:
         std::transform(ids.begin(),
                        ids.end(),
                        std::back_inserter(strings),
-                       [&] (uint id) { return m_strings.at(m_indices.at(id)).string; });
+                       [&] (IndexType id) { return m_strings.at(m_indices.at(id)).string; });
 
         return strings;
     }
@@ -191,24 +198,24 @@ private:
         return {range.first, range.first != range.second};
     }
 
-    void incrementLargerOrEqualIndicesByOne(uint newIndex)
+    void incrementLargerOrEqualIndicesByOne(IndexType newIndex)
     {
         std::transform(m_indices.begin(),
                        m_indices.end(),
                        m_indices.begin(),
-                       [&] (uint index) {
+                       [&] (IndexType index) {
             return index >= newIndex ? ++index : index;
         });
     }
 
-    uint insertString(const_iterator beforeIterator,
+    IndexType insertString(const_iterator beforeIterator,
                       Utils::SmallStringView stringView)
     {
-        auto id = uint(m_indices.size());
+        auto id = IndexType(m_indices.size());
 
         auto inserted = m_strings.emplace(beforeIterator, StringType(stringView), id);
 
-        auto newIndex = uint(std::distance(m_strings.begin(), inserted));
+        auto newIndex = IndexType(std::distance(m_strings.begin(), inserted));
 
         incrementLargerOrEqualIndicesByOne(newIndex);
 
@@ -226,12 +233,13 @@ private:
     }
 
 private:
-    StringCacheEntries<StringType> m_strings;
-    std::vector<uint> m_indices;
+    CacheEnties m_strings;
+    std::vector<IndexType> m_indices;
     mutable Mutex m_mutex;
 };
 
-template <typename Mutex = NonLockingMutex>
-using FilePathCache = StringCache<Utils::PathString, Mutex>;
+template <typename Mutex>
+using FilePathCache = StringCache<Utils::PathString, FilePathIndex, Mutex>;
+using FilePathIndices = std::vector<FilePathIndex>;
 
 } // namespace ClangBackEnd

src/libs/clangsupport/stringcachefwd.h

@@ -29,14 +29,17 @@
 
 namespace ClangBackEnd {
 
+using FilePathIndex = long long int;
+
 class NonLockingMutex;
 
 template <typename StringType,
+          typename IndexType,
           typename Mutex>
 class StringCache;
 
 template <typename Mutex = NonLockingMutex>
-using FilePathCache = StringCache<Utils::PathString, Mutex>;
+using FilePathCache = StringCache<Utils::PathString, FilePathIndex, Mutex>;
 
 } // namespace ClangBackEnd
 

src/libs/utils/smallstringview.h

@@ -140,6 +140,8 @@ public:
         return m_pointer[0] == characterToSearch;
     }
 
+
+
 private:
     const char *m_pointer;
     size_type m_size;

src/tools/clangpchmanagerbackend/clangpchmanagerbackendmain.cpp

@@ -50,7 +50,7 @@ using ClangBackEnd::PchGenerator;
 using ClangBackEnd::PchManagerClientProxy;
 using ClangBackEnd::PchManagerServer;
 using ClangBackEnd::ProjectParts;
-using ClangBackEnd::StringCache;
+using ClangBackEnd::FilePathCache;
 
 class ApplicationEnvironment : public ClangBackEnd::Environment
 {
@@ -103,7 +103,7 @@ int main(int argc, char *argv[])
 
     const QString connection =  processArguments(application);
 
-    StringCache<Utils::PathString> filePathCache;
+    FilePathCache<> filePathCache;
     ClangPathWatcher<QFileSystemWatcher, QTimer> includeWatcher(filePathCache);
     ApplicationEnvironment environment;
     PchGenerator<QProcess> pchGenerator(environment);

src/tools/clangpchmanagerbackend/source/clangpathwatcher.h

@@ -37,8 +37,8 @@ namespace ClangBackEnd {
 class WatcherEntry
 {
 public:
-    uint id;
-    uint path;
+    FilePathIndex id;
+    FilePathIndex path;
 
     friend bool operator==(const WatcherEntry &first, const WatcherEntry &second)
     {
@@ -50,12 +50,12 @@ public:
         return std::tie(first.path, first.id) < std::tie(second.path, second.id);
     }
 
-    friend bool operator<(const WatcherEntry &entry, uint path)
+    friend bool operator<(const WatcherEntry &entry, FilePathIndex path)
     {
         return entry.path < path;
     }
 
-    friend bool operator<(uint path, const WatcherEntry &entry)
+    friend bool operator<(FilePathIndex path, const WatcherEntry &entry)
     {
         return path < entry.path;
     }
@@ -63,12 +63,14 @@ public:
 
 using WatcherEntries = std::vector<WatcherEntry>;
 
+using IdCache = StringCache<Utils::SmallString, FilePathIndex, NonLockingMutex>;
+
 template <typename FileSystemWatcher,
           typename Timer>
 class ClangPathWatcher : public ClangPathWatcherInterface
 {
 public:
-    ClangPathWatcher(StringCache<Utils::PathString> &pathCache,
+    ClangPathWatcher(FilePathCache<> &pathCache,
                      ClangPathWatcherNotifier *notifier=nullptr)
         : m_pathCache(pathCache),
           m_notifier(notifier)
@@ -130,9 +132,9 @@ unittest_public:
         return ids;
     }
 
-    std::vector<uint> convertToIdNumbers(const Utils::SmallStringVector &ids)
+    std::vector<FilePathIndex> convertToIdNumbers(const Utils::SmallStringVector &ids)
     {
-        std::vector<uint> idNumbers = m_idCache.stringIds(ids);
+        std::vector<FilePathIndex> idNumbers = m_idCache.stringIds(ids);
 
         std::sort(idNumbers.begin(), idNumbers.end());
 
@@ -149,19 +151,19 @@ unittest_public:
     }
 
 
-    std::pair<WatcherEntries,std::vector<uint>>
+    std::pair<WatcherEntries,std::vector<FilePathIndex>>
     convertIdPathsToWatcherEntriesAndIds(const std::vector<IdPaths> &idPaths)
     {
         WatcherEntries entries;
         entries.reserve(sizeOfIdPaths(idPaths));
-        std::vector<uint> ids;
+        std::vector<FilePathIndex> ids;
         ids.reserve(ids.size());
 
         auto outputIterator = std::back_inserter(entries);
 
         for (const IdPaths &idPath : idPaths)
         {
-            uint id = m_idCache.stringId(idPath.id);
+            FilePathIndex id = m_idCache.stringId(idPath.id);
 
             ids.push_back(id);
 
@@ -190,7 +192,7 @@ unittest_public:
     }
 
     void removeUnusedEntries(const WatcherEntries &entries,
-                             const std::vector<uint> &ids)
+                             const std::vector<FilePathIndex> &ids)
     {
         auto oldEntries = notAnymoreWatchedEntriesWithIds(entries, ids);
 
@@ -272,7 +274,7 @@ unittest_public:
 
     WatcherEntries notAnymoreWatchedEntriesWithIds(
             const WatcherEntries &newEntries,
-            const std::vector<uint> &ids) const
+            const std::vector<FilePathIndex> &ids) const
     {
         auto oldEntries = notAnymoreWatchedEntries(newEntries, std::less<WatcherEntry>());
 
@@ -328,7 +330,7 @@ unittest_public:
         return m_watchedEntries;
     }
 
-    WatcherEntries removeIdsFromWatchedEntries(const std::vector<uint> &ids)
+    WatcherEntries removeIdsFromWatchedEntries(const std::vector<FilePathIndex> &ids)
     {
 
         auto keep = [&] (const WatcherEntry &entry) {
@@ -368,7 +370,7 @@ unittest_public:
 
     WatcherEntries watchedEntriesForPaths(Utils::PathStringVector &&filePaths)
     {
-        std::vector<uint> pathIds = m_pathCache.stringIds(filePaths);
+        std::vector<FilePathIndex> pathIds = m_pathCache.stringIds(filePaths);
 
         WatcherEntries foundEntries;
 
@@ -414,22 +416,22 @@ unittest_public:
         }
     }
 
-    StringCache<Utils::PathString> &pathCache()
+    FilePathCache<> &pathCache()
     {
         return m_pathCache;
     }
 
-    StringCache<Utils::SmallString> &idCache()
+    IdCache &idCache()
     {
         return m_idCache;
     }
 
 private:
-    StringCache<Utils::SmallString> m_idCache;
+    IdCache m_idCache;
     WatcherEntries m_watchedEntries;
     ChangedFilePathCompressor<Timer> m_changedFilePathCompressor;
     FileSystemWatcher m_fileSystemWatcher;
-    StringCache<Utils::PathString> &m_pathCache;
+    FilePathCache<> &m_pathCache;
     ClangPathWatcherNotifier *m_notifier;
 };
 

src/tools/clangpchmanagerbackend/source/collectincludesaction.h

@@ -38,10 +38,10 @@ namespace ClangBackEnd {
 class CollectIncludesAction final : public clang::PreprocessOnlyAction
 {
 public:
-    CollectIncludesAction(std::vector<uint> &includeIds,
-                          StringCache<Utils::PathString> &filePathCache,
-                          std::vector<uint> &excludedIncludeUID,
-                          std::vector<uint> &alreadyIncludedFileUIDs)
+    CollectIncludesAction(FilePathIndices &includeIds,
+                          FilePathCache<> &filePathCache,
+                          FilePathIndices &excludedIncludeUID,
+                          FilePathIndices &alreadyIncludedFileUIDs)
         : m_includeIds(includeIds),
           m_filePathCache(filePathCache),
           m_excludedIncludeUID(excludedIncludeUID),
@@ -78,10 +78,10 @@ public:
     }
 
 private:
-    std::vector<uint> &m_includeIds;
-    StringCache<Utils::PathString> &m_filePathCache;
-    std::vector<uint> &m_excludedIncludeUID;
-    std::vector<uint> &m_alreadyIncludedFileUIDs;
+    FilePathIndices &m_includeIds;
+    FilePathCache<> &m_filePathCache;
+    FilePathIndices &m_excludedIncludeUID;
+    FilePathIndices &m_alreadyIncludedFileUIDs;
 };
 
 } // namespace ClangBackEnd

src/tools/clangpchmanagerbackend/source/collectincludespreprocessorcallbacks.h

@@ -47,10 +47,10 @@ class CollectIncludesPreprocessorCallbacks final : public clang::PPCallbacks
 {
 public:
     CollectIncludesPreprocessorCallbacks(clang::HeaderSearch &headerSearch,
-                                         std::vector<uint> &includeIds,
-                                         StringCache<Utils::PathString> &filePathCache,
-                                         const std::vector<uint> &excludedIncludeUID,
-                                         std::vector<uint>  &alreadyIncludedFileUIDs)
+                                         std::vector<FilePathIndex> &includeIds,
+                                         FilePathCache<> &filePathCache,
+                                         const std::vector<FilePathIndex> &excludedIncludeUID,
+                                         std::vector<FilePathIndex>  &alreadyIncludedFileUIDs)
         : m_headerSearch(headerSearch),
           m_includeIds(includeIds),
           m_filePathCache(filePathCache),
@@ -78,7 +78,7 @@ public:
                     m_alreadyIncludedFileUIDs.insert(notAlreadyIncluded.second, fileUID);
                     Utils::PathString filePath = filePathFromFile(file);
                     if (!filePath.isEmpty()) {
-                        uint includeId = m_filePathCache.stringId(filePath);
+                        FilePathIndex includeId = m_filePathCache.stringId(filePath);
                         m_includeIds.emplace_back(includeId);
                     }
                 }
@@ -132,7 +132,7 @@ public:
                                    uid);
     }
 
-    std::pair<bool, std::vector<uint>::iterator> isNotAlreadyIncluded(uint uid) const
+    std::pair<bool, std::vector<FilePathIndex>::iterator> isNotAlreadyIncluded(FilePathIndex uid) const
     {
         auto range = std::equal_range(m_alreadyIncludedFileUIDs.begin(),
                                       m_alreadyIncludedFileUIDs.end(),
@@ -174,10 +174,10 @@ public:
 
 private:
     clang::HeaderSearch &m_headerSearch;
-    std::vector<uint> &m_includeIds;
-    StringCache<Utils::PathString> &m_filePathCache;
-    const std::vector<uint> &m_excludedIncludeUID;
-    std::vector<uint> &m_alreadyIncludedFileUIDs;
+    std::vector<FilePathIndex> &m_includeIds;
+    FilePathCache<> &m_filePathCache;
+    const std::vector<FilePathIndex> &m_excludedIncludeUID;
+    std::vector<FilePathIndex> &m_alreadyIncludedFileUIDs;
     bool m_skipInclude = false;
 };
 

src/tools/clangpchmanagerbackend/source/collectincludestoolaction.h

@@ -36,8 +36,8 @@ namespace ClangBackEnd {
 class CollectIncludesToolAction final : public clang::tooling::FrontendActionFactory
 {
 public:
-    CollectIncludesToolAction(std::vector<uint> &includeIds,
-                              StringCache<Utils::PathString> &filePathCache,
+    CollectIncludesToolAction(std::vector<FilePathIndex> &includeIds,
+                              FilePathCache<> &filePathCache,
                               const Utils::PathStringVector &excludedIncludes)
         : m_includeIds(includeIds),
           m_filePathCache(filePathCache),
@@ -72,9 +72,9 @@ public:
                                          m_alreadyIncludedFileUIDs);
     }
 
-    std::vector<uint> generateExcludedIncludeFileUIDs(clang::FileManager &fileManager) const
+    std::vector<FilePathIndex> generateExcludedIncludeFileUIDs(clang::FileManager &fileManager) const
     {
-        std::vector<uint> fileUIDs;
+        std::vector<FilePathIndex> fileUIDs;
         fileUIDs.reserve(m_excludedIncludes.size());
 
         for (const Utils::PathString &filePath : m_excludedIncludes) {
@@ -90,10 +90,10 @@ public:
     }
 
 private:
-    std::vector<uint> m_alreadyIncludedFileUIDs;
-    std::vector<uint> m_excludedIncludeUIDs;
-    std::vector<uint> &m_includeIds;
-    StringCache<Utils::PathString> &m_filePathCache;
+    std::vector<FilePathIndex> m_alreadyIncludedFileUIDs;
+    std::vector<FilePathIndex> m_excludedIncludeUIDs;
+    std::vector<FilePathIndex> &m_includeIds;
+    FilePathCache<> &m_filePathCache;
     const Utils::PathStringVector &m_excludedIncludes;
 };
 

src/tools/clangpchmanagerbackend/source/idpaths.h

@@ -29,13 +29,15 @@
 
 #include <iosfwd>
 
+#include <stringcachefwd.h>
+
 namespace ClangBackEnd {
 
 class IdPaths
 {
 public:
     Utils::SmallString id;
-    std::vector<uint> paths;
+    std::vector<FilePathIndex> paths;
 
     friend bool operator==(const IdPaths &first, const IdPaths &second)
     {

src/tools/clangpchmanagerbackend/source/includecollector.cpp

@@ -33,7 +33,7 @@
 
 namespace ClangBackEnd {
 
-IncludeCollector::IncludeCollector(StringCache<Utils::PathString> &filePathCache)
+IncludeCollector::IncludeCollector(FilePathCache<> &filePathCache)
     :  m_filePathCache(filePathCache)
 {
 }
@@ -69,7 +69,7 @@ void IncludeCollector::setExcludedIncludes(Utils::PathStringVector &&excludedInc
 #endif
 }
 
-std::vector<uint> IncludeCollector::takeIncludeIds()
+std::vector<FilePathIndex> IncludeCollector::takeIncludeIds()
 {
     std::sort(m_includeIds.begin(), m_includeIds.end());
 

src/tools/clangpchmanagerbackend/source/includecollector.h

@@ -34,19 +34,19 @@ namespace ClangBackEnd {
 class IncludeCollector : public ClangTool
 {
 public:
-    IncludeCollector(StringCache<Utils::PathString> &filePathCache);
+    IncludeCollector(FilePathCache<> &filePathCache);
 
     void collectIncludes();
 
     void setExcludedIncludes(Utils::PathStringVector &&excludedIncludes);
 
-    std::vector<uint> takeIncludeIds();
+    std::vector<FilePathIndex> takeIncludeIds();
 
 private:
     Utils::PathStringVector m_excludedIncludes;
-    std::vector<uint> m_includeIds;
+    std::vector<FilePathIndex> m_includeIds;
     Utils::SmallStringVector m_directories;
-    StringCache<Utils::PathString> &m_filePathCache;
+    FilePathCache<> &m_filePathCache;
 };
 
 } // namespace ClangBackEnd

src/tools/clangpchmanagerbackend/source/pchcreator.cpp

@@ -37,7 +37,7 @@
 
 namespace ClangBackEnd {
 
-PchCreator::PchCreator(Environment &environment, StringCache<Utils::PathString> &filePathCache)
+PchCreator::PchCreator(Environment &environment, FilePathCache<> &filePathCache)
    : m_environment(environment),
      m_filePathCache(filePathCache)
 {
@@ -45,7 +45,7 @@ PchCreator::PchCreator(Environment &environment, StringCache<Utils::PathString>
 
 PchCreator::PchCreator(V2::ProjectPartContainers &&projectsParts,
                        Environment &environment,
-                       StringCache<Utils::PathString> &filePathCache,
+                       FilePathCache<> &filePathCache,
                        PchGeneratorInterface *pchGenerator,
                        V2::FileContainers &&generatedFiles)
     : m_projectParts(std::move(projectsParts)),
@@ -239,7 +239,7 @@ Utils::SmallStringVector PchCreator::generateGlobalClangCompilerArguments() cons
     return compilerArguments;
 }
 
-std::vector<uint> PchCreator::generateGlobalPchIncludeIds() const
+std::vector<FilePathIndex> PchCreator::generateGlobalPchIncludeIds() const
 {
     IncludeCollector collector(m_filePathCache);
 
@@ -267,7 +267,7 @@ std::size_t contentSize(const std::vector<Utils::PathString> &includes)
 }
 
 Utils::SmallString PchCreator::generatePchIncludeFileContent(
-        const std::vector<uint> &includeIds) const
+        const std::vector<FilePathIndex> &includeIds) const
 {
     Utils::SmallString fileContent;
     const std::size_t lineTemplateSize = 12;
@@ -461,7 +461,7 @@ Utils::PathStringVector PchCreator::generateProjectPartHeaderAndSourcePaths(
     return includeAndSources;
 }
 
-std::vector<uint> PchCreator::generateProjectPartPchIncludes(
+std::vector<FilePathIndex> PchCreator::generateProjectPartPchIncludes(
         const V2::ProjectPartContainer &projectPart) const
 {
     Utils::SmallString jointedFileContent = generateProjectPartHeaderAndSourcesContent(projectPart);

src/tools/clangpchmanagerbackend/source/pchcreator.h

@@ -48,10 +48,10 @@ class PchCreator final : public PchCreatorInterface
 {
 public:
     PchCreator(Environment &environment,
-               StringCache<Utils::PathString> &filePathCache);
+               FilePathCache<> &filePathCache);
     PchCreator(V2::ProjectPartContainers &&projectsParts,
                Environment &environment,
-               StringCache<Utils::PathString> &filePathCache,
+               FilePathCache<> &filePathCache,
                PchGeneratorInterface *pchGenerator,
                V2::FileContainers &&generatedFiles);
 
@@ -70,9 +70,9 @@ unittest_public:
     Utils::SmallStringVector generateGlobalPchCompilerArguments() const;
     Utils::SmallStringVector generateGlobalClangCompilerArguments() const;
 
-    std::vector<uint> generateGlobalPchIncludeIds() const;
+    std::vector<FilePathIndex> generateGlobalPchIncludeIds() const;
 
-    Utils::SmallString generatePchIncludeFileContent(const std::vector<uint> &includeIds) const;
+    Utils::SmallString generatePchIncludeFileContent(const std::vector<FilePathIndex> &includeIds) const;
     Utils::SmallString generateGlobalPchHeaderFileContent() const;
     std::unique_ptr<QFile> generateGlobalPchHeaderFile();
     void generatePch(Utils::SmallStringVector &&commandLineArguments,
@@ -97,7 +97,7 @@ unittest_public:
             const V2::ProjectPartContainer &projectPart);