Commit f49a1d72 authored by Marco Bubke's avatar Marco Bubke

Clang: Add symbol storage

Extend file path cache to 64 bit integer.

Change-Id: I5627f13d59a3214f389087038482cbcc8d0eb484
Reviewed-by: Tim Jenssen's avatarTim Jenssen <tim.jenssen@qt.io>
parent 81d43b8a
...@@ -25,6 +25,8 @@ ...@@ -25,6 +25,8 @@
#pragma once #pragma once
#include "stringcachefwd.h"
#include <utils/smallstringview.h> #include <utils/smallstringview.h>
#include <utils/smallstringfwd.h> #include <utils/smallstringfwd.h>
...@@ -52,11 +54,11 @@ public: ...@@ -52,11 +54,11 @@ public:
void unlock() {} void unlock() {}
}; };
template <typename StringType> template <typename StringType, typename IndexType>
class StringCacheEntry class StringCacheEntry
{ {
public: public:
StringCacheEntry(StringType &&string, uint id) StringCacheEntry(StringType &&string, IndexType id)
: string(std::move(string)), : string(std::move(string)),
id(id) id(id)
{} {}
...@@ -77,22 +79,27 @@ public: ...@@ -77,22 +79,27 @@ public:
} }
StringType string; StringType string;
uint id; IndexType id;
}; };
template <typename StringType> template <typename StringType, typename IndexType>
using StringCacheEntries = std::vector<StringCacheEntry<StringType>>; using StringCacheEntries = std::vector<StringCacheEntry<StringType, IndexType>>;
using FileCacheCacheEntry = StringCacheEntry<Utils::PathString, FilePathIndex>;
using FileCacheCacheEntries = std::vector<FileCacheCacheEntry>;
template <typename StringType, template <typename StringType,
typename Mutex = NonLockingMutex> typename IndexType,
typename Mutex>
class StringCache class StringCache
{ {
using const_iterator = typename StringCacheEntries<StringType>::const_iterator; using CacheEnties = StringCacheEntries<StringType, IndexType>;
using const_iterator = typename CacheEnties::const_iterator;
class Found class Found
{ {
public: public:
typename StringCacheEntries<StringType>::const_iterator iterator; typename CacheEnties::const_iterator iterator;
bool wasFound; bool wasFound;
}; };
...@@ -103,14 +110,14 @@ public: ...@@ -103,14 +110,14 @@ public:
m_indices.reserve(1024); m_indices.reserve(1024);
} }
void populate(StringCacheEntries<StringType> &&entries) void populate(CacheEnties &&entries)
{ {
uncheckedPopulate(std::move(entries)); uncheckedPopulate(std::move(entries));
checkEntries(); checkEntries();
} }
void uncheckedPopulate(StringCacheEntries<StringType> &&entries) void uncheckedPopulate(CacheEnties &&entries)
{ {
std::sort(entries.begin(), entries.end()); std::sort(entries.begin(), entries.end());
...@@ -123,7 +130,7 @@ public: ...@@ -123,7 +130,7 @@ public:
} }
uint stringId(Utils::SmallStringView stringView) IndexType stringId(Utils::SmallStringView stringView)
{ {
std::lock_guard<Mutex> lock(m_mutex); std::lock_guard<Mutex> lock(m_mutex);
...@@ -136,11 +143,11 @@ public: ...@@ -136,11 +143,11 @@ public:
} }
template <typename Container> template <typename Container>
std::vector<uint> stringIds(const Container &strings) std::vector<IndexType> stringIds(const Container &strings)
{ {
std::lock_guard<Mutex> lock(m_mutex); std::lock_guard<Mutex> lock(m_mutex);
std::vector<uint> ids; std::vector<IndexType> ids;
ids.reserve(strings.size()); ids.reserve(strings.size());
std::transform(strings.begin(), std::transform(strings.begin(),
...@@ -151,19 +158,19 @@ public: ...@@ -151,19 +158,19 @@ public:
return ids; return ids;
} }
std::vector<uint> stringIds(std::initializer_list<StringType> strings) std::vector<IndexType> stringIds(std::initializer_list<StringType> strings)
{ {
return stringIds<std::initializer_list<StringType>>(strings); return stringIds<std::initializer_list<StringType>>(strings);
} }
Utils::SmallStringView string(uint id) const Utils::SmallStringView string(IndexType id) const
{ {
std::lock_guard<Mutex> lock(m_mutex); std::lock_guard<Mutex> lock(m_mutex);
return m_strings.at(m_indices.at(id)).string; return m_strings.at(m_indices.at(id)).string;
} }
std::vector<StringType> strings(const std::vector<uint> &ids) const std::vector<StringType> strings(const std::vector<IndexType> &ids) const
{ {
std::lock_guard<Mutex> lock(m_mutex); std::lock_guard<Mutex> lock(m_mutex);
...@@ -173,7 +180,7 @@ public: ...@@ -173,7 +180,7 @@ public:
std::transform(ids.begin(), std::transform(ids.begin(),
ids.end(), ids.end(),
std::back_inserter(strings), std::back_inserter(strings),
[&] (uint id) { return m_strings.at(m_indices.at(id)).string; }); [&] (IndexType id) { return m_strings.at(m_indices.at(id)).string; });
return strings; return strings;
} }
...@@ -191,24 +198,24 @@ private: ...@@ -191,24 +198,24 @@ private:
return {range.first, range.first != range.second}; return {range.first, range.first != range.second};
} }
void incrementLargerOrEqualIndicesByOne(uint newIndex) void incrementLargerOrEqualIndicesByOne(IndexType newIndex)
{ {
std::transform(m_indices.begin(), std::transform(m_indices.begin(),
m_indices.end(), m_indices.end(),
m_indices.begin(), m_indices.begin(),
[&] (uint index) { [&] (IndexType index) {
return index >= newIndex ? ++index : index; return index >= newIndex ? ++index : index;
}); });
} }
uint insertString(const_iterator beforeIterator, IndexType insertString(const_iterator beforeIterator,
Utils::SmallStringView stringView) Utils::SmallStringView stringView)
{ {
auto id = uint(m_indices.size()); auto id = IndexType(m_indices.size());
auto inserted = m_strings.emplace(beforeIterator, StringType(stringView), id); auto inserted = m_strings.emplace(beforeIterator, StringType(stringView), id);
auto newIndex = uint(std::distance(m_strings.begin(), inserted)); auto newIndex = IndexType(std::distance(m_strings.begin(), inserted));
incrementLargerOrEqualIndicesByOne(newIndex); incrementLargerOrEqualIndicesByOne(newIndex);
...@@ -226,12 +233,13 @@ private: ...@@ -226,12 +233,13 @@ private:
} }
private: private:
StringCacheEntries<StringType> m_strings; CacheEnties m_strings;
std::vector<uint> m_indices; std::vector<IndexType> m_indices;
mutable Mutex m_mutex; mutable Mutex m_mutex;
}; };
template <typename Mutex = NonLockingMutex> template <typename Mutex>
using FilePathCache = StringCache<Utils::PathString, Mutex>; using FilePathCache = StringCache<Utils::PathString, FilePathIndex, Mutex>;
using FilePathIndices = std::vector<FilePathIndex>;
} // namespace ClangBackEnd } // namespace ClangBackEnd
...@@ -29,14 +29,17 @@ ...@@ -29,14 +29,17 @@
namespace ClangBackEnd { namespace ClangBackEnd {
using FilePathIndex = long long int;
class NonLockingMutex; class NonLockingMutex;
template <typename StringType, template <typename StringType,
typename IndexType,
typename Mutex> typename Mutex>
class StringCache; class StringCache;
template <typename Mutex = NonLockingMutex> template <typename Mutex = NonLockingMutex>
using FilePathCache = StringCache<Utils::PathString, Mutex>; using FilePathCache = StringCache<Utils::PathString, FilePathIndex, Mutex>;
} // namespace ClangBackEnd } // namespace ClangBackEnd
This diff is collapsed.
...@@ -140,6 +140,8 @@ public: ...@@ -140,6 +140,8 @@ public:
return m_pointer[0] == characterToSearch; return m_pointer[0] == characterToSearch;
} }
private: private:
const char *m_pointer; const char *m_pointer;
size_type m_size; size_type m_size;
......
...@@ -50,7 +50,7 @@ using ClangBackEnd::PchGenerator; ...@@ -50,7 +50,7 @@ using ClangBackEnd::PchGenerator;
using ClangBackEnd::PchManagerClientProxy; using ClangBackEnd::PchManagerClientProxy;
using ClangBackEnd::PchManagerServer; using ClangBackEnd::PchManagerServer;
using ClangBackEnd::ProjectParts; using ClangBackEnd::ProjectParts;
using ClangBackEnd::StringCache; using ClangBackEnd::FilePathCache;
class ApplicationEnvironment : public ClangBackEnd::Environment class ApplicationEnvironment : public ClangBackEnd::Environment
{ {
...@@ -103,7 +103,7 @@ int main(int argc, char *argv[]) ...@@ -103,7 +103,7 @@ int main(int argc, char *argv[])
const QString connection = processArguments(application); const QString connection = processArguments(application);
StringCache<Utils::PathString> filePathCache; FilePathCache<> filePathCache;
ClangPathWatcher<QFileSystemWatcher, QTimer> includeWatcher(filePathCache); ClangPathWatcher<QFileSystemWatcher, QTimer> includeWatcher(filePathCache);
ApplicationEnvironment environment; ApplicationEnvironment environment;
PchGenerator<QProcess> pchGenerator(environment); PchGenerator<QProcess> pchGenerator(environment);
......
...@@ -37,8 +37,8 @@ namespace ClangBackEnd { ...@@ -37,8 +37,8 @@ namespace ClangBackEnd {
class WatcherEntry class WatcherEntry
{ {
public: public:
uint id; FilePathIndex id;
uint path; FilePathIndex path;
friend bool operator==(const WatcherEntry &first, const WatcherEntry &second) friend bool operator==(const WatcherEntry &first, const WatcherEntry &second)
{ {
...@@ -50,12 +50,12 @@ public: ...@@ -50,12 +50,12 @@ public:
return std::tie(first.path, first.id) < std::tie(second.path, second.id); return std::tie(first.path, first.id) < std::tie(second.path, second.id);
} }
friend bool operator<(const WatcherEntry &entry, uint path) friend bool operator<(const WatcherEntry &entry, FilePathIndex path)
{ {
return entry.path < path; return entry.path < path;
} }
friend bool operator<(uint path, const WatcherEntry &entry) friend bool operator<(FilePathIndex path, const WatcherEntry &entry)
{ {
return path < entry.path; return path < entry.path;
} }
...@@ -63,12 +63,14 @@ public: ...@@ -63,12 +63,14 @@ public:
using WatcherEntries = std::vector<WatcherEntry>; using WatcherEntries = std::vector<WatcherEntry>;
using IdCache = StringCache<Utils::SmallString, FilePathIndex, NonLockingMutex>;
template <typename FileSystemWatcher, template <typename FileSystemWatcher,
typename Timer> typename Timer>
class ClangPathWatcher : public ClangPathWatcherInterface class ClangPathWatcher : public ClangPathWatcherInterface
{ {
public: public:
ClangPathWatcher(StringCache<Utils::PathString> &pathCache, ClangPathWatcher(FilePathCache<> &pathCache,
ClangPathWatcherNotifier *notifier=nullptr) ClangPathWatcherNotifier *notifier=nullptr)
: m_pathCache(pathCache), : m_pathCache(pathCache),
m_notifier(notifier) m_notifier(notifier)
...@@ -130,9 +132,9 @@ unittest_public: ...@@ -130,9 +132,9 @@ unittest_public:
return ids; return ids;
} }
std::vector<uint> convertToIdNumbers(const Utils::SmallStringVector &ids) std::vector<FilePathIndex> convertToIdNumbers(const Utils::SmallStringVector &ids)
{ {
std::vector<uint> idNumbers = m_idCache.stringIds(ids); std::vector<FilePathIndex> idNumbers = m_idCache.stringIds(ids);
std::sort(idNumbers.begin(), idNumbers.end()); std::sort(idNumbers.begin(), idNumbers.end());
...@@ -149,19 +151,19 @@ unittest_public: ...@@ -149,19 +151,19 @@ unittest_public:
} }
std::pair<WatcherEntries,std::vector<uint>> std::pair<WatcherEntries,std::vector<FilePathIndex>>
convertIdPathsToWatcherEntriesAndIds(const std::vector<IdPaths> &idPaths) convertIdPathsToWatcherEntriesAndIds(const std::vector<IdPaths> &idPaths)
{ {
WatcherEntries entries; WatcherEntries entries;
entries.reserve(sizeOfIdPaths(idPaths)); entries.reserve(sizeOfIdPaths(idPaths));
std::vector<uint> ids; std::vector<FilePathIndex> ids;
ids.reserve(ids.size()); ids.reserve(ids.size());
auto outputIterator = std::back_inserter(entries); auto outputIterator = std::back_inserter(entries);
for (const IdPaths &idPath : idPaths) for (const IdPaths &idPath : idPaths)
{ {
uint id = m_idCache.stringId(idPath.id); FilePathIndex id = m_idCache.stringId(idPath.id);
ids.push_back(id); ids.push_back(id);
...@@ -190,7 +192,7 @@ unittest_public: ...@@ -190,7 +192,7 @@ unittest_public:
} }
void removeUnusedEntries(const WatcherEntries &entries, void removeUnusedEntries(const WatcherEntries &entries,
const std::vector<uint> &ids) const std::vector<FilePathIndex> &ids)
{ {
auto oldEntries = notAnymoreWatchedEntriesWithIds(entries, ids); auto oldEntries = notAnymoreWatchedEntriesWithIds(entries, ids);
...@@ -272,7 +274,7 @@ unittest_public: ...@@ -272,7 +274,7 @@ unittest_public:
WatcherEntries notAnymoreWatchedEntriesWithIds( WatcherEntries notAnymoreWatchedEntriesWithIds(
const WatcherEntries &newEntries, const WatcherEntries &newEntries,
const std::vector<uint> &ids) const const std::vector<FilePathIndex> &ids) const
{ {
auto oldEntries = notAnymoreWatchedEntries(newEntries, std::less<WatcherEntry>()); auto oldEntries = notAnymoreWatchedEntries(newEntries, std::less<WatcherEntry>());
...@@ -328,7 +330,7 @@ unittest_public: ...@@ -328,7 +330,7 @@ unittest_public:
return m_watchedEntries; return m_watchedEntries;
} }
WatcherEntries removeIdsFromWatchedEntries(const std::vector<uint> &ids) WatcherEntries removeIdsFromWatchedEntries(const std::vector<FilePathIndex> &ids)
{ {
auto keep = [&] (const WatcherEntry &entry) { auto keep = [&] (const WatcherEntry &entry) {
...@@ -368,7 +370,7 @@ unittest_public: ...@@ -368,7 +370,7 @@ unittest_public:
WatcherEntries watchedEntriesForPaths(Utils::PathStringVector &&filePaths) WatcherEntries watchedEntriesForPaths(Utils::PathStringVector &&filePaths)
{ {
std::vector<uint> pathIds = m_pathCache.stringIds(filePaths); std::vector<FilePathIndex> pathIds = m_pathCache.stringIds(filePaths);
WatcherEntries foundEntries; WatcherEntries foundEntries;
...@@ -414,22 +416,22 @@ unittest_public: ...@@ -414,22 +416,22 @@ unittest_public:
} }
} }
StringCache<Utils::PathString> &pathCache() FilePathCache<> &pathCache()
{ {
return m_pathCache; return m_pathCache;
} }
StringCache<Utils::SmallString> &idCache() IdCache &idCache()
{ {
return m_idCache; return m_idCache;
} }
private: private:
StringCache<Utils::SmallString> m_idCache; IdCache m_idCache;
WatcherEntries m_watchedEntries; WatcherEntries m_watchedEntries;
ChangedFilePathCompressor<Timer> m_changedFilePathCompressor; ChangedFilePathCompressor<Timer> m_changedFilePathCompressor;
FileSystemWatcher m_fileSystemWatcher; FileSystemWatcher m_fileSystemWatcher;
StringCache<Utils::PathString> &m_pathCache; FilePathCache<> &m_pathCache;
ClangPathWatcherNotifier *m_notifier; ClangPathWatcherNotifier *m_notifier;
}; };
......
...@@ -38,10 +38,10 @@ namespace ClangBackEnd { ...@@ -38,10 +38,10 @@ namespace ClangBackEnd {
class CollectIncludesAction final : public clang::PreprocessOnlyAction class CollectIncludesAction final : public clang::PreprocessOnlyAction
{ {
public: public:
CollectIncludesAction(std::vector<uint> &includeIds, CollectIncludesAction(FilePathIndices &includeIds,
StringCache<Utils::PathString> &filePathCache, FilePathCache<> &filePathCache,
std::vector<uint> &excludedIncludeUID, FilePathIndices &excludedIncludeUID,
std::vector<uint> &alreadyIncludedFileUIDs) FilePathIndices &alreadyIncludedFileUIDs)
: m_includeIds(includeIds), : m_includeIds(includeIds),
m_filePathCache(filePathCache), m_filePathCache(filePathCache),
m_excludedIncludeUID(excludedIncludeUID), m_excludedIncludeUID(excludedIncludeUID),
...@@ -78,10 +78,10 @@ public: ...@@ -78,10 +78,10 @@ public:
} }
private: private:
std::vector<uint> &m_includeIds; FilePathIndices &m_includeIds;
StringCache<Utils::PathString> &m_filePathCache; FilePathCache<> &m_filePathCache;
std::vector<uint> &m_excludedIncludeUID; FilePathIndices &m_excludedIncludeUID;
std::vector<uint> &m_alreadyIncludedFileUIDs; FilePathIndices &m_alreadyIncludedFileUIDs;
}; };
} // namespace ClangBackEnd } // namespace ClangBackEnd
...@@ -47,10 +47,10 @@ class CollectIncludesPreprocessorCallbacks final : public clang::PPCallbacks ...@@ -47,10 +47,10 @@ class CollectIncludesPreprocessorCallbacks final : public clang::PPCallbacks
{ {
public: public:
CollectIncludesPreprocessorCallbacks(clang::HeaderSearch &headerSearch, CollectIncludesPreprocessorCallbacks(clang::HeaderSearch &headerSearch,
std::vector<uint> &includeIds, std::vector<FilePathIndex> &includeIds,
StringCache<Utils::PathString> &filePathCache, FilePathCache<> &filePathCache,
const std::vector<uint> &excludedIncludeUID, const std::vector<FilePathIndex> &excludedIncludeUID,
std::vector<uint> &alreadyIncludedFileUIDs) std::vector<FilePathIndex> &alreadyIncludedFileUIDs)
: m_headerSearch(headerSearch), : m_headerSearch(headerSearch),
m_includeIds(includeIds), m_includeIds(includeIds),
m_filePathCache(filePathCache), m_filePathCache(filePathCache),
...@@ -78,7 +78,7 @@ public: ...@@ -78,7 +78,7 @@ public:
m_alreadyIncludedFileUIDs.insert(notAlreadyIncluded.second, fileUID); m_alreadyIncludedFileUIDs.insert(notAlreadyIncluded.second, fileUID);
Utils::PathString filePath = filePathFromFile(file); Utils::PathString filePath = filePathFromFile(file);
if (!filePath.isEmpty()) { if (!filePath.isEmpty()) {
uint includeId = m_filePathCache.stringId(filePath); FilePathIndex includeId = m_filePathCache.stringId(filePath);
m_includeIds.emplace_back(includeId); m_includeIds.emplace_back(includeId);
} }