Commit f49a1d72 authored by Marco Bubke's avatar Marco Bubke

Clang: Add symbol storage

Extend file path cache to 64 bit integer.

Change-Id: I5627f13d59a3214f389087038482cbcc8d0eb484
Reviewed-by: Tim Jenssen's avatarTim Jenssen <tim.jenssen@qt.io>
parent 81d43b8a
......@@ -25,6 +25,8 @@
#pragma once
#include "stringcachefwd.h"
#include <utils/smallstringview.h>
#include <utils/smallstringfwd.h>
......@@ -52,11 +54,11 @@ public:
void unlock() {}
};
template <typename StringType>
template <typename StringType, typename IndexType>
class StringCacheEntry
{
public:
StringCacheEntry(StringType &&string, uint id)
StringCacheEntry(StringType &&string, IndexType id)
: string(std::move(string)),
id(id)
{}
......@@ -77,22 +79,27 @@ public:
}
StringType string;
uint id;
IndexType id;
};
template <typename StringType>
using StringCacheEntries = std::vector<StringCacheEntry<StringType>>;
template <typename StringType, typename IndexType>
using StringCacheEntries = std::vector<StringCacheEntry<StringType, IndexType>>;
using FileCacheCacheEntry = StringCacheEntry<Utils::PathString, FilePathIndex>;
using FileCacheCacheEntries = std::vector<FileCacheCacheEntry>;
template <typename StringType,
typename Mutex = NonLockingMutex>
typename IndexType,
typename Mutex>
class StringCache
{
using const_iterator = typename StringCacheEntries<StringType>::const_iterator;
using CacheEnties = StringCacheEntries<StringType, IndexType>;
using const_iterator = typename CacheEnties::const_iterator;
class Found
{
public:
typename StringCacheEntries<StringType>::const_iterator iterator;
typename CacheEnties::const_iterator iterator;
bool wasFound;
};
......@@ -103,14 +110,14 @@ public:
m_indices.reserve(1024);
}
void populate(StringCacheEntries<StringType> &&entries)
void populate(CacheEnties &&entries)
{
uncheckedPopulate(std::move(entries));
checkEntries();
}
void uncheckedPopulate(StringCacheEntries<StringType> &&entries)
void uncheckedPopulate(CacheEnties &&entries)
{
std::sort(entries.begin(), entries.end());
......@@ -123,7 +130,7 @@ public:
}
uint stringId(Utils::SmallStringView stringView)
IndexType stringId(Utils::SmallStringView stringView)
{
std::lock_guard<Mutex> lock(m_mutex);
......@@ -136,11 +143,11 @@ public:
}
template <typename Container>
std::vector<uint> stringIds(const Container &strings)
std::vector<IndexType> stringIds(const Container &strings)
{
std::lock_guard<Mutex> lock(m_mutex);
std::vector<uint> ids;
std::vector<IndexType> ids;
ids.reserve(strings.size());
std::transform(strings.begin(),
......@@ -151,19 +158,19 @@ public:
return ids;
}
std::vector<uint> stringIds(std::initializer_list<StringType> strings)
std::vector<IndexType> stringIds(std::initializer_list<StringType> strings)
{
return stringIds<std::initializer_list<StringType>>(strings);
}
Utils::SmallStringView string(uint id) const
Utils::SmallStringView string(IndexType id) const
{
std::lock_guard<Mutex> lock(m_mutex);
return m_strings.at(m_indices.at(id)).string;
}
std::vector<StringType> strings(const std::vector<uint> &ids) const
std::vector<StringType> strings(const std::vector<IndexType> &ids) const
{
std::lock_guard<Mutex> lock(m_mutex);
......@@ -173,7 +180,7 @@ public:
std::transform(ids.begin(),
ids.end(),
std::back_inserter(strings),
[&] (uint id) { return m_strings.at(m_indices.at(id)).string; });
[&] (IndexType id) { return m_strings.at(m_indices.at(id)).string; });
return strings;
}
......@@ -191,24 +198,24 @@ private:
return {range.first, range.first != range.second};
}
void incrementLargerOrEqualIndicesByOne(uint newIndex)
void incrementLargerOrEqualIndicesByOne(IndexType newIndex)
{
std::transform(m_indices.begin(),
m_indices.end(),
m_indices.begin(),
[&] (uint index) {
[&] (IndexType index) {
return index >= newIndex ? ++index : index;
});
}
uint insertString(const_iterator beforeIterator,
IndexType insertString(const_iterator beforeIterator,
Utils::SmallStringView stringView)
{
auto id = uint(m_indices.size());
auto id = IndexType(m_indices.size());
auto inserted = m_strings.emplace(beforeIterator, StringType(stringView), id);
auto newIndex = uint(std::distance(m_strings.begin(), inserted));
auto newIndex = IndexType(std::distance(m_strings.begin(), inserted));
incrementLargerOrEqualIndicesByOne(newIndex);
......@@ -226,12 +233,13 @@ private:
}
private:
StringCacheEntries<StringType> m_strings;
std::vector<uint> m_indices;
CacheEnties m_strings;
std::vector<IndexType> m_indices;
mutable Mutex m_mutex;
};
template <typename Mutex = NonLockingMutex>
using FilePathCache = StringCache<Utils::PathString, Mutex>;
template <typename Mutex>
using FilePathCache = StringCache<Utils::PathString, FilePathIndex, Mutex>;
using FilePathIndices = std::vector<FilePathIndex>;
} // namespace ClangBackEnd
......@@ -29,14 +29,17 @@
namespace ClangBackEnd {
using FilePathIndex = long long int;
class NonLockingMutex;
template <typename StringType,
typename IndexType,
typename Mutex>
class StringCache;
template <typename Mutex = NonLockingMutex>
using FilePathCache = StringCache<Utils::PathString, Mutex>;
using FilePathCache = StringCache<Utils::PathString, FilePathIndex, Mutex>;
} // namespace ClangBackEnd
This diff is collapsed.
......@@ -140,6 +140,8 @@ public:
return m_pointer[0] == characterToSearch;
}
private:
const char *m_pointer;
size_type m_size;
......
......@@ -50,7 +50,7 @@ using ClangBackEnd::PchGenerator;
using ClangBackEnd::PchManagerClientProxy;
using ClangBackEnd::PchManagerServer;
using ClangBackEnd::ProjectParts;
using ClangBackEnd::StringCache;
using ClangBackEnd::FilePathCache;
class ApplicationEnvironment : public ClangBackEnd::Environment
{
......@@ -103,7 +103,7 @@ int main(int argc, char *argv[])
const QString connection = processArguments(application);
StringCache<Utils::PathString> filePathCache;
FilePathCache<> filePathCache;
ClangPathWatcher<QFileSystemWatcher, QTimer> includeWatcher(filePathCache);
ApplicationEnvironment environment;
PchGenerator<QProcess> pchGenerator(environment);
......
......@@ -37,8 +37,8 @@ namespace ClangBackEnd {
class WatcherEntry
{
public:
uint id;
uint path;
FilePathIndex id;
FilePathIndex path;
friend bool operator==(const WatcherEntry &first, const WatcherEntry &second)
{
......@@ -50,12 +50,12 @@ public:
return std::tie(first.path, first.id) < std::tie(second.path, second.id);
}
friend bool operator<(const WatcherEntry &entry, uint path)
friend bool operator<(const WatcherEntry &entry, FilePathIndex path)
{
return entry.path < path;
}
friend bool operator<(uint path, const WatcherEntry &entry)
friend bool operator<(FilePathIndex path, const WatcherEntry &entry)
{
return path < entry.path;
}
......@@ -63,12 +63,14 @@ public:
using WatcherEntries = std::vector<WatcherEntry>;
using IdCache = StringCache<Utils::SmallString, FilePathIndex, NonLockingMutex>;
template <typename FileSystemWatcher,
typename Timer>
class ClangPathWatcher : public ClangPathWatcherInterface
{
public:
ClangPathWatcher(StringCache<Utils::PathString> &pathCache,
ClangPathWatcher(FilePathCache<> &pathCache,
ClangPathWatcherNotifier *notifier=nullptr)
: m_pathCache(pathCache),
m_notifier(notifier)
......@@ -130,9 +132,9 @@ unittest_public:
return ids;
}
std::vector<uint> convertToIdNumbers(const Utils::SmallStringVector &ids)
std::vector<FilePathIndex> convertToIdNumbers(const Utils::SmallStringVector &ids)
{
std::vector<uint> idNumbers = m_idCache.stringIds(ids);
std::vector<FilePathIndex> idNumbers = m_idCache.stringIds(ids);
std::sort(idNumbers.begin(), idNumbers.end());
......@@ -149,19 +151,19 @@ unittest_public:
}
std::pair<WatcherEntries,std::vector<uint>>
std::pair<WatcherEntries,std::vector<FilePathIndex>>
convertIdPathsToWatcherEntriesAndIds(const std::vector<IdPaths> &idPaths)
{
WatcherEntries entries;
entries.reserve(sizeOfIdPaths(idPaths));
std::vector<uint> ids;
std::vector<FilePathIndex> ids;
ids.reserve(ids.size());
auto outputIterator = std::back_inserter(entries);
for (const IdPaths &idPath : idPaths)
{
uint id = m_idCache.stringId(idPath.id);
FilePathIndex id = m_idCache.stringId(idPath.id);
ids.push_back(id);
......@@ -190,7 +192,7 @@ unittest_public:
}
void removeUnusedEntries(const WatcherEntries &entries,
const std::vector<uint> &ids)
const std::vector<FilePathIndex> &ids)
{
auto oldEntries = notAnymoreWatchedEntriesWithIds(entries, ids);
......@@ -272,7 +274,7 @@ unittest_public:
WatcherEntries notAnymoreWatchedEntriesWithIds(
const WatcherEntries &newEntries,
const std::vector<uint> &ids) const
const std::vector<FilePathIndex> &ids) const
{
auto oldEntries = notAnymoreWatchedEntries(newEntries, std::less<WatcherEntry>());
......@@ -328,7 +330,7 @@ unittest_public:
return m_watchedEntries;
}
WatcherEntries removeIdsFromWatchedEntries(const std::vector<uint> &ids)
WatcherEntries removeIdsFromWatchedEntries(const std::vector<FilePathIndex> &ids)
{
auto keep = [&] (const WatcherEntry &entry) {
......@@ -368,7 +370,7 @@ unittest_public:
WatcherEntries watchedEntriesForPaths(Utils::PathStringVector &&filePaths)
{
std::vector<uint> pathIds = m_pathCache.stringIds(filePaths);
std::vector<FilePathIndex> pathIds = m_pathCache.stringIds(filePaths);
WatcherEntries foundEntries;
......@@ -414,22 +416,22 @@ unittest_public:
}
}
StringCache<Utils::PathString> &pathCache()
FilePathCache<> &pathCache()
{
return m_pathCache;
}
StringCache<Utils::SmallString> &idCache()
IdCache &idCache()
{
return m_idCache;
}
private:
StringCache<Utils::SmallString> m_idCache;
IdCache m_idCache;
WatcherEntries m_watchedEntries;
ChangedFilePathCompressor<Timer> m_changedFilePathCompressor;
FileSystemWatcher m_fileSystemWatcher;
StringCache<Utils::PathString> &m_pathCache;
FilePathCache<> &m_pathCache;
ClangPathWatcherNotifier *m_notifier;
};
......
......@@ -38,10 +38,10 @@ namespace ClangBackEnd {
class CollectIncludesAction final : public clang::PreprocessOnlyAction
{
public:
CollectIncludesAction(std::vector<uint> &includeIds,
StringCache<Utils::PathString> &filePathCache,
std::vector<uint> &excludedIncludeUID,
std::vector<uint> &alreadyIncludedFileUIDs)
CollectIncludesAction(FilePathIndices &includeIds,
FilePathCache<> &filePathCache,
FilePathIndices &excludedIncludeUID,
FilePathIndices &alreadyIncludedFileUIDs)
: m_includeIds(includeIds),
m_filePathCache(filePathCache),
m_excludedIncludeUID(excludedIncludeUID),
......@@ -78,10 +78,10 @@ public:
}
private:
std::vector<uint> &m_includeIds;
StringCache<Utils::PathString> &m_filePathCache;
std::vector<uint> &m_excludedIncludeUID;
std::vector<uint> &m_alreadyIncludedFileUIDs;
FilePathIndices &m_includeIds;
FilePathCache<> &m_filePathCache;
FilePathIndices &m_excludedIncludeUID;
FilePathIndices &m_alreadyIncludedFileUIDs;
};
} // namespace ClangBackEnd
......@@ -47,10 +47,10 @@ class CollectIncludesPreprocessorCallbacks final : public clang::PPCallbacks
{
public:
CollectIncludesPreprocessorCallbacks(clang::HeaderSearch &headerSearch,
std::vector<uint> &includeIds,
StringCache<Utils::PathString> &filePathCache,
const std::vector<uint> &excludedIncludeUID,
std::vector<uint> &alreadyIncludedFileUIDs)
std::vector<FilePathIndex> &includeIds,
FilePathCache<> &filePathCache,
const std::vector<FilePathIndex> &excludedIncludeUID,
std::vector<FilePathIndex> &alreadyIncludedFileUIDs)
: m_headerSearch(headerSearch),
m_includeIds(includeIds),
m_filePathCache(filePathCache),
......@@ -78,7 +78,7 @@ public:
m_alreadyIncludedFileUIDs.insert(notAlreadyIncluded.second, fileUID);
Utils::PathString filePath = filePathFromFile(file);
if (!filePath.isEmpty()) {
uint includeId = m_filePathCache.stringId(filePath);
FilePathIndex includeId = m_filePathCache.stringId(filePath);
m_includeIds.emplace_back(includeId);
}
}
......@@ -132,7 +132,7 @@ public:
uid);
}
std::pair<bool, std::vector<uint>::iterator> isNotAlreadyIncluded(uint uid) const
std::pair<bool, std::vector<FilePathIndex>::iterator> isNotAlreadyIncluded(FilePathIndex uid) const
{
auto range = std::equal_range(m_alreadyIncludedFileUIDs.begin(),
m_alreadyIncludedFileUIDs.end(),
......@@ -174,10 +174,10 @@ public:
private:
clang::HeaderSearch &m_headerSearch;
std::vector<uint> &m_includeIds;
StringCache<Utils::PathString> &m_filePathCache;
const std::vector<uint> &m_excludedIncludeUID;
std::vector<uint> &m_alreadyIncludedFileUIDs;
std::vector<FilePathIndex> &m_includeIds;
FilePathCache<> &m_filePathCache;
const std::vector<FilePathIndex> &m_excludedIncludeUID;
std::vector<FilePathIndex> &m_alreadyIncludedFileUIDs;
bool m_skipInclude = false;
};
......
......@@ -36,8 +36,8 @@ namespace ClangBackEnd {
class CollectIncludesToolAction final : public clang::tooling::FrontendActionFactory
{
public:
CollectIncludesToolAction(std::vector<uint> &includeIds,
StringCache<Utils::PathString> &filePathCache,
CollectIncludesToolAction(std::vector<FilePathIndex> &includeIds,
FilePathCache<> &filePathCache,
const Utils::PathStringVector &excludedIncludes)
: m_includeIds(includeIds),
m_filePathCache(filePathCache),
......@@ -72,9 +72,9 @@ public:
m_alreadyIncludedFileUIDs);
}
std::vector<uint> generateExcludedIncludeFileUIDs(clang::FileManager &fileManager) const
std::vector<FilePathIndex> generateExcludedIncludeFileUIDs(clang::FileManager &fileManager) const
{
std::vector<uint> fileUIDs;
std::vector<FilePathIndex> fileUIDs;
fileUIDs.reserve(m_excludedIncludes.size());
for (const Utils::PathString &filePath : m_excludedIncludes) {
......@@ -90,10 +90,10 @@ public:
}
private:
std::vector<uint> m_alreadyIncludedFileUIDs;
std::vector<uint> m_excludedIncludeUIDs;
std::vector<uint> &m_includeIds;
StringCache<Utils::PathString> &m_filePathCache;
std::vector<FilePathIndex> m_alreadyIncludedFileUIDs;
std::vector<FilePathIndex> m_excludedIncludeUIDs;
std::vector<FilePathIndex> &m_includeIds;
FilePathCache<> &m_filePathCache;
const Utils::PathStringVector &m_excludedIncludes;
};
......
......@@ -29,13 +29,15 @@
#include <iosfwd>
#include <stringcachefwd.h>
namespace ClangBackEnd {
class IdPaths
{
public:
Utils::SmallString id;
std::vector<uint> paths;
std::vector<FilePathIndex> paths;
friend bool operator==(const IdPaths &first, const IdPaths &second)
{
......
......@@ -33,7 +33,7 @@
namespace ClangBackEnd {
IncludeCollector::IncludeCollector(StringCache<Utils::PathString> &filePathCache)
IncludeCollector::IncludeCollector(FilePathCache<> &filePathCache)
: m_filePathCache(filePathCache)
{
}
......@@ -69,7 +69,7 @@ void IncludeCollector::setExcludedIncludes(Utils::PathStringVector &&excludedInc
#endif
}
std::vector<uint> IncludeCollector::takeIncludeIds()
std::vector<FilePathIndex> IncludeCollector::takeIncludeIds()
{
std::sort(m_includeIds.begin(), m_includeIds.end());
......
......@@ -34,19 +34,19 @@ namespace ClangBackEnd {
class IncludeCollector : public ClangTool
{
public:
IncludeCollector(StringCache<Utils::PathString> &filePathCache);
IncludeCollector(FilePathCache<> &filePathCache);
void collectIncludes();
void setExcludedIncludes(Utils::PathStringVector &&excludedIncludes);
std::vector<uint> takeIncludeIds();
std::vector<FilePathIndex> takeIncludeIds();
private:
Utils::PathStringVector m_excludedIncludes;
std::vector<uint> m_includeIds;
std::vector<FilePathIndex> m_includeIds;
Utils::SmallStringVector m_directories;
StringCache<Utils::PathString> &m_filePathCache;
FilePathCache<> &m_filePathCache;
};
} // namespace ClangBackEnd
......@@ -37,7 +37,7 @@
namespace ClangBackEnd {
PchCreator::PchCreator(Environment &environment, StringCache<Utils::PathString> &filePathCache)
PchCreator::PchCreator(Environment &environment, FilePathCache<> &filePathCache)
: m_environment(environment),
m_filePathCache(filePathCache)
{
......@@ -45,7 +45,7 @@ PchCreator::PchCreator(Environment &environment, StringCache<Utils::PathString>
PchCreator::PchCreator(V2::ProjectPartContainers &&projectsParts,
Environment &environment,
StringCache<Utils::PathString> &filePathCache,
FilePathCache<> &filePathCache,
PchGeneratorInterface *pchGenerator,
V2::FileContainers &&generatedFiles)
: m_projectParts(std::move(projectsParts)),
......@@ -239,7 +239,7 @@ Utils::SmallStringVector PchCreator::generateGlobalClangCompilerArguments() cons
return compilerArguments;
}
std::vector<uint> PchCreator::generateGlobalPchIncludeIds() const
std::vector<FilePathIndex> PchCreator::generateGlobalPchIncludeIds() const
{
IncludeCollector collector(m_filePathCache);
......@@ -267,7 +267,7 @@ std::size_t contentSize(const std::vector<Utils::PathString> &includes)
}
Utils::SmallString PchCreator::generatePchIncludeFileContent(
const std::vector<uint> &includeIds) const
const std::vector<FilePathIndex> &includeIds) const
{
Utils::SmallString fileContent;
const std::size_t lineTemplateSize = 12;
......@@ -461,7 +461,7 @@ Utils::PathStringVector PchCreator::generateProjectPartHeaderAndSourcePaths(
return includeAndSources;
}
std::vector<uint> PchCreator::generateProjectPartPchIncludes(
std::vector<FilePathIndex> PchCreator::generateProjectPartPchIncludes(
const V2::ProjectPartContainer &projectPart) const
{
Utils::SmallString jointedFileContent = generateProjectPartHeaderAndSourcesContent(projectPart);
......
......@@ -48,10 +48,10 @@ class PchCreator final : public PchCreatorInterface
{
public:
PchCreator(Environment &environment,
StringCache<Utils::PathString> &filePathCache);
FilePathCache<> &filePathCache);
PchCreator(V2::ProjectPartContainers &&projectsParts,
Environment &environment,
StringCache<Utils::PathString> &filePathCache,
FilePathCache<> &filePathCache,
PchGeneratorInterface *pchGenerator,
V2::FileContainers &&generatedFiles);
......@@ -70,9 +70,9 @@ unittest_public:
Utils::SmallStringVector generateGlobalPchCompilerArguments() const;
Utils::SmallStringVector generateGlobalClangCompilerArguments() const;
std::vector<uint> generateGlobalPchIncludeIds() const;
std::vector<FilePathIndex> generateGlobalPchIncludeIds() const;
Utils::SmallString generatePchIncludeFileContent(const std::vector<uint> &includeIds) const;
Utils::SmallString generatePchIncludeFileContent(const std::vector<FilePathIndex> &includeIds) const;
Utils::SmallString generateGlobalPchHeaderFileContent() const;
std::unique_ptr<QFile> generateGlobalPchHeaderFile();
void generatePch(Utils::SmallStringVector &&commandLineArguments,
......@@ -97,7 +97,7 @@ unittest_public:
const V2::ProjectPartContainer &projectPart);