Commit d2e15e5f authored by Marco Bubke's avatar Marco Bubke Committed by Tim Jenssen

Clang: Add file cache

The database is using file path integer ids to handle file paths because
otherwise we would save many redundant data. This patch is improving it
further with the introduction of a database based file path cache. The
entries are now divided in a directory path and file name. This is quite
handy for directory based file watching.

Change-Id: I03f2e388e43f3d521d6bf8e39dfb95eb2309dc73
Reviewed-by: Tim Jenssen's avatarTim Jenssen <tim.jenssen@qt.io>
parent 0be82400
......@@ -67,7 +67,6 @@ SOURCES += \
$$PWD/requestsourcelocationforrenamingmessage.cpp \
$$PWD/requestsourcerangesanddiagnosticsforquerymessage.cpp \
$$PWD/requestsourcerangesforquerymessage.cpp \
$$PWD/sourcefilepathcontainerbase.cpp \
$$PWD/sourcelocationcontainer.cpp \
$$PWD/sourcelocationcontainerv2.cpp \
$$PWD/sourcelocationscontainer.cpp \
......@@ -82,7 +81,9 @@ SOURCES += \
$$PWD/updatepchprojectpartsmessage.cpp \
$$PWD/updatetranslationunitsforeditormessage.cpp \
$$PWD/updatevisibletranslationunitsmessage.cpp \
$$PWD/writemessageblock.cpp
$$PWD/writemessageblock.cpp \
$$PWD/filepathcaching.cpp \
$$PWD/filepathid.cpp
HEADERS += \
$$PWD/cancelmessage.h \
......@@ -150,7 +151,6 @@ HEADERS += \
$$PWD/requestsourcelocationforrenamingmessage.h \
$$PWD/requestsourcerangesanddiagnosticsforquerymessage.h \
$$PWD/requestsourcerangesforquerymessage.h \
$$PWD/sourcefilepathcontainerbase.h \
$$PWD/sourcelocationcontainer.h \
$$PWD/sourcelocationcontainerv2.h \
$$PWD/sourcelocationscontainer.h \
......@@ -172,6 +172,15 @@ HEADERS += \
$$PWD/stringcachefwd.h \
$$PWD/stringcachealgorithms.h \
$$PWD/projectmanagementserverinterface.h \
$$PWD/refactoringdatabaseinitializer.h
$$PWD/refactoringdatabaseinitializer.h \
$$PWD/filepathcache.h \
$$PWD/filepathid.h \
$$PWD/filepathstorage.h \
$$PWD/filepathstoragesqlitestatementfactory.h \
$$PWD/filepathstoragesources.h \
$$PWD/filepathexceptions.h \
$$PWD/filepathcachinginterface.h \
$$PWD/filepathcaching.h \
$$PWD/filepathcachingfwd.h
contains(QT_CONFIG, reduce_exports):CONFIG += hide_symbols
......@@ -94,9 +94,7 @@ public:
DynamicASTMatcherDiagnosticMessageContainer clone() const
{
return DynamicASTMatcherDiagnosticMessageContainer(m_sourceRange.clone(),
m_errorType,
m_arguments.clone());
return *this;
}
private:
......
......@@ -63,22 +63,27 @@ public:
{
}
FilePath(const Utils::PathString &directory, const Utils::PathString &name)
: m_path({std::move(directory), "/", std::move(name)}),
FilePath(Utils::SmallStringView directory, Utils::SmallStringView name)
: m_path({directory, "/", name}),
m_slashIndex(directory.size())
{}
Utils::SmallStringView directory() const
Utils::SmallStringView directory() const noexcept
{
return m_path.mid(0, m_slashIndex);
}
Utils::SmallStringView name() const
Utils::SmallStringView name() const noexcept
{
return m_path.mid(m_slashIndex + 1, m_path.size() - m_slashIndex - 1);
}
const Utils::PathString &path() const
const Utils::PathString &path() const noexcept
{
return m_path;
}
operator Utils::PathString() const noexcept
{
return m_path;
}
......@@ -105,9 +110,7 @@ public:
friend std::ostream &operator<<(std::ostream &out, const FilePath &filePath)
{
out << filePath.directory() << "/" << filePath.name();
return out;
return out << "(" << filePath.path() << ", " << filePath.slashIndex() << ")";
}
friend bool operator==(const FilePath &first, const FilePath &second)
......@@ -115,6 +118,16 @@ public:
return first.m_path == second.m_path;
}
friend bool operator==(const FilePath &first, const Utils::SmallStringView &second)
{
return first.path() == second;
}
friend bool operator==(const Utils::SmallStringView &first, const FilePath&second)
{
return second == first;
}
friend bool operator<(const FilePath &first, const FilePath &second)
{
return first.m_path < second.m_path;
......@@ -125,11 +138,18 @@ public:
return *this;
}
std::size_t slashIndex() const
{
return m_slashIndex;
}
private:
Utils::PathString m_path = "/";
std::size_t m_slashIndex = 0;
};
using FilePaths = std::vector<FilePath>;
CMBIPC_EXPORT QDebug operator<<(QDebug debug, const FilePath &filePath);
} // namespace ClangBackEnd
/****************************************************************************
**
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "filepathexceptions.h"
#include "filepathid.h"
#include "filepath.h"
#include "stringcache.h"
#include <utils/smallstringview.h>
#include <algorithm>
namespace ClangBackEnd {
class FilePathCacheBase
{
public:
static
std::ptrdiff_t lastSlashIndex(Utils::SmallStringView filePath)
{
auto foundReverse = std::find(filePath.rbegin(), filePath.rend(), '/');
auto found = foundReverse.base();
--found;
return std::distance(filePath.begin(), found);
}
static
Utils::SmallStringView directoryPath(Utils::SmallStringView filePath, std::ptrdiff_t slashIndex)
{
return {filePath.data(), std::size_t(std::max(std::ptrdiff_t(0), slashIndex))};
}
static
Utils::SmallStringView fileName(Utils::SmallStringView filePath, std::ptrdiff_t slashIndex)
{
return {filePath.data() + slashIndex + 1, filePath.size() - std::size_t(slashIndex) - 1};
}
};
template <typename FilePathStorage>
class FilePathCache final : private FilePathCacheBase
{
using DirectoryPathCache = StringCache<Utils::PathString,
int,
std::mutex,
decltype(&Utils::reverseCompare),
Utils::reverseCompare>;
using FileNameCache = StringCache<Utils::SmallString,
int,
std::mutex,
decltype(&Utils::compare),
Utils::compare>;
public:
FilePathCache(FilePathStorage &filePathStorage)
: m_filePathStorage(filePathStorage)
{}
FilePathId filePathId(Utils::SmallStringView filePath) const
{
std::ptrdiff_t slashIndex = lastSlashIndex(filePath);
Utils::SmallStringView directoryPath = this->directoryPath(filePath, slashIndex);
int directoryId = m_directyPathCache.stringId(directoryPath,
[&] (const Utils::SmallStringView) {
return m_filePathStorage.fetchDirectoryId(directoryPath);
});
Utils::SmallStringView fileName = this->fileName(filePath, slashIndex);
int fileNameId = m_fileNameCache.stringId(fileName,
[&] (const Utils::SmallStringView) {
return m_filePathStorage.fetchSourceId(directoryId, fileName);
});
return {directoryId, fileNameId};
}
FilePath filePath(FilePathId filePathId) const
{
if (Q_UNLIKELY(!filePathId.isValid()))
throw NoFilePathForInvalidFilePathId();
auto fetchFilePath = [&] (int id) { return m_filePathStorage.fetchDirectoryPath(id); };
Utils::PathString directoryPath = m_directyPathCache.string(filePathId.directoryId,
fetchFilePath);
auto fetchSoureName = [&] (int id) { return m_filePathStorage.fetchSourceName(id); };
Utils::SmallString fileName = m_fileNameCache.string(filePathId.fileNameId,
fetchSoureName);
return {directoryPath, fileName};
}
private:
mutable DirectoryPathCache m_directyPathCache;
mutable FileNameCache m_fileNameCache;
FilePathStorage &m_filePathStorage;
};
} // namespace ClangBackEnd
......@@ -23,20 +23,18 @@
**
****************************************************************************/
#include "sourcelocationentry.h"
#include <utils/smallstringio.h>
#include "filepathcaching.h"
namespace ClangBackEnd {
std::ostream &operator<<(std::ostream &out, const SourceLocationEntry &entry)
FilePathId FilePathCaching::filePathId(Utils::SmallStringView filePath) const
{
out << "("
<< entry.fileId << ", "
<< entry.line << ", "
<< entry.column << ")";
return m_cache.filePathId(filePath);
}
return out;
FilePath FilePathCaching::filePath(FilePathId filePathId) const
{
return m_cache.filePath(filePathId);
}
} // namespace ClangBackEnd
/****************************************************************************
**
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "clangsupport_global.h"
#include "filepathcachinginterface.h"
#include "filepathcache.h"
#include "filepathstoragesqlitestatementfactory.h"
#include "filepathstorage.h"
#include <sqlitedatabase.h>
#include <sqlitereadstatement.h>
#include <sqlitewritestatement.h>
namespace ClangBackEnd {
class CMBIPC_EXPORT FilePathCaching final : public FilePathCachingInterface
{
using Factory = FilePathStorageSqliteStatementFactory<Sqlite::Database,
Sqlite::ReadStatement,
Sqlite::WriteStatement>;
using Storage = FilePathStorage<Factory>;
using Cache = FilePathCache<Storage>;
public:
FilePathCaching(Sqlite::Database &database)
: m_factory(database)
{}
FilePathId filePathId(Utils::SmallStringView filePath) const override;
FilePath filePath(FilePathId filePathId) const override;
private:
Factory m_factory;
Storage m_storage{m_factory};
Cache m_cache{m_storage};
};
} // namespace ClangBackEnd
/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
......@@ -23,10 +23,10 @@
**
****************************************************************************/
#include "sourcefilepathcontainerbase.h"
#pragma once
namespace ClangBackEnd {
class FilePathCachingInterface;
} // namespace ClangBackEnd
/****************************************************************************
**
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "filepath.h"
#include "filepathid.h"
namespace ClangBackEnd {
class FilePathCachingInterface
{
public:
virtual FilePathId filePathId(Utils::SmallStringView filePath) const = 0;
virtual FilePath filePath(FilePathId filePathId) const = 0;
template <typename Container>
FilePathIds filePathIds(Container &&filePaths) const
{
FilePathIds filePathIds;
filePathIds.reserve(filePaths.size());
std::transform(filePaths.begin(),
filePaths.end(),
std::back_inserter(filePathIds),
[&] (const auto &filePath) { return this->filePathId(filePath); });
return filePathIds;
}
template <typename Element>
FilePathIds filePathIds(std::initializer_list<Element> filePaths) const
{
return filePathIds(std::vector<Element>(filePaths));
}
FilePaths filePaths(const FilePathIds &filePathIds) const
{
FilePaths filePaths;
filePaths.reserve(filePathIds.size());
std::transform(filePathIds.begin(),
filePathIds.end(),
std::back_inserter(filePaths),
[&] (auto filePathId) { return this->filePath(filePathId); });
return filePaths;
}
};
} // namespace ClangBackEnd
/****************************************************************************
**
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include <exception>
namespace ClangBackEnd {
class NoFilePathForInvalidFilePathId : std::exception
{
public:
const char *what() const noexcept override
{
return "You cannot get a file path for an invalid file path id!";
}
};
class SourceNameIdDoesNotExists : std::exception
{
public:
const char *what() const noexcept override
{
return "The source name id does not exists in the database!";
}
};
class DirectoryPathIdDoesNotExists : std::exception
{
public:
const char *what() const noexcept override
{
return "The directory path id does not exists in the database!";
}
};
} // namespace ClangBackEnd
/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
......@@ -23,19 +23,17 @@
**
****************************************************************************/
#include "idpaths.h"
#include "filepathid.h"
#include <utils/smallstringio.h>
#include <QDebug>
namespace ClangBackEnd {
std::ostream &operator<<(std::ostream &out, const IdPaths &idPaths)
QDebug operator<<(QDebug debug, const FilePathId &filePathId)
{
out << "("
<< idPaths.id << ", "
<< idPaths.paths << ")";
debug.nospace() << "(" << filePathId.directoryId << ", " << filePathId.fileNameId << ")";
return out;
return debug;
}
} // namespace ClangBackEnd
/****************************************************************************
**
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "clangsupport_global.h"
#include <QDataStream>
#include <cstdint>
#include <tuple>
#include <vector>
namespace ClangBackEnd {
class FilePathId
{
public: