Commit 66802ef8 authored by Oswald Buddenhagen's avatar Oswald Buddenhagen
Browse files

implement simple VFS to support caching during project parsing



this tremendously speeds up loading of qt 5.1 based projects (including
qt itself) under mac os, as these look up the sdk dynamically, and use
caching to avoid doing that in every subproject.

Change-Id: I833253f81c3159056fab2ff888f293b36cc2ef56
Reviewed-by: default avatarDaniel Teske <daniel.teske@digia.com>
parent 57123478
......@@ -56,6 +56,7 @@
#include <utils/hostosinfo.h>
#include <utils/stringutils.h>
#include <proparser/prowriter.h>
#include <proparser/qmakevfs.h>
#include <algorithm>
#include <QDebug>
......@@ -1135,8 +1136,9 @@ void Qt4PriFileNode::changeFiles(const FileType fileType,
lines = contents.split(QLatin1Char('\n'));
}
QMakeVfs vfs;
QtSupport::ProMessageHandler handler;
QMakeParser parser(0, &handler);
QMakeParser parser(0, &vfs, &handler);
includeFile = parser.parsedProBlock(contents, m_projectFilePath, 1);
}
......
......@@ -46,6 +46,7 @@
#include <coreplugin/documentmanager.h>
#include <cpptools/cppmodelmanagerinterface.h>
#include <qmljstools/qmljsmodelmanager.h>
#include <projectexplorer/buildmanager.h>
#include <projectexplorer/buildtargetinfo.h>
#include <projectexplorer/deploymentdata.h>
#include <projectexplorer/toolchain.h>
......@@ -53,6 +54,7 @@
#include <projectexplorer/target.h>
#include <projectexplorer/projectexplorer.h>
#include <projectexplorer/projectmacroexpander.h>
#include <proparser/qmakevfs.h>
#include <qtsupport/profilereader.h>
#include <qtsupport/qtkitinformation.h>
......@@ -344,6 +346,7 @@ Qt4Project::Qt4Project(Qt4Manager *manager, const QString& fileName) :
m_nodesWatcher(new Internal::Qt4NodesWatcher(this)),
m_fileInfo(new Qt4ProjectFile(fileName, this)),
m_projectFiles(new Qt4ProjectFiles),
m_qmakeVfs(new QMakeVfs),
m_qmakeGlobals(0),
m_asyncUpdateFutureInterface(0),
m_pendingEvaluateFuturesCount(0),
......@@ -358,6 +361,10 @@ Qt4Project::Qt4Project(Qt4Manager *manager, const QString& fileName) :
m_asyncUpdateTimer.setSingleShot(true);
m_asyncUpdateTimer.setInterval(3000);
connect(&m_asyncUpdateTimer, SIGNAL(timeout()), this, SLOT(asyncUpdate()));
connect(ProjectExplorerPlugin::instance()->buildManager(),
SIGNAL(buildQueueFinished(bool)),
SLOT(buildFinished(bool)));
}
Qt4Project::~Qt4Project()
......@@ -365,6 +372,7 @@ Qt4Project::~Qt4Project()
m_codeModelFuture.cancel();
m_asyncUpdateState = ShuttingDown;
m_manager->unregisterProject(this);
delete m_qmakeVfs;
delete m_projectFiles;
m_cancelEvaluate = true;
// Deleting the root node triggers a few things, make sure rootProjectNode
......@@ -847,6 +855,9 @@ void Qt4Project::asyncUpdate()
{
if (debug)
qDebug()<<"async update, timer expired, doing now";
m_qmakeVfs->invalidateCache();
Q_ASSERT(!m_asyncUpdateFutureInterface);
m_asyncUpdateFutureInterface = new QFutureInterface<void>();
......@@ -877,6 +888,12 @@ void Qt4Project::asyncUpdate()
m_asyncUpdateState = AsyncUpdateInProgress;
}
void Qt4Project::buildFinished(bool success)
{
if (success)
m_qmakeVfs->invalidateContents();
}
ProjectExplorer::IProjectManager *Qt4Project::projectManager() const
{
return m_manager;
......@@ -1002,7 +1019,7 @@ QtSupport::ProFileReader *Qt4Project::createProFileReader(const Qt4ProFileNode *
}
++m_qmakeGlobalsRefCnt;
QtSupport::ProFileReader *reader = new QtSupport::ProFileReader(m_qmakeGlobals);
QtSupport::ProFileReader *reader = new QtSupport::ProFileReader(m_qmakeGlobals, m_qmakeVfs);
reader->setOutputDir(qt4ProFileNode->buildDir());
......
......@@ -42,6 +42,7 @@
QT_BEGIN_NAMESPACE
class ProFileGlobals;
class QMakeVfs;
QT_END_NAMESPACE
namespace ProjectExplorer { class DeploymentData; }
......@@ -158,6 +159,7 @@ protected:
private slots:
void asyncUpdate();
void buildFinished(bool success);
void activeTargetWasChanged();
......@@ -198,6 +200,8 @@ private:
// cached lists of all of files
Internal::Qt4ProjectFiles *m_projectFiles;
QMakeVfs *m_qmakeVfs;
// cached data during project rescan
ProFileGlobals *m_qmakeGlobals;
int m_qmakeGlobalsRefCnt;
......
......@@ -36,6 +36,7 @@
#include "qtversionmanager.h"
#include "profilereader.h"
#include <proparser/qmakevfs.h>
#include <projectexplorer/toolchainmanager.h>
#include <projectexplorer/toolchain.h>
#include <projectexplorer/projectexplorer.h>
......@@ -780,12 +781,13 @@ void BaseQtVersion::ensureMkSpecParsed() const
if (mkspecPath().isEmpty())
return;
QMakeVfs vfs;
ProFileGlobals option;
option.setProperties(versionInfo());
ProMessageHandler msgHandler(true);
ProFileCacheManager::instance()->incRefCount();
QMakeParser parser(ProFileCacheManager::instance()->cache(), &msgHandler);
ProFileEvaluator evaluator(&option, &parser, &msgHandler);
QMakeParser parser(ProFileCacheManager::instance()->cache(), &vfs, &msgHandler);
ProFileEvaluator evaluator(&option, &parser, &vfs, &msgHandler);
evaluator.loadNamedSpec(mkspecPath().toString(), false);
parseMkSpec(&evaluator);
......
......@@ -65,9 +65,9 @@ void ProMessageHandler::fileMessage(const QString &)
}
ProFileReader::ProFileReader(ProFileGlobals *option)
: QMakeParser(ProFileCacheManager::instance()->cache(), this)
, ProFileEvaluator(option, this, this)
ProFileReader::ProFileReader(ProFileGlobals *option, QMakeVfs *vfs)
: QMakeParser(ProFileCacheManager::instance()->cache(), vfs, this)
, ProFileEvaluator(option, this, vfs, this)
, m_ignoreLevel(0)
{
}
......
......@@ -69,7 +69,7 @@ class QTSUPPORT_EXPORT ProFileReader : public ProMessageHandler, public QMakePar
Q_OBJECT
public:
ProFileReader(ProFileGlobals *option);
ProFileReader(ProFileGlobals *option, QMakeVfs *vfs);
~ProFileReader();
QList<ProFile*> includeFiles() const;
......
......@@ -31,6 +31,8 @@
#include "profilereader.h"
#include "baseqtversion.h"
#include <proparser/qmakevfs.h>
#include <extensionsystem/pluginmanager.h>
#include <utils/environment.h>
......@@ -61,12 +63,13 @@ BaseQtVersion *QtVersionFactory::createQtVersionFromQMakePath(const Utils::FileN
return 0;
Utils::FileName mkspec = BaseQtVersion::mkspecFromVersionInfo(versionInfo);
QMakeVfs vfs;
ProFileGlobals globals;
globals.setProperties(versionInfo);
ProMessageHandler msgHandler(true);
ProFileCacheManager::instance()->incRefCount();
QMakeParser parser(ProFileCacheManager::instance()->cache(), &msgHandler);
ProFileEvaluator evaluator(&globals, &parser, &msgHandler);
QMakeParser parser(ProFileCacheManager::instance()->cache(), &vfs, &msgHandler);
ProFileEvaluator evaluator(&globals, &parser, &vfs, &msgHandler);
evaluator.loadNamedSpec(mkspec.toString(), false);
QList<QtVersionFactory *> factories = ExtensionSystem::PluginManager::getObjects<QtVersionFactory>();
......
......@@ -43,9 +43,9 @@ void ProFileEvaluator::initialize()
QMakeEvaluator::initStatics();
}
ProFileEvaluator::ProFileEvaluator(ProFileGlobals *option, QMakeParser *parser,
ProFileEvaluator::ProFileEvaluator(ProFileGlobals *option, QMakeParser *parser, QMakeVfs *vfs,
QMakeHandler *handler)
: d(new QMakeEvaluator(option, parser, handler))
: d(new QMakeEvaluator(option, parser, vfs, handler))
{
}
......
......@@ -40,6 +40,7 @@
QT_BEGIN_NAMESPACE
class QMakeVfs;
class QMakeParser;
class QMakeEvaluator;
class QMakeHandler;
......@@ -65,7 +66,8 @@ public:
// Call this from a concurrency-free context
static void initialize();
ProFileEvaluator(ProFileGlobals *option, QMakeParser *parser, QMakeHandler *handler);
ProFileEvaluator(ProFileGlobals *option, QMakeParser *parser, QMakeVfs *vfs,
QMakeHandler *handler);
~ProFileEvaluator();
ProFileEvaluator::TemplateType templateType() const;
......
......@@ -14,6 +14,7 @@ HEADERS += \
profileevaluator.h \
proitems.h \
prowriter.h \
qmakevfs.h \
ioutils.h
SOURCES += \
......@@ -24,6 +25,7 @@ SOURCES += \
qmakebuiltins.cpp \
proitems.cpp \
prowriter.cpp \
qmakevfs.cpp \
ioutils.cpp
RESOURCES += proparser.qrc
......
......@@ -32,6 +32,7 @@
#include "qmakeevaluator_p.h"
#include "qmakeglobals.h"
#include "qmakeparser.h"
#include "qmakevfs.h"
#include "ioutils.h"
#include <qbytearray.h>
......@@ -269,41 +270,12 @@ quoteValue(const ProString &val)
return ret;
}
static bool
doWriteFile(const QString &name, QIODevice::OpenMode mode, const QString &contents, QString *errStr)
{
QByteArray bytes = contents.toLocal8Bit();
QFile cfile(name);
if (!(mode & QIODevice::Append) && cfile.open(QIODevice::ReadOnly | QIODevice::Text)) {
if (cfile.readAll() == bytes)
return true;
cfile.close();
}
if (!cfile.open(mode | QIODevice::WriteOnly | QIODevice::Text)) {
*errStr = cfile.errorString();
return false;
}
cfile.write(bytes);
cfile.close();
if (cfile.error() != QFile::NoError) {
*errStr = cfile.errorString();
return false;
}
return true;
}
QMakeEvaluator::VisitReturn
QMakeEvaluator::writeFile(const QString &ctx, const QString &fn, QIODevice::OpenMode mode,
const QString &contents)
{
QFileInfo qfi(fn);
if (!QDir::current().mkpath(qfi.path())) {
evalError(fL1S("Cannot create %1directory %2.")
.arg(ctx, QDir::toNativeSeparators(qfi.path())));
return ReturnFalse;
}
QString errStr;
if (!doWriteFile(fn, mode, contents, &errStr)) {
if (!m_vfs->writeFile(fn, mode, contents, &errStr)) {
evalError(fL1S("Cannot write %1file %2: %3.")
.arg(ctx, QDir::toNativeSeparators(fn), errStr));
return ReturnFalse;
......@@ -1413,6 +1385,9 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::evaluateBuiltinConditional(
}
const QString &file = resolvePath(m_option->expandEnvVars(args.at(0).toQString(m_tmp1)));
// Don't use VFS here:
// - it supports neither listing nor even directories
// - it's unlikely that somebody would test for files they created themselves
if (IoUtils::exists(file))
return ReturnTrue;
int slsh = file.lastIndexOf(QLatin1Char('/'));
......@@ -1444,7 +1419,6 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::evaluateBuiltinConditional(
evalError(fL1S("write_file(name, [content var, [append]]) requires one to three arguments."));
return ReturnFalse;
}
#ifdef PROEVALUATOR_FULL
QIODevice::OpenMode mode = QIODevice::Truncate;
QString contents;
if (args.count() >= 2) {
......@@ -1456,9 +1430,6 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::evaluateBuiltinConditional(
mode = QIODevice::Append;
}
return writeFile(QString(), resolvePath(args.at(0).toQString(m_tmp1)), mode, contents);
#else
return ReturnTrue;
#endif
}
case T_TOUCH: {
if (args.count() != 2) {
......@@ -1510,7 +1481,6 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::evaluateBuiltinConditional(
evalError(fL1S("cache(var, [set|add|sub] [transient] [super], [srcvar]) requires one to three arguments."));
return ReturnFalse;
}
#ifdef PROEVALUATOR_FULL
bool persist = true;
bool super = false;
enum { CacheSet, CacheAdd, CacheSub } mode = CacheSet;
......@@ -1636,9 +1606,6 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::evaluateBuiltinConditional(
fn = m_cachefile;
}
return writeFile(fL1S("cache "), fn, QIODevice::Append, varstr);
#else
return ReturnTrue;
#endif
}
default:
evalError(fL1S("Function '%1' is not implemented.").arg(function.toQString(m_tmp1)));
......
......@@ -32,6 +32,7 @@
#include "qmakeglobals.h"
#include "qmakeparser.h"
#include "qmakevfs.h"
#include "ioutils.h"
#include <qbytearray.h>
......@@ -162,13 +163,13 @@ const ProKey &QMakeEvaluator::map(const ProKey &var)
}
QMakeEvaluator::QMakeEvaluator(QMakeGlobals *option,
QMakeParser *parser, QMakeHandler *handler)
QMakeEvaluator::QMakeEvaluator(QMakeGlobals *option, QMakeParser *parser, QMakeVfs *vfs,
QMakeHandler *handler)
:
#ifdef PROEVALUATOR_DEBUG
m_debugLevel(option->debugLevel),
#endif
m_option(option), m_parser(parser), m_handler(handler)
m_option(option), m_parser(parser), m_handler(handler), m_vfs(vfs)
{
// So that single-threaded apps don't have to call initialize() for now.
initStatics();
......@@ -1049,7 +1050,7 @@ bool QMakeEvaluator::prepareProject(const QString &inDir)
superdir = m_outputDir;
forever {
QString superfile = superdir + QLatin1String("/.qmake.super");
if (IoUtils::exists(superfile)) {
if (m_vfs->exists(superfile)) {
m_superfile = QDir::cleanPath(superfile);
break;
}
......@@ -1064,10 +1065,10 @@ bool QMakeEvaluator::prepareProject(const QString &inDir)
QString dir = m_outputDir;
forever {
conffile = sdir + QLatin1String("/.qmake.conf");
if (!IoUtils::exists(conffile))
if (!m_vfs->exists(conffile))
conffile.clear();
cachefile = dir + QLatin1String("/.qmake.cache");
if (!IoUtils::exists(cachefile))
if (!m_vfs->exists(cachefile))
cachefile.clear();
if (!conffile.isEmpty() || !cachefile.isEmpty()) {
if (dir != sdir)
......@@ -1158,7 +1159,7 @@ bool QMakeEvaluator::loadSpec()
m_hostBuild ? m_option->qmakespec : m_option->xqmakespec);
{
QMakeEvaluator evaluator(m_option, m_parser, m_handler);
QMakeEvaluator evaluator(m_option, m_parser, m_vfs, m_handler);
if (!m_superfile.isEmpty()) {
valuesRef(ProKey("_QMAKE_SUPER_CACHE_")) << ProString(m_superfile);
if (evaluator.evaluateFile(
......@@ -1315,7 +1316,7 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::visitProFile(
locker.unlock();
#endif
QMakeEvaluator *baseEval = new QMakeEvaluator(m_option, m_parser, m_handler);
QMakeEvaluator *baseEval = new QMakeEvaluator(m_option, m_parser, m_vfs, m_handler);
baseEnv->evaluator = baseEval;
baseEval->m_superfile = m_superfile;
baseEval->m_conffile = m_conffile;
......@@ -1810,7 +1811,7 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::evaluateFile(
#endif
return ok;
} else {
if (!(flags & LoadSilent) && !IoUtils::exists(fileName))
if (!(flags & LoadSilent) && !m_vfs->exists(fileName))
evalError(fL1S("WARNING: Include file %1 not found").arg(fileName));
return ReturnFalse;
}
......@@ -1893,7 +1894,7 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::evaluateFeatureFile(
QMakeEvaluator::VisitReturn QMakeEvaluator::evaluateFileInto(
const QString &fileName, ProValueMap *values, LoadFlags flags)
{
QMakeEvaluator visitor(m_option, m_parser, m_handler);
QMakeEvaluator visitor(m_option, m_parser, m_vfs, m_handler);
visitor.m_caller = this;
visitor.m_outputDir = m_outputDir;
visitor.m_featureRoots = m_featureRoots;
......
......@@ -96,7 +96,7 @@ public:
static void initStatics();
static void initFunctionStatics();
QMakeEvaluator(QMakeGlobals *option, QMakeParser *parser,
QMakeEvaluator(QMakeGlobals *option, QMakeParser *parser, QMakeVfs *vfs,
QMakeHandler *handler);
~QMakeEvaluator();
......@@ -282,6 +282,7 @@ public:
QMakeGlobals *m_option;
QMakeParser *m_parser;
QMakeHandler *m_handler;
QMakeVfs *m_vfs;
};
Q_DECLARE_OPERATORS_FOR_FLAGS(QMakeEvaluator::LoadFlags)
......
......@@ -29,6 +29,7 @@
#include "qmakeparser.h"
#include "qmakevfs.h"
#include "ioutils.h"
using namespace QMakeInternal;
......@@ -130,9 +131,10 @@ void QMakeParser::initialize()
statics.strLITERAL_WHITESPACE = QLatin1String("LITERAL_WHITESPACE");
}
QMakeParser::QMakeParser(ProFileCache *cache, QMakeParserHandler *handler)
QMakeParser::QMakeParser(ProFileCache *cache, QMakeVfs *vfs, QMakeParserHandler *handler)
: m_cache(cache)
, m_handler(handler)
, m_vfs(vfs)
{
// So that single-threaded apps don't have to call initialize() for now.
initialize();
......@@ -218,24 +220,14 @@ void QMakeParser::discardFileFromCache(const QString &fileName)
bool QMakeParser::read(ProFile *pro)
{
QFile file(pro->fileName());
if (!file.open(QIODevice::ReadOnly)) {
if (m_handler && IoUtils::exists(pro->fileName()))
QString content;
QString errStr;
if (!m_vfs->readFile(pro->fileName(), &content, &errStr)) {
if (m_handler && m_vfs->exists(pro->fileName()))
m_handler->message(QMakeParserHandler::ParserIoError,
fL1S("Cannot read %1: %2").arg(pro->fileName(), file.errorString()));
fL1S("Cannot read %1: %2").arg(pro->fileName(), errStr));
return false;
}
QByteArray bcont = file.readAll();
if (bcont.startsWith("\xef\xbb\xbf")) {
// UTF-8 BOM will cause subtle errors
m_handler->message(QMakeParserHandler::ParserIoError,
fL1S("Unexpected UTF-8 BOM in %1").arg(pro->fileName()));
return false;
}
QString content(QString::fromLocal8Bit(bcont));
bcont.clear();
file.close();
return read(pro, content, 1, FullGrammar);
}
......
......@@ -67,6 +67,7 @@ public:
};
class ProFileCache;
class QMakeVfs;
class QMAKE_EXPORT QMakeParser
{
......@@ -74,7 +75,7 @@ public:
// Call this from a concurrency-free context
static void initialize();
QMakeParser(ProFileCache *cache, QMakeParserHandler *handler);
QMakeParser(ProFileCache *cache, QMakeVfs *vfs, QMakeParserHandler *handler);
enum SubGrammar { FullGrammar, TestGrammar, ValueGrammar };
// fileName is expected to be absolute and cleanPath()ed.
......@@ -163,6 +164,7 @@ private:
ProFileCache *m_cache;
QMakeParserHandler *m_handler;
QMakeVfs *m_vfs;
// This doesn't help gcc 3.3 ...
template<typename T> friend class QTypeInfo;
......
/****************************************************************************
**
** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
** Contact: http://www.qt-project.org/legal
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Digia. For licensing terms and
** conditions see http://qt.digia.com/licensing. For further information
** use the contact form at http://qt.digia.com/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Digia gives you certain additional
** rights. These rights are described in the Digia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
****************************************************************************/
#include "qmakevfs.h"
#include "ioutils.h"
using namespace QMakeInternal;
#include <qdir.h>
#include <qfile.h>
#include <qfileinfo.h>
#define fL1S(s) QString::fromLatin1(s)
QMakeVfs::QMakeVfs()
#ifndef PROEVALUATOR_FULL
: m_magicMissing(fL1S("missing"))
, m_magicExisting(fL1S("existing"))
#endif
{
}
bool QMakeVfs::writeFile(const QString &fn, QIODevice::OpenMode mode, const QString &contents,
QString *errStr)
{
#ifndef PROEVALUATOR_FULL
# ifdef PROEVALUATOR_THREAD_SAFE
QMutexLocker locker(&m_mutex);
# endif
QString *cont = &m_files[fn];
if (mode & QIODevice::Append)
*cont += contents;
else
*cont = contents;
Q_UNUSED(errStr)
return true;
#else
QFileInfo qfi(fn);
if (!QDir::current().mkpath(qfi.path())) {
*errStr = fL1S("Cannot create parent directory");
return false;
}
QByteArray bytes = contents.toLocal8Bit();
QFile cfile(fn);
if (!(mode & QIODevice::Append) && cfile.open(QIODevice::ReadOnly | QIODevice::Text)) {
if (cfile.readAll() == bytes)
return true;
cfile.close();
}
if (!cfile.open(mode | QIODevice::WriteOnly | QIODevice::Text)) {
*errStr = cfile.errorString();
return false;
}
cfile.write(bytes);
cfile.close();
if (cfile.error() != QFile::NoError) {
*errStr = cfile.errorString();
return false;
}
return true;
#endif
}
bool QMakeVfs::readFile(const QString &fn, QString *contents, QString *errStr)
{
#ifndef PROEVALUATOR_FULL
# ifdef PROEVALUATOR_THREAD_SAFE
QMutexLocker locker(&m_mutex);
# endif
QHash<QString, QString>::ConstIterator it = m_files.constFind(fn);
if (it != m_files.constEnd()) {
if (it->constData() == m_magicMissing.constData()) {
*errStr = fL1S("No such file or directory");
return false;
}
if (it->constData() != m_magicExisting.constData()) {
*contents = *it;