Commit e8c0c789 authored by con's avatar con
Browse files

Use iterator instead of collecting all the files to search beforehand.

Especially for the file system filter, it's better not to block while
collecting all the file names by first iterating over the file system.
Now, the filters return an iterator, and the search thread takes a file
from it, searches it, and takes the next file.

This also unifies the file iterator for the custom locator filters and
the find in files on file system search filter.

Task-number: QTCREATORBUG-1690
parent 4214193d
...@@ -70,11 +70,11 @@ namespace { ...@@ -70,11 +70,11 @@ namespace {
void runFileSearch(QFutureInterface<FileSearchResult> &future, void runFileSearch(QFutureInterface<FileSearchResult> &future,
QString searchTerm, QString searchTerm,
QStringList files, FileIterator *files,
QTextDocument::FindFlags flags, QTextDocument::FindFlags flags,
QMap<QString, QString> fileToContentsMap) QMap<QString, QString> fileToContentsMap)
{ {
future.setProgressRange(0, files.size()); future.setProgressRange(0, files->maxProgress());
int numFilesSearched = 0; int numFilesSearched = 0;
int numMatches = 0; int numMatches = 0;
...@@ -95,11 +95,13 @@ void runFileSearch(QFutureInterface<FileSearchResult> &future, ...@@ -95,11 +95,13 @@ void runFileSearch(QFutureInterface<FileSearchResult> &future,
QFile file; QFile file;
QBuffer buffer; QBuffer buffer;
foreach (const QString &s, files) { while (files->hasNext()) {
const QString &s = files->next();
future.setProgressRange(0, files->maxProgress());
if (future.isPaused()) if (future.isPaused())
future.waitForResume(); future.waitForResume();
if (future.isCanceled()) { if (future.isCanceled()) {
future.setProgressValueAndText(numFilesSearched, msgCanceled(searchTerm, numMatches, numFilesSearched)); future.setProgressValueAndText(files->currentProgress(), msgCanceled(searchTerm, numMatches, numFilesSearched));
break; break;
} }
QIODevice *device; QIODevice *device;
...@@ -180,20 +182,22 @@ void runFileSearch(QFutureInterface<FileSearchResult> &future, ...@@ -180,20 +182,22 @@ void runFileSearch(QFutureInterface<FileSearchResult> &future,
firstChunk = false; firstChunk = false;
} }
++numFilesSearched; ++numFilesSearched;
future.setProgressValueAndText(numFilesSearched, msgFound(searchTerm, numMatches, numFilesSearched, files.size())); if (future.isProgressUpdateNeeded())
future.setProgressValueAndText(files->currentProgress(), msgFound(searchTerm, numMatches, numFilesSearched));
device->close(); device->close();
} }
if (!future.isCanceled()) if (!future.isCanceled())
future.setProgressValueAndText(numFilesSearched, msgFound(searchTerm, numMatches, numFilesSearched)); future.setProgressValueAndText(files->currentProgress(), msgFound(searchTerm, numMatches, numFilesSearched));
delete files;
} }
void runFileSearchRegExp(QFutureInterface<FileSearchResult> &future, void runFileSearchRegExp(QFutureInterface<FileSearchResult> &future,
QString searchTerm, QString searchTerm,
QStringList files, FileIterator *files,
QTextDocument::FindFlags flags, QTextDocument::FindFlags flags,
QMap<QString, QString> fileToContentsMap) QMap<QString, QString> fileToContentsMap)
{ {
future.setProgressRange(0, files.size()); future.setProgressRange(0, files->maxProgress());
int numFilesSearched = 0; int numFilesSearched = 0;
int numMatches = 0; int numMatches = 0;
if (flags & QTextDocument::FindWholeWords) if (flags & QTextDocument::FindWholeWords)
...@@ -204,11 +208,13 @@ void runFileSearchRegExp(QFutureInterface<FileSearchResult> &future, ...@@ -204,11 +208,13 @@ void runFileSearchRegExp(QFutureInterface<FileSearchResult> &future,
QFile file; QFile file;
QString str; QString str;
QTextStream stream; QTextStream stream;
foreach (const QString &s, files) { while (files->hasNext()) {
const QString &s = files->next();
future.setProgressRange(0, files->maxProgress());
if (future.isPaused()) if (future.isPaused())
future.waitForResume(); future.waitForResume();
if (future.isCanceled()) { if (future.isCanceled()) {
future.setProgressValueAndText(numFilesSearched, msgCanceled(searchTerm, numMatches, numFilesSearched)); future.setProgressValueAndText(files->currentProgress(), msgCanceled(searchTerm, numMatches, numFilesSearched));
break; break;
} }
...@@ -237,28 +243,30 @@ void runFileSearchRegExp(QFutureInterface<FileSearchResult> &future, ...@@ -237,28 +243,30 @@ void runFileSearchRegExp(QFutureInterface<FileSearchResult> &future,
++lineNr; ++lineNr;
} }
++numFilesSearched; ++numFilesSearched;
future.setProgressValueAndText(numFilesSearched, msgFound(searchTerm, numMatches, numFilesSearched, files.size())); if (future.isProgressUpdateNeeded())
future.setProgressValueAndText(files->currentProgress(), msgFound(searchTerm, numMatches, numFilesSearched));
if (needsToCloseFile) if (needsToCloseFile)
file.close(); file.close();
} }
if (!future.isCanceled()) if (!future.isCanceled())
future.setProgressValueAndText(numFilesSearched, msgFound(searchTerm, numMatches, numFilesSearched)); future.setProgressValueAndText(files->currentProgress(), msgFound(searchTerm, numMatches, numFilesSearched));
delete files;
} }
} // namespace } // namespace
QFuture<FileSearchResult> Utils::findInFiles(const QString &searchTerm, const QStringList &files, QFuture<FileSearchResult> Utils::findInFiles(const QString &searchTerm, FileIterator *files,
QTextDocument::FindFlags flags, QMap<QString, QString> fileToContentsMap) QTextDocument::FindFlags flags, QMap<QString, QString> fileToContentsMap)
{ {
return QtConcurrent::run<FileSearchResult, QString, QStringList, QTextDocument::FindFlags, QMap<QString, QString> > return QtConcurrent::run<FileSearchResult, QString, FileIterator *, QTextDocument::FindFlags, QMap<QString, QString> >
(runFileSearch, searchTerm, files, flags, fileToContentsMap); (runFileSearch, searchTerm, files, flags, fileToContentsMap);
} }
QFuture<FileSearchResult> Utils::findInFilesRegExp(const QString &searchTerm, const QStringList &files, QFuture<FileSearchResult> Utils::findInFilesRegExp(const QString &searchTerm, FileIterator *files,
QTextDocument::FindFlags flags, QMap<QString, QString> fileToContentsMap) QTextDocument::FindFlags flags, QMap<QString, QString> fileToContentsMap)
{ {
return QtConcurrent::run<FileSearchResult, QString, QStringList, QTextDocument::FindFlags, QMap<QString, QString> > return QtConcurrent::run<FileSearchResult, QString, FileIterator *, QTextDocument::FindFlags, QMap<QString, QString> >
(runFileSearchRegExp, searchTerm, files, flags, fileToContentsMap); (runFileSearchRegExp, searchTerm, files, flags, fileToContentsMap);
} }
...@@ -294,3 +302,133 @@ QString Utils::expandRegExpReplacement(const QString &replaceText, const QString ...@@ -294,3 +302,133 @@ QString Utils::expandRegExpReplacement(const QString &replaceText, const QString
} }
return result; return result;
} }
// #pragma mark -- FileIterator
FileIterator::FileIterator()
: m_list(QStringList()),
m_iterator(0),
m_index(0)
{
}
FileIterator::FileIterator(const QStringList &fileList)
: m_list(fileList),
m_iterator(new QStringListIterator(m_list)),
m_index(0)
{
}
FileIterator::~FileIterator()
{
if (m_iterator)
delete m_iterator;
}
bool FileIterator::hasNext() const
{
Q_ASSERT(m_iterator);
return m_iterator->hasNext();
}
QString FileIterator::next()
{
Q_ASSERT(m_iterator);
++m_index;
return m_iterator->next();
}
int FileIterator::maxProgress() const
{
return m_list.size();
}
int FileIterator::currentProgress() const
{
return m_index;
}
// #pragma mark -- SubDirFileIterator
namespace {
const int MAX_PROGRESS = 360;
}
SubDirFileIterator::SubDirFileIterator(const QStringList &directories, const QStringList &filters)
: m_filters(filters), m_progress(0)
{
int maxPer = MAX_PROGRESS/directories.count();
foreach (const QString &directoryEntry, directories) {
if (!directoryEntry.isEmpty()) {
m_dirs.push(QDir(directoryEntry));
m_progressValues.push(maxPer);
m_processedValues.push(false);
}
}
}
bool SubDirFileIterator::hasNext() const
{
if (!m_currentFiles.isEmpty())
return true;
while(!m_dirs.isEmpty() && m_currentFiles.isEmpty()) {
QDir dir = m_dirs.pop();
int dirProgressMax = m_progressValues.pop();
bool processed = m_processedValues.pop();
if (dir.exists()) {
QStringList subDirs;
if (!processed) {
subDirs = dir.entryList(QDir::Dirs|QDir::Hidden|QDir::NoDotAndDotDot);
}
if (subDirs.isEmpty()) {
QStringList fileEntries = dir.entryList(m_filters,
QDir::Files|QDir::Hidden);
QStringListIterator it(fileEntries);
it.toBack();
while (it.hasPrevious()) {
const QString &file = it.previous();
m_currentFiles.append(dir.path()+ QLatin1Char('/') +file);
}
m_progress += dirProgressMax;
} else {
int subProgress = dirProgressMax/(subDirs.size()+1);
int selfProgress = subProgress + dirProgressMax%(subDirs.size()+1);
m_dirs.push(dir);
m_progressValues.push(selfProgress);
m_processedValues.push(true);
QStringListIterator it(subDirs);
it.toBack();
while (it.hasPrevious()) {
const QString &directory = it.previous();
m_dirs.push(QDir(dir.path()+ QLatin1Char('/') + directory));
m_progressValues.push(subProgress);
m_processedValues.push(false);
}
}
} else {
m_progress += dirProgressMax;
}
}
if (m_currentFiles.isEmpty()) {
m_progress = MAX_PROGRESS;
return false;
}
return true;
}
QString SubDirFileIterator::next()
{
Q_ASSERT(!m_currentFiles.isEmpty());
return m_currentFiles.takeFirst();
}
int SubDirFileIterator::maxProgress() const
{
return MAX_PROGRESS;
}
int SubDirFileIterator::currentProgress() const
{
return m_progress;
}
...@@ -35,10 +35,49 @@ ...@@ -35,10 +35,49 @@
#include <QtCore/QStringList> #include <QtCore/QStringList>
#include <QtCore/QFuture> #include <QtCore/QFuture>
#include <QtCore/QMap> #include <QtCore/QMap>
#include <QtCore/QStack>
#include <QtCore/QDir>
#include <QtGui/QTextDocument> #include <QtGui/QTextDocument>
namespace Utils { namespace Utils {
class QTCREATOR_UTILS_EXPORT FileIterator
{
public:
FileIterator();
explicit FileIterator(const QStringList &fileList);
~FileIterator();
virtual bool hasNext() const;
virtual QString next();
virtual int maxProgress() const;
virtual int currentProgress() const;
private:
QStringList m_list;
QStringListIterator *m_iterator;
int m_index;
};
class QTCREATOR_UTILS_EXPORT SubDirFileIterator : public FileIterator
{
public:
SubDirFileIterator(const QStringList &directories, const QStringList &filters);
bool hasNext() const;
QString next();
int maxProgress() const;
int currentProgress() const;
private:
QStringList m_filters;
mutable QStack<QDir> m_dirs;
mutable QStack<int> m_progressValues;
mutable QStack<bool> m_processedValues;
mutable int m_progress;
mutable QStringList m_currentFiles;
};
class QTCREATOR_UTILS_EXPORT FileSearchResult class QTCREATOR_UTILS_EXPORT FileSearchResult
{ {
public: public:
...@@ -62,10 +101,10 @@ public: ...@@ -62,10 +101,10 @@ public:
QStringList regexpCapturedTexts; QStringList regexpCapturedTexts;
}; };
QTCREATOR_UTILS_EXPORT QFuture<FileSearchResult> findInFiles(const QString &searchTerm, const QStringList &files, QTCREATOR_UTILS_EXPORT QFuture<FileSearchResult> findInFiles(const QString &searchTerm, FileIterator *files,
QTextDocument::FindFlags flags, QMap<QString, QString> fileToContentsMap = QMap<QString, QString>()); QTextDocument::FindFlags flags, QMap<QString, QString> fileToContentsMap = QMap<QString, QString>());
QTCREATOR_UTILS_EXPORT QFuture<FileSearchResult> findInFilesRegExp(const QString &searchTerm, const QStringList &files, QTCREATOR_UTILS_EXPORT QFuture<FileSearchResult> findInFilesRegExp(const QString &searchTerm, FileIterator *files,
QTextDocument::FindFlags flags, QMap<QString, QString> fileToContentsMap = QMap<QString, QString>()); QTextDocument::FindFlags flags, QMap<QString, QString> fileToContentsMap = QMap<QString, QString>());
QTCREATOR_UTILS_EXPORT QString expandRegExpReplacement(const QString &replaceText, const QStringList &capturedTexts); QTCREATOR_UTILS_EXPORT QString expandRegExpReplacement(const QString &replaceText, const QStringList &capturedTexts);
......
...@@ -36,6 +36,7 @@ ...@@ -36,6 +36,7 @@
#include <QtGui/QMessageBox> #include <QtGui/QMessageBox>
#include <qtconcurrent/QtConcurrentTools> #include <qtconcurrent/QtConcurrentTools>
#include <utils/filesearch.h>
using namespace Locator; using namespace Locator;
using namespace Locator::Internal; using namespace Locator::Internal;
...@@ -181,76 +182,35 @@ void DirectoryFilter::updateOptionButtons() ...@@ -181,76 +182,35 @@ void DirectoryFilter::updateOptionButtons()
void DirectoryFilter::refresh(QFutureInterface<void> &future) void DirectoryFilter::refresh(QFutureInterface<void> &future)
{ {
const int MAX = 360; QStringList directories;
future.setProgressRange(0, MAX); {
if (m_directories.count() < 1) {
QMutexLocker locker(&m_lock); QMutexLocker locker(&m_lock);
files().clear(); if (m_directories.count() < 1) {
generateFileNames(); files().clear();
future.setProgressValueAndText(MAX, tr("%1 filter update: 0 files").arg(m_name)); generateFileNames();
return; future.setProgressRange(0, 1);
} future.setProgressValueAndText(1, tr("%1 filter update: 0 files").arg(m_name));
int progress = 0; return;
int MAX_PER = MAX;
QStringList filesFound;
QStack<QDir> dirs;
QStack<int> progressValues;
QStack<bool> processedValues;
{ // initialize
QMutexLocker locker(&m_lock);
MAX_PER = MAX/m_directories.count();
foreach (const QString &directoryEntry, m_directories) {
if (!directoryEntry.isEmpty()) {
dirs.push(QDir(directoryEntry));
progressValues.push(MAX_PER);
processedValues.push(false);
}
} }
directories = m_directories;
} }
while (!dirs.isEmpty() && !future.isCanceled()) { Utils::SubDirFileIterator it(directories, m_filters);
future.setProgressRange(0, it.maxProgress());
QStringList filesFound;
while (!future.isCanceled() && it.hasNext()) {
filesFound << it.next();
if (future.isProgressUpdateNeeded()) { if (future.isProgressUpdateNeeded()) {
future.setProgressValueAndText(progress, future.setProgressValueAndText(it.currentProgress(),
tr("%1 filter update: %n files", 0, filesFound.size()).arg(m_name)); tr("%1 filter update: %n files", 0, filesFound.size()).arg(m_name));
} }
QDir dir = dirs.pop();
int dirProgressMax = progressValues.pop();
bool processed = processedValues.pop();
if (dir.exists()) {
QStringList subDirs;
if (!processed) {
subDirs = dir.entryList(QDir::Dirs|QDir::Hidden|QDir::NoDotAndDotDot,
QDir::Name|QDir::IgnoreCase|QDir::LocaleAware);
}
if (subDirs.isEmpty()) {
QStringList fileEntries = dir.entryList(m_filters,
QDir::Files|QDir::Hidden,
QDir::Name|QDir::IgnoreCase|QDir::LocaleAware);
foreach (const QString &file, fileEntries)
filesFound.append(dir.path()+ QLatin1Char('/') +file);
progress += dirProgressMax;
} else {
int subProgress = dirProgressMax/(subDirs.size()+1);
int selfProgress = subProgress + dirProgressMax%(subDirs.size()+1);
dirs.push(dir);
progressValues.push(selfProgress);
processedValues.push(true);
foreach (const QString &directory, subDirs) {
dirs.push(QDir(dir.path()+ QLatin1Char('/') + directory));
progressValues.push(subProgress);
processedValues.push(false);
}
}
} else {
progress += dirProgressMax;
}
} }
if (!future.isCanceled()) { if (!future.isCanceled()) {
QMutexLocker locker(&m_lock); QMutexLocker locker(&m_lock);
files() = filesFound; files() = filesFound;
generateFileNames(); generateFileNames();
future.setProgressValue(MAX); future.setProgressValue(it.maxProgress());
} else { } else {
future.setProgressValueAndText(progress, tr("%1 filter update: canceled").arg(m_name)); future.setProgressValueAndText(it.currentProgress(), tr("%1 filter update: canceled").arg(m_name));
} }
} }
...@@ -77,7 +77,7 @@ QKeySequence AllProjectsFind::defaultShortcut() const ...@@ -77,7 +77,7 @@ QKeySequence AllProjectsFind::defaultShortcut() const
return QKeySequence(); return QKeySequence();
} }
QStringList AllProjectsFind::files() Utils::FileIterator *AllProjectsFind::files()
{ {
Q_ASSERT(m_plugin->session()); Q_ASSERT(m_plugin->session());
QList<QRegExp> filterRegs; QList<QRegExp> filterRegs;
...@@ -103,7 +103,7 @@ QStringList AllProjectsFind::files() ...@@ -103,7 +103,7 @@ QStringList AllProjectsFind::files()
} }
} }
files.removeDuplicates(); files.removeDuplicates();
return files; return new Utils::FileIterator(files);
} }
QWidget *AllProjectsFind::createConfigWidget() QWidget *AllProjectsFind::createConfigWidget()
......
...@@ -61,7 +61,7 @@ public: ...@@ -61,7 +61,7 @@ public:
void readSettings(QSettings *settings); void readSettings(QSettings *settings);
protected: protected:
QStringList files(); Utils::FileIterator *files();
private: private:
ProjectExplorerPlugin *m_plugin; ProjectExplorerPlugin *m_plugin;
......
...@@ -75,7 +75,7 @@ QKeySequence CurrentProjectFind::defaultShortcut() const ...@@ -75,7 +75,7 @@ QKeySequence CurrentProjectFind::defaultShortcut() const
return QKeySequence(); return QKeySequence();
} }
QStringList CurrentProjectFind::files() Utils::FileIterator *CurrentProjectFind::files()
{ {
Project *project = m_plugin->currentProject(); Project *project = m_plugin->currentProject();
Q_ASSERT(project); Q_ASSERT(project);
...@@ -98,7 +98,7 @@ QStringList CurrentProjectFind::files() ...@@ -98,7 +98,7 @@ QStringList CurrentProjectFind::files()
files += project->files(Project::AllFiles); files += project->files(Project::AllFiles);
} }
files.removeDuplicates(); files.removeDuplicates();
return files; return new Utils::FileIterator(files);
} }
QWidget *CurrentProjectFind::createConfigWidget() QWidget *CurrentProjectFind::createConfigWidget()
......
...@@ -64,7 +64,7 @@ public: ...@@ -64,7 +64,7 @@ public:
void readSettings(QSettings *settings); void readSettings(QSettings *settings);
protected: protected:
QStringList files(); Utils::FileIterator *files();
private: private:
ProjectExplorerPlugin *m_plugin; ProjectExplorerPlugin *m_plugin;
......
...@@ -71,7 +71,7 @@ public: ...@@ -71,7 +71,7 @@ public:
const QList<Find::SearchResultItem> &items); const QList<Find::SearchResultItem> &items);
protected: protected:
virtual QStringList files() = 0; virtual Utils::FileIterator *files() = 0;
void writeCommonSettings(QSettings *settings); void writeCommonSettings(QSettings *settings);
void readCommonSettings(QSettings *settings, const QString &defaultFilter); void readCommonSettings(QSettings *settings, const QString &defaultFilter);
QWidget *createPatternWidget(); QWidget *createPatternWidget();
......
...@@ -67,12 +67,12 @@ QKeySequence FindInCurrentFile::defaultShortcut() const ...@@ -67,12 +67,12 @@ QKeySequence FindInCurrentFile::defaultShortcut() const
return QKeySequence(); return QKeySequence();
} }
QStringList FindInCurrentFile::files() Utils::FileIterator *FindInCurrentFile::files()
{ {
<