File search: Avoid use of QtConcurrent

QtConcurrent limits resource usage to a global number of simultaneous
threads. That means that if some QtConcurrent based algorithm currently
grabs all threads, any other use of QtConcurrent blocks, which is not
what we want.
Use the new threading methods of C++11 instead, but still use
QFuture(Interface) manually for the progress, result and status
reporting.

Task-number: QTCREATORBUG-14640
Change-Id: I6379d2f2a01b6d200811ef4be0bbfcd4493dd154
Reviewed-by: Orgad Shaneh <orgads@gmail.com>

src/libs/utils/filesearch.h

@@ -56,6 +56,8 @@ public:
         QTextCodec *encoding;
     };
 
+    typedef Item value_type;
+
     class const_iterator
     {
     public:
@@ -65,27 +67,27 @@ public:
         typedef const value_type *pointer;
         typedef const value_type &reference;
 
-        const_iterator(FileIterator *parent, Item item, int id)
+        const_iterator(const FileIterator *parent, Item item, int id)
             : m_parent(parent), m_item(item), m_index(id)
         {}
         const Item operator*() const { return m_item; }
         const Item *operator->() const { return &m_item; }
-        void operator++() { m_parent->next(this); }
+        void operator++() { m_parent->advance(this); }
         bool operator==(const const_iterator &other) const
         {
             return m_parent == other.m_parent && m_index == other.m_index;
         }
         bool operator!=(const const_iterator &other) const { return !operator==(other); }
 
-        FileIterator *m_parent;
+        const FileIterator *m_parent;
         Item m_item;
         int m_index; // -1 == end
     };
 
     virtual ~FileIterator() {}
-    void next(const_iterator *it);
-    const_iterator begin();
-    const_iterator end();
+    void advance(const_iterator *it) const;
+    const_iterator begin() const;
+    const_iterator end() const;
 
     virtual int maxProgress() const = 0;
     virtual int currentProgress() const = 0;

src/libs/utils/runextensions.h

@@ -31,12 +31,18 @@
 #ifndef RUNEXTENSIONS_H
 #define RUNEXTENSIONS_H
 
+#include "qtcassert.h"
+
 #include <qrunnable.h>
 #include <qfuture.h>
 #include <qfutureinterface.h>
 #include <qthreadpool.h>
 
+#include <chrono>
 #include <functional>
+#include <future>
+#include <thread>
+#include <vector>
 
 QT_BEGIN_NAMESPACE
 
@@ -431,4 +437,116 @@ QFuture<T> run(const std::function<void (QFutureInterface<T> &)> &fn)
 
 QT_END_NAMESPACE
 
+namespace Utils {
+
+template<typename T>
+typename std::vector<std::future<T>>::iterator
+waitForAny(std::vector<std::future<T>> &futures)
+{
+    // Wait for any future to have a result ready.
+    // Unfortunately we have to do that in a busy loop because future doesn't have a feature to
+    // wait for any of a set of futures (yet? possibly when_any in C++17).
+    auto end = futures.end();
+    QTC_ASSERT(!futures.empty(), return end);
+    auto futureIterator = futures.begin();
+    forever {
+        if (futureIterator->wait_for(std::chrono::duration<quint64>::zero()) == std::future_status::ready)
+            return futureIterator;
+        ++futureIterator;
+        if (futureIterator == end)
+            futureIterator = futures.begin();
+    }
+}
+
+namespace Internal {
+
+template<typename T>
+void swapErase(std::vector<T> &vec, typename std::vector<T>::iterator it)
+{
+    // efficient erasing by swapping with back element
+    *it = std::move(vec.back());
+    vec.pop_back();
+}
+
+template <typename MapResult, typename State, typename ReduceResult, typename ReduceFunction>
+void reduceOne(QFutureInterface<ReduceResult> &futureInterface,
+               std::vector<std::future<MapResult>> &futures,
+               State &state, const ReduceFunction &reduce)
+{
+    auto futureIterator = waitForAny(futures);
+    if (futureIterator != futures.end()) {
+        reduce(futureInterface, state, futureIterator->get());
+        swapErase(futures, futureIterator);
+    }
+}
+
+// This together with reduceOne can be replaced by std::transformReduce (parallelism TS)
+// when that becomes widely available in C++ implementations
+template <typename Container, typename MapFunction, typename State, typename ReduceResult, typename ReduceFunction>
+void mapReduceLoop(QFutureInterface<ReduceResult> &futureInterface, const Container &container,
+               const MapFunction &map, State &state, const ReduceFunction &reduce)
+{
+    const unsigned MAX_THREADS = std::thread::hardware_concurrency();
+    using MapResult = typename std::result_of<MapFunction(QFutureInterface<ReduceResult>,typename Container::value_type)>::type;
+    std::vector<std::future<MapResult>> futures;
+    futures.reserve(MAX_THREADS);
+    auto fileIterator = container.begin();
+    auto end = container.end();
+    while (!futureInterface.isCanceled() && (fileIterator != end || futures.size() != 0)) {
+        if (futures.size() >= MAX_THREADS || fileIterator == end) {
+            // We don't want to start a new thread (yet), so try to find a future that is ready and
+            // handle its result.
+            reduceOne(futureInterface, futures, state, reduce);
+        } else { // start a new thread
+            futures.push_back(std::async(std::launch::async,
+                                         map, futureInterface, *fileIterator));
+            ++fileIterator;
+        }
+    }
+}
+
+template <typename Container, typename InitFunction, typename MapFunction, typename ReduceResult,
+          typename ReduceFunction, typename CleanUpFunction>
+void blockingMapReduce(QFutureInterface<ReduceResult> futureInterface, const Container &container,
+                       const InitFunction &init, const MapFunction &map,
+                       const ReduceFunction &reduce, const CleanUpFunction &cleanup)
+{
+    auto state = init(futureInterface);
+    futureInterface.reportStarted();
+    mapReduceLoop(futureInterface, container, map, state, reduce);
+    cleanup(futureInterface, state);
+    if (futureInterface.isPaused())
+        futureInterface.waitForResume();
+    futureInterface.reportFinished();
+}
+
+} // Internal
+
+template <typename ReduceResult, typename Container, typename InitFunction, typename MapFunction,
+          typename ReduceFunction, typename CleanUpFunction>
+QFuture<ReduceResult> mapReduce(std::reference_wrapper<Container> containerWrapper,
+                                const InitFunction &init, const MapFunction &map,
+                                const ReduceFunction &reduce, const CleanUpFunction &cleanup)
+{
+    auto fi = QFutureInterface<ReduceResult>();
+    QFuture<ReduceResult> future = fi.future();
+    std::thread(Internal::blockingMapReduce<Container, InitFunction, MapFunction, ReduceResult, ReduceFunction, CleanUpFunction>,
+                fi, containerWrapper, init, map, reduce, cleanup).detach();
+    return future;
+}
+
+template <typename ReduceResult, typename Container, typename InitFunction, typename MapFunction,
+          typename ReduceFunction, typename CleanUpFunction>
+QFuture<ReduceResult> mapReduce(const Container &container, const InitFunction &init, const MapFunction &map,
+               const ReduceFunction &reduce, const CleanUpFunction &cleanup)
+{
+    auto fi = QFutureInterface<ReduceResult>();
+    QFuture<ReduceResult> future = fi.future();
+    std::thread(Internal::blockingMapReduce<Container, InitFunction, MapFunction, ReduceResult, ReduceFunction, CleanUpFunction>,
+                fi, container, init, map, reduce, cleanup).detach();
+    return future;
+}
+
+} // Utils
+
 #endif // RUNEXTENSIONS_H