Clang: Support second translation unit

A TranslationUnit is owned by TranslationUnits now. TranslationUnits
allows to add another TranslationUnit and to update/query the recently
and previously parsed translation unit.

This does not change any behavior yet.

Change-Id: I8a2f0cc05d3e51bf739dd5d7c4da14b54147f3ab
Reviewed-by: David Schulz <david.schulz@qt.io>

src/tools/clangbackend/ipcsource/clangbackend_global.h

+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+namespace ClangBackEnd {
+
+enum class PreferredTranslationUnit
+{
+    RecentlyParsed,
+    PreviouslyParsed,
+};
+
+} // namespace ClangBackEnd

src/tools/clangbackend/ipcsource/clangbackendclangipc-source.pri

@@ -29,6 +29,7 @@ HEADERS += $$PWD/clangcodemodelserver.h \
     $$PWD/highlightingmarksiterator.h \
     $$PWD/utf8positionfromlinecolumn.h \
     $$PWD/clangasyncjob.h \
+    $$PWD/clangbackend_global.h \
     $$PWD/clangcompletecodejob.h \
     $$PWD/clangcreateinitialdocumentpreamblejob.h \
     $$PWD/clangfilepath.h \
@@ -44,6 +45,7 @@ HEADERS += $$PWD/clangcodemodelserver.h \
     $$PWD/clangexceptions.h \
     $$PWD/clangdocumentprocessor.h \
     $$PWD/clangdocumentprocessors.h \
+    $$PWD/clangtranslationunits.h
 
 SOURCES += $$PWD/clangcodemodelserver.cpp \
     $$PWD/codecompleter.cpp \
@@ -85,3 +87,4 @@ SOURCES += $$PWD/clangcodemodelserver.cpp \
     $$PWD/clangexceptions.cpp \
     $$PWD/clangdocumentprocessor.cpp \
     $$PWD/clangdocumentprocessors.cpp \
+    $$PWD/clangtranslationunits.cpp

src/tools/clangbackend/ipcsource/clangdocument.cpp

@@ -32,6 +32,7 @@
 #include "projectpart.h"
 #include "clangexceptions.h"
 #include "clangtranslationunit.h"
+#include "clangtranslationunits.h"
 #include "clangtranslationunitupdater.h"
 #include "unsavedfiles.h"
 #include "unsavedfile.h"
@@ -64,8 +65,7 @@ public:
     ProjectPart projectPart;
     time_point lastProjectPartChangeTimePoint;
 
-    CXTranslationUnit translationUnit = nullptr;
-    CXIndex index = nullptr;
+    TranslationUnits translationUnits;
 
     QSet<Utf8String> dependedFilePaths;
 
@@ -86,15 +86,15 @@ DocumentData::DocumentData(const Utf8String &filePath,
       fileArguments(fileArguments),
       projectPart(projectPart),
       lastProjectPartChangeTimePoint(std::chrono::steady_clock::now()),
+      translationUnits(filePath),
       needsToBeReparsedChangeTimePoint(lastProjectPartChangeTimePoint)
 {
     dependedFilePaths.insert(filePath);
+    translationUnits.createAndAppend();
 }
 
 DocumentData::~DocumentData()
 {
-    clang_disposeTranslationUnit(translationUnit);
-    clang_disposeIndex(index);
 }
 
 Document::Document(const Utf8String &filePath,
@@ -282,8 +282,13 @@ TranslationUnitUpdateInput Document::createUpdateInput() const
 
 TranslationUnitUpdater Document::createUpdater() const
 {
+    TranslationUnit unit = translationUnit();
+
     const TranslationUnitUpdateInput updateInput = createUpdateInput();
-    TranslationUnitUpdater updater(d->index, d->translationUnit, updateInput);
+    TranslationUnitUpdater updater(unit.id(),
+                                   unit.cxIndex(),
+                                   unit.cxTranslationUnit(),
+                                   updateInput);
 
     return updater;
 }
@@ -304,9 +309,13 @@ void Document::incorporateUpdaterResult(const TranslationUnitUpdateResult &resul
     if (result.hasParsed())
         d->lastProjectPartChangeTimePoint = result.parseTimePoint;
 
-    if (result.hasParsed() || result.hasReparsed())
+    if (result.hasParsed() || result.hasReparsed()) {
         d->dependedFilePaths = result.dependedOnFilePaths;
 
+        const time_point timePoint = qMax(result.parseTimePoint, result.reparseTimePoint);
+        d->translationUnits.updateParseTimePoint(result.translationUnitId, timePoint);
+    }
+
     d->documents.addWatchedFiles(d->dependedFilePaths);
 
     if (result.hasReparsed()
@@ -315,11 +324,16 @@ void Document::incorporateUpdaterResult(const TranslationUnitUpdateResult &resul
     }
 }
 
-TranslationUnit Document::translationUnit() const
+TranslationUnit Document::translationUnit(PreferredTranslationUnit preferredTranslationUnit) const
 {
     checkIfNull();
 
-    return TranslationUnit(d->filePath, d->index, d->translationUnit);
+    return d->translationUnits.get(preferredTranslationUnit);
+}
+
+TranslationUnits &Document::translationUnits() const
+{
+    return d->translationUnits;
 }
 
 void Document::parse() const

src/tools/clangbackend/ipcsource/clangdocument.h

@@ -27,6 +27,7 @@
 
 #include "clangtranslationunitupdater.h"
 
+#include "clangbackend_global.h"
 #include "clangtranslationunit.h"
 
 #include <utf8stringvector.h>
@@ -44,6 +45,7 @@ class Utf8String;
 namespace ClangBackEnd {
 
 class TranslationUnit;
+class TranslationUnits;
 class DocumentData;
 class TranslationUnitUpdateResult;
 class ProjectPart;
@@ -104,7 +106,9 @@ public:
     TranslationUnitUpdateInput createUpdateInput() const;
     void incorporateUpdaterResult(const TranslationUnitUpdateResult &result) const;
 
-    TranslationUnit translationUnit() const;
+    TranslationUnit translationUnit(PreferredTranslationUnit preferredTranslationUnit
+                                        = PreferredTranslationUnit::RecentlyParsed) const;
+    TranslationUnits &translationUnits() const;
 
 public: // for tests
     void parse() const;

src/tools/clangbackend/ipcsource/clangexceptions.cpp

@@ -94,4 +94,11 @@ DocumentProcessorDoesNotExist::DocumentProcessorDoesNotExist(const Utf8String &f
            + Utf8StringLiteral("' does not exist!");
 }
 
+TranslationUnitDoesNotExist::TranslationUnitDoesNotExist(const Utf8String &filePath)
+{
+    m_info += Utf8StringLiteral("TranslationUnit for file '")
+            + filePath
+            + Utf8StringLiteral("' does not exist.");
+}
+
 } // namespace ClangBackEnd

src/tools/clangbackend/ipcsource/clangexceptions.h

@@ -88,4 +88,10 @@ public:
                                   const Utf8String &projectPartId);
 };
 
+class TranslationUnitDoesNotExist : public ClangBaseException
+{
+public:
+    TranslationUnitDoesNotExist(const Utf8String &filePath);
+};
+
 } // namespace ClangBackEnd

src/tools/clangbackend/ipcsource/clangtranslationunit.cpp

@@ -38,10 +38,12 @@
 
 namespace ClangBackEnd {
 
-TranslationUnit::TranslationUnit(const Utf8String &filepath,
+TranslationUnit::TranslationUnit(const Utf8String &id,
+                                 const Utf8String &filepath,
                                  CXIndex &cxIndex,
                                  CXTranslationUnit &cxTranslationUnit)
-    : m_filePath(filepath)
+    : m_id(id)
+    , m_filePath(filepath)
     , m_cxIndex(cxIndex)
     , m_cxTranslationUnit(cxTranslationUnit)
 {
@@ -49,7 +51,12 @@ TranslationUnit::TranslationUnit(const Utf8String &filepath,
 
 bool TranslationUnit::isNull() const
 {
-    return !m_cxTranslationUnit || !m_cxIndex || m_filePath.isEmpty();
+    return !m_cxTranslationUnit || !m_cxIndex || m_filePath.isEmpty() || m_id.isEmpty();
+}
+
+Utf8String TranslationUnit::id() const
+{
+    return m_id;
 }
 
 Utf8String TranslationUnit::filePath() const
@@ -70,7 +77,7 @@ CXTranslationUnit &TranslationUnit::cxTranslationUnit() const
 TranslationUnitUpdateResult TranslationUnit::update(
         const TranslationUnitUpdateInput &parseInput) const
 {
-    TranslationUnitUpdater updater(cxIndex(), cxTranslationUnit(), parseInput);
+    TranslationUnitUpdater updater(id(), cxIndex(), cxTranslationUnit(), parseInput);
 
     return updater.update(TranslationUnitUpdater::UpdateMode::AsNeeded);
 }
@@ -78,7 +85,7 @@ TranslationUnitUpdateResult TranslationUnit::update(
 TranslationUnitUpdateResult TranslationUnit::parse(
         const TranslationUnitUpdateInput &parseInput) const
 {
-    TranslationUnitUpdater updater(cxIndex(), cxTranslationUnit(), parseInput);
+    TranslationUnitUpdater updater(id(), cxIndex(), cxTranslationUnit(), parseInput);
 
     return updater.update(TranslationUnitUpdater::UpdateMode::ParseIfNeeded);
 }
@@ -86,7 +93,7 @@ TranslationUnitUpdateResult TranslationUnit::parse(
 TranslationUnitUpdateResult TranslationUnit::reparse(
         const TranslationUnitUpdateInput &parseInput) const
 {
-    TranslationUnitUpdater updater(cxIndex(), cxTranslationUnit(), parseInput);
+    TranslationUnitUpdater updater(id(), cxIndex(), cxTranslationUnit(), parseInput);
 
     return updater.update(TranslationUnitUpdater::UpdateMode::ForceReparse);
 }

src/tools/clangbackend/ipcsource/clangtranslationunit.h

@@ -57,12 +57,15 @@ public:
     };
 
 public:
-    TranslationUnit(const Utf8String &filePath,
+    TranslationUnit(const Utf8String &id,
+                    const Utf8String &filePath,
                     CXIndex &cxIndex,
                     CXTranslationUnit &cxTranslationUnit);
 
     bool isNull() const;
 
+    Utf8String id() const;
+
     Utf8String filePath() const;
     CXIndex &cxIndex() const;
     CXTranslationUnit &cxTranslationUnit() const;
@@ -94,6 +97,7 @@ public:
     SkippedSourceRanges skippedSourceRanges() const;
 
 private:
+    const Utf8String m_id;
     const Utf8String m_filePath;
     CXIndex &m_cxIndex;
     CXTranslationUnit &m_cxTranslationUnit;

src/tools/clangbackend/ipcsource/clangtranslationunits.cpp

+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "clangtranslationunits.h"
+
+#include "clangexceptions.h"
+#include "clangtranslationunit.h"
+
+#include <utils/algorithm.h>
+#include <utils/qtcassert.h>
+
+#include <QFileInfo>
+#include <QLoggingCategory>
+#include <QUuid>
+
+#include <algorithm>
+
+Q_LOGGING_CATEGORY(tuLog, "qtc.clangbackend.translationunits");
+
+namespace ClangBackEnd {
+
+TranslationUnits::TranslationUnits(const Utf8String &filePath)
+    : m_filePath(filePath)
+{
+}
+
+TranslationUnits::~TranslationUnits()
+{
+    foreach (const TranslationUnitData &unit, m_tuDatas) {
+        clang_disposeTranslationUnit(unit.cxTranslationUnit);
+        clang_disposeIndex(unit.cxIndex);
+    }
+}
+
+TranslationUnit TranslationUnits::createAndAppend()
+{
+    const Utf8String id = Utf8String::fromByteArray(QUuid::createUuid().toByteArray());
+    qCDebug(tuLog) << "Creating TranslationUnit" << id << "for" << QFileInfo(m_filePath).fileName();
+
+    m_tuDatas.append(TranslationUnitData(id));
+    TranslationUnitData &translationUnitData = m_tuDatas.last();
+
+    return toTranslationUnit(translationUnitData);
+}
+
+TranslationUnit TranslationUnits::get(PreferredTranslationUnit type)
+{
+    if (m_tuDatas.isEmpty())
+        throw TranslationUnitDoesNotExist(m_filePath);
+
+    if (m_tuDatas.size() == 1 || !areAllTranslationUnitsParsed())
+        return toTranslationUnit(m_tuDatas.first());
+
+    return getPreferredTranslationUnit(type);
+}
+
+void TranslationUnits::updateParseTimePoint(const Utf8String &translationUnitId,
+                                            time_point timePoint)
+{
+    TranslationUnitData &unit = findUnit(translationUnitId);
+
+    QTC_CHECK(timePoint != time_point());
+    unit.parseTimePoint = timePoint;
+
+    qCDebug(tuLog) << "Updated" << translationUnitId << "for" << QFileInfo(m_filePath).fileName()
+        << "RecentlyParsed:" << get(PreferredTranslationUnit::RecentlyParsed).id()
+        << "PreviouslyParsed:" << get(PreferredTranslationUnit::PreviouslyParsed).id();
+}
+
+bool TranslationUnits::areAllTranslationUnitsParsed() const
+{
+    return Utils::allOf(m_tuDatas, [](const TranslationUnitData &unit) {
+        return unit.parseTimePoint != time_point();
+    });
+}
+
+TranslationUnit TranslationUnits::getPreferredTranslationUnit(PreferredTranslationUnit type)
+{
+    using TuData = TranslationUnitData;
+
+    const auto lessThan = [](const TuData &a, const TuData &b) {
+        return a.parseTimePoint < b.parseTimePoint;
+    };
+    auto translationUnitData = type == PreferredTranslationUnit::RecentlyParsed
+            ? std::max_element(m_tuDatas.begin(), m_tuDatas.end(), lessThan)
+            : std::min_element(m_tuDatas.begin(), m_tuDatas.end(), lessThan);
+
+    if (translationUnitData == m_tuDatas.end())
+        throw TranslationUnitDoesNotExist(m_filePath);
+
+    return toTranslationUnit(*translationUnitData);
+}
+
+TranslationUnits::TranslationUnitData &TranslationUnits::findUnit(
+        const Utf8String &translationUnitId)
+{
+    for (TranslationUnitData &unit : m_tuDatas) {
+        if (translationUnitId == unit.id)
+            return unit;
+    }
+
+    throw TranslationUnitDoesNotExist(m_filePath);
+}
+
+TranslationUnit TranslationUnits::toTranslationUnit(TranslationUnits::TranslationUnitData &unit)
+{
+    return TranslationUnit(unit.id,
+                           m_filePath,
+                           unit.cxIndex,
+                           unit.cxTranslationUnit);
+}
+
+} // namespace ClangBackEnd

src/tools/clangbackend/ipcsource/clangtranslationunits.h

+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "clangbackend_global.h"
+
+#include <utf8string.h>
+
+#include <clang-c/Index.h>
+
+#include <QList>
+
+#include <chrono>
+
+namespace ClangBackEnd {
+
+using time_point = std::chrono::steady_clock::time_point;
+
+class TranslationUnit;
+
+class TranslationUnits
+{
+public:
+    class TranslationUnitData {
+    public:
+        TranslationUnitData(const Utf8String &id)
+            : id(id)
+        {}
+
+        Utf8String id;
+
+        CXTranslationUnit cxTranslationUnit = nullptr;
+        CXIndex cxIndex = nullptr;
+
+        time_point parseTimePoint;
+    };
+
+public:
+    TranslationUnits(const Utf8String &filePath);
+    ~TranslationUnits();
+
+    TranslationUnit createAndAppend();
+    TranslationUnit get(PreferredTranslationUnit type = PreferredTranslationUnit::RecentlyParsed);
+    void updateParseTimePoint(const Utf8String &translationUnitId, time_point timePoint);
+
+private:
+    bool areAllTranslationUnitsParsed() const;
+    TranslationUnit getPreferredTranslationUnit(PreferredTranslationUnit type);
+    TranslationUnitData &findUnit(const Utf8String &translationUnitId);
+    TranslationUnit toTranslationUnit(TranslationUnitData &unit);
+
+private:
+    Utf8String m_filePath;
+    QList<TranslationUnitData> m_tuDatas;
+};
+
+} // namespace ClangBackEnd

src/tools/clangbackend/ipcsource/clangtranslationunitupdater.cpp

@@ -40,13 +40,15 @@ static bool isVerboseModeEnabled()
 
 namespace ClangBackEnd {
 
-TranslationUnitUpdater::TranslationUnitUpdater(CXIndex &index,
+TranslationUnitUpdater::TranslationUnitUpdater(const Utf8String translationUnitId,
+                                               CXIndex &index,
                                                CXTranslationUnit &cxTranslationUnit,
                                                const TranslationUnitUpdateInput &updateData)
     : m_cxIndex(index)
     , m_cxTranslationUnit(cxTranslationUnit)
     , m_in(updateData)
 {
+    m_out.translationUnitId = translationUnitId;
 }
 
 TranslationUnitUpdateResult TranslationUnitUpdater::update(UpdateMode mode)

src/tools/clangbackend/ipcsource/clangtranslationunitupdater.h

@@ -63,8 +63,9 @@ public:
     { return reparseTimePoint != time_point(); }
 
 public:
-    bool hasParseOrReparseFailed = false;
+    Utf8String translationUnitId;
 
+    bool hasParseOrReparseFailed = false;
     time_point parseTimePoint;
     time_point reparseTimePoint;
     time_point needsToBeReparsedChangeTimePoint;
@@ -81,7 +82,8 @@ public:
     };
 
 public:
-    TranslationUnitUpdater(CXIndex &index,
+    TranslationUnitUpdater(const Utf8String translationUnitId,
+                           CXIndex &index,
                            CXTranslationUnit &cxTranslationUnit,
                            const TranslationUnitUpdateInput &in);
 

tests/unit/unittest/clangdocument-test.cpp

@@ -27,6 +27,8 @@
 
 #include <clangfilepath.h>
 #include <clangtranslationunitupdater.h>
+#include <clangtranslationunits.h>
+#include <clangtranslationunit.h>
 #include <commandlinearguments.h>
 #include <diagnosticset.h>
 #include <highlightingmarks.h>
@@ -56,6 +58,8 @@ using ClangBackEnd::ProjectPart;
 using ClangBackEnd::ProjectPartContainer;
 using ClangBackEnd::Documents;
 using ClangBackEnd::TranslationUnitUpdateResult;
+using ClangBackEnd::TranslationUnit;
+using ClangBackEnd::TranslationUnits;
 
 using testing::IsNull;
 using testing::NotNull;
@@ -332,6 +336,7 @@ TEST_F(Document, IncorporateUpdaterResultResetsDirtyness)
     TranslationUnitUpdateResult result;
     result.reparseTimePoint = std::chrono::steady_clock::now();
     result.needsToBeReparsedChangeTimePoint = document.isNeededReparseChangeTimePoint();
+    result.translationUnitId = document.translationUnit().id();
 
     document.incorporateUpdaterResult(result);
 
@@ -343,6 +348,7 @@ TEST_F(Document, IncorporateUpdaterResultDoesNotResetDirtynessIfItWasChanged)
     TranslationUnitUpdateResult result;
     result.reparseTimePoint = std::chrono::steady_clock::now();
     result.needsToBeReparsedChangeTimePoint = std::chrono::steady_clock::now();
+    result.translationUnitId = document.translationUnit().id();
     document.setDirtyIfDependencyIsMet(document.filePath());
 
     document.incorporateUpdaterResult(result);
@@ -350,6 +356,25 @@ TEST_F(Document, IncorporateUpdaterResultDoesNotResetDirtynessIfItWasChanged)
     ASSERT_TRUE(document.isNeedingReparse());
 }
 
+TEST_F(Document, IncorporateUpdaterResultUpdatesTranslationUnitsReparseTimePoint)
+{
+    TranslationUnits &translationUnits = document.translationUnits();
+    const TranslationUnit initialTranslationUnit = translationUnits.get();
+    translationUnits.updateParseTimePoint(initialTranslationUnit.id(), std::chrono::steady_clock::now());
+    const TranslationUnit alternativeTranslationUnit = translationUnits.createAndAppend();
+    translationUnits.updateParseTimePoint(alternativeTranslationUnit.id(), std::chrono::steady_clock::now());
+    TranslationUnitUpdateResult result;
+    result.reparseTimePoint = std::chrono::steady_clock::now();
+    result.needsToBeReparsedChangeTimePoint = std::chrono::steady_clock::now();
+    result.translationUnitId = initialTranslationUnit.id();
+    document.setDirtyIfDependencyIsMet(document.filePath());
+    ASSERT_THAT(translationUnits.get().id(), Eq(alternativeTranslationUnit.id()));
+
+    document.incorporateUpdaterResult(result);
+
+    ASSERT_THAT(translationUnits.get().id(), Eq(initialTranslationUnit.id()));
+}
+
 void Document::SetUp()
 {
     projects.createOrUpdate({ProjectPartContainer(projectPartId)});

tests/unit/unittest/clangtranslationunits-test.cpp

+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms