AutoTest: Support plaintext output for Qt tests

Change-Id: I88ec477777d79c69e699dd906ec4ef1550bcaf44
Reviewed-by: David Schulz <david.schulz@qt.io>

src/plugins/autotest/qtest/qttestconfiguration.cpp

@@ -35,7 +35,17 @@ namespace Internal {
 TestOutputReader *QtTestConfiguration::outputReader(const QFutureInterface<TestResultPtr> &fi,
                                                     QProcess *app) const
 {
-    return new QtTestOutputReader(fi, app, buildDirectory());
+    static const Core::Id id
+            = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
+    TestFrameworkManager *manager = TestFrameworkManager::instance();
+    auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
+    if (qtSettings.isNull())
+        return nullptr;
+
+    if (qtSettings->useXMLOutput)
+        return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::XML);
+    else
+        return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::PlainText);
 }
 
 QStringList QtTestConfiguration::argumentsForTestRunner() const
@@ -43,14 +53,15 @@ QStringList QtTestConfiguration::argumentsForTestRunner() const
     static const Core::Id id
             = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
 
-    QStringList arguments("-xml");
-    if (testCases().count())
-        arguments << testCases();
-
+    QStringList arguments;
     TestFrameworkManager *manager = TestFrameworkManager::instance();
     auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
     if (qtSettings.isNull())
         return arguments;
+    if (qtSettings->useXMLOutput)
+        arguments << "-xml";
+    if (testCases().count())
+        arguments << testCases();
 
     const QString &metricsOption = QtTestSettings::metricsTypeToOption(qtSettings->metrics);
     if (!metricsOption.isEmpty())

src/plugins/autotest/qtest/qttestoutputreader.h

@@ -40,14 +40,32 @@ class QtTestOutputReader : public TestOutputReader
     Q_DECLARE_TR_FUNCTIONS(Autotest::Internal::QtTestOutputReader)
 
 public:
+    enum OutputMode
+    {
+        XML,
+        PlainText
+    };
+
     QtTestOutputReader(const QFutureInterface<TestResultPtr> &futureInterface,
-                       QProcess *testApplication, const QString &buildDirectory);
+                       QProcess *testApplication, const QString &buildDirectory,
+                       OutputMode mode);
 
 protected:
     void processOutput(const QByteArray &outputLine) override;
 
 private:
+    void processXMLOutput(const QByteArray &outputLine);
+    void processPlainTextOutput(const QByteArray &outputLine);
+    void processResultOutput(const QString &result, const QString &message);
+    void processLocationOutput(const QString &fileWithLine);
+    void processSummaryFinishOutput();
+    // helper functions
     QtTestResult *createDefaultResult() const;
+    void sendCompleteInformation();
+    void sendMessageCurrentTest();
+    void sendStartMessage(bool isFunction);
+    void sendFinishMessage(bool isFunction);
+    void handleAndSendConfigMessage(const QRegExp &config);
 
     enum CDATAMode
     {
@@ -70,6 +88,8 @@ private:
     int m_lineNumber = 0;
     QString m_duration;
     QXmlStreamReader m_xmlReader;
+    OutputMode m_mode = XML;
+
 };
 
 } // namespace Internal

src/plugins/autotest/qtest/qttestsettings.cpp

@@ -30,6 +30,7 @@ namespace Internal {
 
 static const char metricsKey[]           = "Metrics";
 static const char noCrashhandlerKey[]    = "NoCrashhandlerOnDebug";
+static const char useXMLOutputKey[]      = "UseXMLOutput";
 
 static MetricsType intToMetrics(int value)
 {
@@ -58,12 +59,14 @@ void QtTestSettings::fromFrameworkSettings(const QSettings *s)
 {
     metrics = intToMetrics(s->value(metricsKey, Walltime).toInt());
     noCrashHandler = s->value(noCrashhandlerKey, true).toBool();
+    useXMLOutput = s->value(useXMLOutputKey, true).toBool();
 }
 
 void QtTestSettings::toFrameworkSettings(QSettings *s) const
 {
     s->setValue(metricsKey, metrics);
     s->setValue(noCrashhandlerKey, noCrashHandler);
+    s->setValue(useXMLOutputKey, useXMLOutput);
 }
 
 QString QtTestSettings::metricsTypeToOption(const MetricsType type)

src/plugins/autotest/qtest/qttestsettings.h

@@ -48,6 +48,7 @@ public:
 
     MetricsType metrics = Walltime;
     bool noCrashHandler = true;
+    bool useXMLOutput = true;
 
 protected:
     void fromFrameworkSettings(const QSettings *s) override;

src/plugins/autotest/qtest/qttestsettingspage.cpp

@@ -46,6 +46,7 @@ QtTestSettingsWidget::QtTestSettingsWidget(QWidget *parent)
 void QtTestSettingsWidget::setSettings(const QtTestSettings &settings)
 {
     m_ui.disableCrashhandlerCB->setChecked(settings.noCrashHandler);
+    m_ui.useXMLOutputCB->setChecked(settings.useXMLOutput);
     switch (settings.metrics) {
     case MetricsType::Walltime:
         m_ui.walltimeRB->setChecked(true);
@@ -72,6 +73,7 @@ QtTestSettings QtTestSettingsWidget::settings() const
     QtTestSettings result;
 
     result.noCrashHandler = m_ui.disableCrashhandlerCB->isChecked();
+    result.useXMLOutput = m_ui.useXMLOutputCB->isChecked();
     if (m_ui.walltimeRB->isChecked())
         result.metrics = MetricsType::Walltime;
     else if (m_ui.tickcounterRB->isChecked())

src/plugins/autotest/qtest/qttestsettingspage.ui

@@ -31,6 +31,21 @@
          </property>
         </widget>
        </item>
+       <item>
+        <widget class="QCheckBox" name="useXMLOutputCB">
+         <property name="toolTip">
+          <string>XML output recommended as it avoids parsing issues, while plain text is more human readable.
+
+Warning: Plain text output is missing some information (e.g. duration)</string>
+         </property>
+         <property name="text">
+          <string>Use XML output</string>
+         </property>
+         <property name="checked">
+          <bool>true</bool>
+         </property>
+        </widget>
+       </item>
        <item>
         <widget class="QGroupBox" name="groupBox">
          <property name="title">

src/plugins/autotest/quick/quicktestconfiguration.cpp

@@ -35,7 +35,16 @@ namespace Internal {
 TestOutputReader *QuickTestConfiguration::outputReader(const QFutureInterface<TestResultPtr> &fi,
                                                        QProcess *app) const
 {
-    return new QtTestOutputReader(fi, app, buildDirectory());
+    static const Core::Id id
+            = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
+    TestFrameworkManager *manager = TestFrameworkManager::instance();
+    auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
+    if (qtSettings.isNull())
+        return nullptr;
+    if (qtSettings->useXMLOutput)
+        return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::XML);
+    else
+        return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::PlainText);
 }
 
 QStringList QuickTestConfiguration::argumentsForTestRunner() const
@@ -43,14 +52,15 @@ QStringList QuickTestConfiguration::argumentsForTestRunner() const
     static const Core::Id id
             = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
 
-    QStringList arguments("-xml");
-    if (testCases().count())
-        arguments << testCases();
-
+    QStringList arguments;
     TestFrameworkManager *manager = TestFrameworkManager::instance();
     auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
     if (qtSettings.isNull())
         return arguments;
+    if (qtSettings->useXMLOutput)
+        arguments << "-xml";
+    if (testCases().count())
+        arguments << testCases();
 
     const QString &metricsOption = QtTestSettings::metricsTypeToOption(qtSettings->metrics);
     if (!metricsOption.isEmpty())

src/plugins/autotest/testresult.cpp

@@ -56,7 +56,7 @@ Result::Type TestResult::resultFromString(const QString &resultString)
 {
     if (resultString == "pass")
         return Result::Pass;
-    if (resultString == "fail")
+    if (resultString == "fail" || resultString == "fail!")
         return Result::Fail;
     if (resultString == "xfail")
         return Result::ExpectedFail;
@@ -64,11 +64,13 @@ Result::Type TestResult::resultFromString(const QString &resultString)
         return Result::UnexpectedPass;
     if (resultString == "skip")
         return Result::Skip;
+    if (resultString == "result")
+        return Result::Benchmark;
     if (resultString == "qdebug")
         return Result::MessageDebug;
-    if (resultString == "qinfo")
+    if (resultString == "qinfo" || resultString == "info")
         return Result::MessageInfo;
-    if (resultString == "warn" || resultString == "qwarn")
+    if (resultString == "warn" || resultString == "qwarn" || resultString == "warning")
         return Result::MessageWarn;
     if (resultString == "qfatal")
         return Result::MessageFatal;