testoutputreader.cpp 18 KB
Newer Older
Tim Jenssen's avatar
Tim Jenssen committed
1
2
/****************************************************************************
**
Christian Stenger's avatar
Christian Stenger committed
3
** Copyright (C) 2015 The Qt Company Ltd
Tim Jenssen's avatar
Tim Jenssen committed
4
** All rights reserved.
Christian Stenger's avatar
Christian Stenger committed
5
6
** For any questions to The Qt Company, please use contact form at
** http://www.qt.io/contact-us
Tim Jenssen's avatar
Tim Jenssen committed
7
8
9
10
11
12
**
** This file is part of the Qt Creator Enterprise Auto Test Add-on.
**
** Licensees holding valid Qt Enterprise licenses may use this file in
** accordance with the Qt Enterprise License Agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
Christian Stenger's avatar
Christian Stenger committed
13
** a written agreement between you and The Qt Company.
Tim Jenssen's avatar
Tim Jenssen committed
14
15
**
** If you have questions regarding the use of this file, please use
Christian Stenger's avatar
Christian Stenger committed
16
** contact form at http://www.qt.io/contact-us
Tim Jenssen's avatar
Tim Jenssen committed
17
18
19
**
****************************************************************************/

20
#include "testoutputreader.h"
Tim Jenssen's avatar
Tim Jenssen committed
21
22
#include "testresult.h"

23
#include <utils/hostosinfo.h>
24
#include <utils/qtcassert.h>
25

26
#include <QDebug>
27
#include <QRegExp>
Tim Jenssen's avatar
Tim Jenssen committed
28
29
30
#include <QProcess>
#include <QFileInfo>
#include <QDir>
31
#include <QXmlStreamReader>
Tim Jenssen's avatar
Tim Jenssen committed
32
33
34
35

namespace Autotest {
namespace Internal {

36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
static QString decode(const QString& original)
{
    QString result(original);
    static QRegExp regex(QLatin1String("&#((x[0-9A-F]+)|([0-9]+));"), Qt::CaseInsensitive);
    regex.setMinimal(true);

    int pos = 0;
    while ((pos = regex.indexIn(original, pos)) != -1) {
        const QString value = regex.cap(1);
        if (value.startsWith(QLatin1Char('x')))
            result.replace(regex.cap(0), QChar(value.mid(1).toInt(0, 16)));
        else
            result.replace(regex.cap(0), QChar(value.toInt(0, 10)));
        pos += regex.matchedLength();
    }

    return result;
}

55
56
57
58
59
60
61
62
63
64
65
static QString constructSourceFilePath(const QString &path, const QString &filePath,
                                       const QString &app)
{
    if (Utils::HostOsInfo::isMacHost() && !app.isEmpty()) {
        const QString fileName(QFileInfo(app).fileName());
        return QFileInfo(path.left(path.lastIndexOf(fileName + QLatin1String(".app"))), filePath)
                   .canonicalFilePath();
    }
    return QFileInfo(path, filePath).canonicalFilePath();
}

Tim Jenssen's avatar
Tim Jenssen committed
66
67
68
69
// adapted from qplaintestlogger.cpp
static QString formatResult(double value)
{
    //NAN is not supported with visual studio 2010
70
71
    if (value < 0)// || value == NAN)
        return QLatin1String("NAN");
Tim Jenssen's avatar
Tim Jenssen committed
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
    if (value == 0)
        return QLatin1String("0");

    int significantDigits = 0;
    qreal divisor = 1;

    while (value / divisor >= 1) {
        divisor *= 10;
        ++significantDigits;
    }

    QString beforeDecimalPoint = QString::number(value, 'f', 0);
    QString afterDecimalPoint = QString::number(value, 'f', 20);
    afterDecimalPoint.remove(0, beforeDecimalPoint.count() + 1);

    const int beforeUse = qMin(beforeDecimalPoint.count(), significantDigits);
    const int beforeRemove = beforeDecimalPoint.count() - beforeUse;

    beforeDecimalPoint.chop(beforeRemove);
    for (int i = 0; i < beforeRemove; ++i)
        beforeDecimalPoint.append(QLatin1Char('0'));

    int afterUse = significantDigits - beforeUse;
    if (beforeDecimalPoint == QLatin1String("0") && !afterDecimalPoint.isEmpty()) {
        ++afterUse;
        int i = 0;
        while (i < afterDecimalPoint.count() && afterDecimalPoint.at(i) == QLatin1Char('0'))
            ++i;
        afterUse += i;
    }

    const int afterRemove = afterDecimalPoint.count() - afterUse;
    afterDecimalPoint.chop(afterRemove);

    QString result = beforeDecimalPoint;
    if (afterUse > 0)
        result.append(QLatin1Char('.'));
    result += afterDecimalPoint;

    return result;
}

114
static QString constructBenchmarkInformation(const QString &metric, double value, int iterations)
Tim Jenssen's avatar
Tim Jenssen committed
115
{
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
    QString metricsText;
    if (metric == QLatin1String("WalltimeMilliseconds"))         // default
        metricsText = QLatin1String("msecs");
    else if (metric == QLatin1String("CPUTicks"))                // -tickcounter
        metricsText = QLatin1String("CPU ticks");
    else if (metric == QLatin1String("Events"))                  // -eventcounter
        metricsText = QLatin1String("events");
    else if (metric == QLatin1String("InstructionReads"))        // -callgrind
        metricsText = QLatin1String("instruction reads");
    else if (metric == QLatin1String("CPUCycles"))               // -perf
        metricsText = QLatin1String("CPU cycles");
    return QObject::tr("%1 %2 per iteration (total: %3, iterations: %4)")
            .arg(formatResult(value))
            .arg(metricsText)
            .arg(formatResult(value * (double)iterations))
            .arg(iterations);
Tim Jenssen's avatar
Tim Jenssen committed
132
133
}

134
135
136
TestOutputReader::TestOutputReader(QProcess *testApplication, OutputType type)
    : m_testApplication(testApplication)
    , m_type(type)
Tim Jenssen's avatar
Tim Jenssen committed
137
138
139
{
}

140
141
142
143
144
enum CDATAMode {
    None,
    DataTag,
    Description,
    QtVersion,
145
    QtBuild,
146
147
    QTestVersion
};
Tim Jenssen's avatar
Tim Jenssen committed
148

149
void TestOutputReader::processOutput()
Tim Jenssen's avatar
Tim Jenssen committed
150
151
152
{
    if (!m_testApplication || m_testApplication->state() != QProcess::Running)
        return;
153
154
155
156
    static QStringList validEndTags = { QStringLiteral("Incident"),
                                        QStringLiteral("Message"),
                                        QStringLiteral("BenchmarkResult"),
                                        QStringLiteral("QtVersion"),
157
                                        QStringLiteral("QtBuild"),
158
159
                                        QStringLiteral("QTestVersion") };
    static CDATAMode cdataMode = None;
Tim Jenssen's avatar
Tim Jenssen committed
160
161
162
    static QString className;
    static QString testCase;
    static QString dataTag;
Christian Stenger's avatar
Christian Stenger committed
163
    static Result::Type result = Result::Invalid;
Tim Jenssen's avatar
Tim Jenssen committed
164
165
166
167
    static QString description;
    static QString file;
    static int lineNumber = 0;
    static QString duration;
168
    static QXmlStreamReader xmlReader;
Tim Jenssen's avatar
Tim Jenssen committed
169
170

    while (m_testApplication->canReadLine()) {
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
        xmlReader.addData(m_testApplication->readLine());
        while (!xmlReader.atEnd()) {
            QXmlStreamReader::TokenType token = xmlReader.readNext();
            switch (token) {
            case QXmlStreamReader::StartDocument:
                className.clear();
                break;
            case QXmlStreamReader::EndDocument:
                xmlReader.clear();
                return;
            case QXmlStreamReader::StartElement: {
                const QString currentTag = xmlReader.name().toString();
                if (currentTag == QStringLiteral("TestCase")) {
                    className = xmlReader.attributes().value(QStringLiteral("name")).toString();
                    QTC_ASSERT(!className.isEmpty(), continue);
                    auto testResult = new TestResult(className);
Christian Stenger's avatar
Christian Stenger committed
187
                    testResult->setResult(Result::MessageTestCaseStart);
188
189
190
191
192
193
                    testResult->setDescription(tr("Executing test case %1").arg(className));
                    testResultCreated(testResult);
                } else if (currentTag == QStringLiteral("TestFunction")) {
                    testCase = xmlReader.attributes().value(QStringLiteral("name")).toString();
                    QTC_ASSERT(!testCase.isEmpty(), continue);
                    auto testResult = new TestResult();
Christian Stenger's avatar
Christian Stenger committed
194
                    testResult->setResult(Result::MessageCurrentTest);
195
196
197
198
199
200
201
202
203
204
205
206
                    testResult->setDescription(tr("Entering test function %1::%2").arg(className,
                                                                                       testCase));
                    testResultCreated(testResult);
                } else if (currentTag == QStringLiteral("Duration")) {
                    duration = xmlReader.attributes().value(QStringLiteral("msecs")).toString();
                    QTC_ASSERT(!duration.isEmpty(), continue);
                } else if (currentTag == QStringLiteral("Message")
                           || currentTag == QStringLiteral("Incident")) {
                    dataTag.clear();
                    description.clear();
                    duration.clear();
                    file.clear();
Christian Stenger's avatar
Christian Stenger committed
207
                    result = Result::Invalid;
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
                    lineNumber = 0;
                    const QXmlStreamAttributes &attributes = xmlReader.attributes();
                    result = TestResult::resultFromString(
                                attributes.value(QStringLiteral("type")).toString());
                    file = decode(attributes.value(QStringLiteral("file")).toString());
                    if (!file.isEmpty())
                        file = constructSourceFilePath(m_testApplication->workingDirectory(), file,
                                                       m_testApplication->program());
                    lineNumber = attributes.value(QStringLiteral("line")).toInt();
                } else if (currentTag == QStringLiteral("BenchmarkResult")) {
                    const QXmlStreamAttributes &attributes = xmlReader.attributes();
                    const QString metric = attributes.value(QStringLiteral("metrics")).toString();
                    const double value = attributes.value(QStringLiteral("value")).toDouble();
                    const int iterations = attributes.value(QStringLiteral("iterations")).toInt();
                    description = constructBenchmarkInformation(metric, value, iterations);
Christian Stenger's avatar
Christian Stenger committed
223
                    result = Result::Benchmark;
224
225
226
227
228
                } else if (currentTag == QStringLiteral("DataTag")) {
                    cdataMode = DataTag;
                } else if (currentTag == QStringLiteral("Description")) {
                    cdataMode = Description;
                } else if (currentTag == QStringLiteral("QtVersion")) {
Christian Stenger's avatar
Christian Stenger committed
229
                    result = Result::MessageInternal;
230
                    cdataMode = QtVersion;
231
                } else if (currentTag == QStringLiteral("QtBuild")) {
Christian Stenger's avatar
Christian Stenger committed
232
                    result = Result::MessageInternal;
233
                    cdataMode = QtBuild;
234
                } else if (currentTag == QStringLiteral("QTestVersion")) {
Christian Stenger's avatar
Christian Stenger committed
235
                    result = Result::MessageInternal;
236
237
238
                    cdataMode = QTestVersion;
                }
                break;
Tim Jenssen's avatar
Tim Jenssen committed
239
            }
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
            case QXmlStreamReader::Characters: {
                QStringRef text = xmlReader.text().trimmed();
                if (text.isEmpty())
                    break;

                switch (cdataMode) {
                case DataTag:
                    dataTag = text.toString();
                    break;
                case Description:
                    if (!description.isEmpty())
                        description.append(QLatin1Char('\n'));
                    description.append(text);
                    break;
                case QtVersion:
                    description = tr("Qt version: %1").arg(text.toString());
                    break;
257
                case QtBuild:
Christian Stenger's avatar
Christian Stenger committed
258
                    description = tr("Qt build: %1").arg(text.toString());
259
                    break;
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
                case QTestVersion:
                    description = tr("QTest version: %1").arg(text.toString());
                    break;
                default:
                    QString message = QString::fromLatin1("unexpected cdatamode %1 for text \"%2\"")
                            .arg(cdataMode)
                            .arg(text.toString());
                    QTC_ASSERT(false, qWarning() << message);
                    break;
                }
                break;
            }
            case QXmlStreamReader::EndElement: {
                cdataMode = None;
                const QStringRef currentTag = xmlReader.name();
                if (currentTag == QStringLiteral("TestFunction")) {
                    if (!duration.isEmpty()) {
                        auto testResult = new TestResult(className);
                        testResult->setTestCase(testCase);
Christian Stenger's avatar
Christian Stenger committed
279
                        testResult->setResult(Result::MessageInternal);
280
281
282
283
                        testResult->setDescription(tr("Execution took %1 ms.").arg(duration));
                        testResultCreated(testResult);
                    }
                    emit increaseProgress();
284
                } else if (currentTag == QStringLiteral("TestCase")) {
285
                    auto testResult = new TestResult(className);
Christian Stenger's avatar
Christian Stenger committed
286
                    testResult->setResult(Result::MessageTestCaseEnd);
287
288
289
                    testResult->setDescription(
                            duration.isEmpty() ? tr("Test finished.")
                                               : tr("Test execution took %1 ms.").arg(duration));
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
                    testResultCreated(testResult);
                } else if (validEndTags.contains(currentTag.toString())) {
                    auto testResult = new TestResult(className);
                    testResult->setTestCase(testCase);
                    testResult->setDataTag(dataTag);
                    testResult->setResult(result);
                    testResult->setFileName(file);
                    testResult->setLine(lineNumber);
                    testResult->setDescription(description);
                    testResultCreated(testResult);
                }
                break;
            }
            default:
                break;
Tim Jenssen's avatar
Tim Jenssen committed
305
306
307
308
309
            }
        }
    }
}

310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
void TestOutputReader::processGTestOutput()
{
    if (!m_testApplication || m_testApplication->state() != QProcess::Running)
        return;

    static QRegExp newTestStarts(QStringLiteral("^\\[-{10}\\] \\d+ tests? from (.*)$"));
    static QRegExp testEnds(QStringLiteral("^\\[-{10}\\] \\d+ tests? from (.*) \\((.*)\\)$"));
    static QRegExp newTestSetStarts(QStringLiteral("^\\[ RUN      \\] (.*)$"));
    static QRegExp testSetSuccess(QStringLiteral("^\\[       OK \\] (.*) \\((.*)\\)$"));
    static QRegExp testSetFail(QStringLiteral("^\\\[  FAILED  \\] (.*) \\((.*)\\)$"));
    static QRegExp disabledTests(QStringLiteral("^  YOU HAVE (\\d+) DISABLED TESTS?$"));

    static QString currentTestName;
    static QString currentTestSet;
    static QString description;
    static QByteArray unprocessed;

    while (m_testApplication->canReadLine())
        unprocessed.append(m_testApplication->readLine());

    int lineBreak;
    while ((lineBreak = unprocessed.indexOf('\n')) != -1) {
        const QString line = QLatin1String(unprocessed.left(lineBreak));
        unprocessed.remove(0, lineBreak + 1);
        if (line.isEmpty()) {
            continue;
        }
        if (!line.startsWith(QLatin1Char('['))) {
            description.append(line).append(QLatin1Char('\n'));
            if (line.startsWith(QStringLiteral("Note:"))) {
                auto testResult = new TestResult();
                testResult->setResult(Result::MessageInternal);
                testResult->setDescription(line);
                testResultCreated(testResult);
                description.clear();
            } else if (disabledTests.exactMatch(line)) {
                auto testResult = new TestResult();
                testResult->setResult(Result::MessageInternal);
                int disabled = disabledTests.cap(1).toInt();
                testResult->setDescription(tr("You have %n disabled test(s).", 0, disabled));
                testResultCreated(testResult);
                description.clear();
            }
            continue;
        }

        if (testEnds.exactMatch(line)) {
            auto testResult = new TestResult(currentTestName);
            testResult->setTestCase(currentTestSet);
            testResult->setResult(Result::MessageTestCaseEnd);
            testResult->setDescription(tr("Test execution took %1").arg(testEnds.cap(2)));
            testResultCreated(testResult);
            currentTestName.clear();
            currentTestSet.clear();
        } else if (newTestStarts.exactMatch(line)) {
            currentTestName = newTestStarts.cap(1);
            auto testResult = new TestResult(currentTestName);
            testResult->setResult(Result::MessageTestCaseStart);
            testResult->setDescription(tr("Executing test case %1").arg(currentTestName));
            testResultCreated(testResult);
        } else if (newTestSetStarts.exactMatch(line)) {
            currentTestSet = newTestSetStarts.cap(1);
            auto testResult = new TestResult();
            testResult->setResult(Result::MessageCurrentTest);
            testResult->setDescription(tr("Entering test set %1").arg(currentTestSet));
            testResultCreated(testResult);
        } else if (testSetSuccess.exactMatch(line)) {
            auto testResult = new TestResult(currentTestName);
            testResult->setTestCase(currentTestSet);
            testResult->setResult(Result::Pass);
            testResultCreated(testResult);
            testResult = new TestResult(currentTestName);
            testResult->setTestCase(currentTestSet);
            testResult->setResult(Result::MessageInternal);
            testResult->setDescription(tr("Execution took %1.").arg(testSetSuccess.cap(2)));
            testResultCreated(testResult);
            emit increaseProgress();
        } else if (testSetFail.exactMatch(line)) {
            auto testResult = new TestResult(currentTestName);
            testResult->setTestCase(currentTestSet);
            testResult->setResult(Result::Fail);
            description.chop(1);
            testResult->setDescription(description);
            int firstColon = description.indexOf(QLatin1Char(':'));
            if (firstColon != -1) {
                int secondColon = description.indexOf(QLatin1Char(':'), firstColon + 1);
                QString file = constructSourceFilePath(m_testApplication->workingDirectory(),
                                                       description.left(firstColon),
                                                       m_testApplication->program());
                QString line = description.mid(firstColon + 1, secondColon - firstColon - 1);
                testResult->setFileName(file);
                testResult->setLine(line.toInt());
            }
            testResultCreated(testResult);
            description.clear();
            testResult = new TestResult(currentTestName);
            testResult->setTestCase(currentTestSet);
            testResult->setResult(Result::MessageInternal);
            testResult->setDescription(tr("Execution took %1.").arg(testSetFail.cap(2)));
            testResultCreated(testResult);
            emit increaseProgress();
        }
    }
}

Tim Jenssen's avatar
Tim Jenssen committed
415
416
} // namespace Internal
} // namespace Autotest