Commit ba61f294 authored by Christian Stenger's avatar Christian Stenger

AutoTest: Add some gtest related settings

This patch enables execution of disabled tests, repetition of
test runs and shuffling of test sets.

Change-Id: I3668afde5766d18acdf1afb7edd913655b13a76c
Reviewed-by: default avatarDavid Schulz <david.schulz@theqtcompany.com>
Reviewed-by: default avatarLeena Miettinen <riitta-leena.miettinen@theqtcompany.com>
parent a4b74730
......@@ -498,6 +498,7 @@ static void handleGTest(QFutureInterface<TestParseResult> futureInterface, const
parseResult.testCaseName = testSpec.testCaseName;
parseResult.parameterized = testSpec.parameterized;
parseResult.typed = testSpec.typed;
parseResult.disabled = testSpec.disabled;
parseResult.proFile = proFile;
parseResult.dataTagsOrTestSets.insert(QString(), result.value(testSpec));
futureInterface.reportResult(parseResult);
......
......@@ -314,6 +314,7 @@ void GTestOutputReader::processOutput()
static QRegExp testSetFail(QStringLiteral("^\\[ FAILED \\] (.*) \\((.*)\\)$"));
static QRegExp disabledTests(QStringLiteral("^ YOU HAVE (\\d+) DISABLED TESTS?$"));
static QRegExp failureLocation(QStringLiteral("^(.*):(\\d+): Failure$"));
static QRegExp iterations(QStringLiteral("^Repeating all tests \\(iteration (\\d+)\\) . . .$"));
while (m_testApplication->canReadLine()) {
if (m_futureInterface.isCanceled())
......@@ -337,7 +338,10 @@ void GTestOutputReader::processOutput()
if (!line.startsWith(QLatin1Char('['))) {
m_description.append(line).append(QLatin1Char('\n'));
if (line.startsWith(QStringLiteral("Note:"))) {
if (iterations.exactMatch(line)) {
m_iteration = iterations.cap(1).toInt();
m_description.clear();
} else if (line.startsWith(QStringLiteral("Note:"))) {
auto testResult = new GTestResult();
testResult->setResult(Result::MessageInternal);
testResult->setDescription(line);
......@@ -366,8 +370,14 @@ void GTestOutputReader::processOutput()
} else if (newTestStarts.exactMatch(line)) {
m_currentTestName = newTestStarts.cap(1);
auto testResult = new GTestResult(m_currentTestName);
testResult->setResult(Result::MessageTestCaseStart);
testResult->setDescription(tr("Executing test case %1").arg(m_currentTestName));
if (m_iteration > 1) {
testResult->setResult(Result::MessageTestCaseRepetition);
testResult->setDescription(tr("Repeating test case %1 (iteration %2)")
.arg(m_currentTestName).arg(m_iteration));
} else {
testResult->setResult(Result::MessageTestCaseStart);
testResult->setDescription(tr("Executing test case %1").arg(m_currentTestName));
}
m_futureInterface.reportResult(testResult);
} else if (newTestSetStarts.exactMatch(line)) {
m_currentTestSet = newTestSetStarts.cap(1);
......
......@@ -100,6 +100,7 @@ private:
QString m_currentTestSet;
QString m_description;
QByteArray m_unprocessed;
int m_iteration = 0;
};
......
......@@ -56,6 +56,7 @@ enum Type {
MessageTestCaseWarn,
MessageTestCaseFail,
MessageTestCaseEnd,
MessageTestCaseRepetition,
MessageCurrentTest, INTERNAL_MESSAGES_END = MessageCurrentTest,
Invalid,
......
......@@ -292,7 +292,8 @@ void TestResultFilterModel::enableAllResultTypes()
<< Result::BlacklistedFail << Result::Benchmark
<< Result::MessageCurrentTest << Result::MessageTestCaseStart
<< Result::MessageTestCaseSuccess << Result::MessageTestCaseWarn
<< Result::MessageTestCaseFail << Result::MessageTestCaseEnd;
<< Result::MessageTestCaseFail << Result::MessageTestCaseEnd
<< Result::MessageTestCaseRepetition;
invalidateFilter();
}
......
......@@ -118,9 +118,11 @@ void TestRunner::setSelectedTests(const QList<TestConfiguration *> &selected)
}
static void performTestRun(QFutureInterface<TestResult *> &futureInterface,
const QList<TestConfiguration *> selectedTests, const int timeout,
const QString metricsOption)
const QList<TestConfiguration *> selectedTests,
const TestSettings &settings)
{
const int timeout = settings.timeout;
const QString &metricsOption = TestSettings::metricsTypeToOption(settings.metrics);
QEventLoop eventLoop;
int testCaseCount = 0;
foreach (TestConfiguration *config, selectedTests) {
......@@ -177,13 +179,21 @@ static void performTestRun(QFutureInterface<TestResult *> &futureInterface,
argumentList << testConfiguration->testCases();
testProcess.setArguments(argumentList);
} else { // TestTypeGTest
QStringList argumentList;
const QStringList &testSets = testConfiguration->testCases();
if (testSets.size()) {
QStringList argumentList;
argumentList << QLatin1String("--gtest_filter=")
+ testSets.join(QLatin1Char(':'));
testProcess.setArguments(argumentList);
}
if (settings.gtestRunDisabled)
argumentList << QLatin1String("--gtest_also_run_disabled_tests");
if (settings.gtestRepeat)
argumentList << QString::fromLatin1("--gtest_repeat=%1").arg(settings.gtestIterations);
if (settings.gtestShuffle) {
argumentList << QLatin1String("--gtest_shuffle");
argumentList << QString::fromLatin1("--gtest_random_seed=%1").arg(settings.gtestSeed);
}
testProcess.setArguments(argumentList);
}
testProcess.setWorkingDirectory(testConfiguration->workingDirectory());
......@@ -282,11 +292,8 @@ void TestRunner::prepareToRunTests()
void TestRunner::runTests()
{
const QSharedPointer<TestSettings> settings = AutotestPlugin::instance()->settings();
const QString &metricsOption = TestSettings::metricsTypeToOption(settings->metrics);
QFuture<TestResult *> future = Utils::runAsync(&performTestRun, m_selectedTests,
settings->timeout, metricsOption);
*AutotestPlugin::instance()->settings());
m_futureWatcher.setFuture(future);
Core::ProgressManager::addTask(future, tr("Running Tests"), Autotest::Constants::TASK_INDEX);
}
......
......@@ -38,6 +38,11 @@ static const char omitRunConfigWarnKey[] = "OmitRCWarnings";
static const char limitResultOutputKey[] = "LimitResultOutput";
static const char autoScrollKey[] = "AutoScrollResults";
static const char alwaysParseKey[] = "AlwaysParse";
static const char gtestRunDisabledKey[] = "RunDisabledGTests";
static const char gtestRepeatKey[] = "RepeatGTests";
static const char gtestShuffleKey[] = "ShuffleGTests";
static const char gtestIterationsKey[] = "IterationsGTests";
static const char gtestSeedKey[] = "SeedGTests";
static const int defaultTimeout = 60000;
TestSettings::TestSettings()
......@@ -56,6 +61,11 @@ void TestSettings::toSettings(QSettings *s) const
s->setValue(QLatin1String(limitResultOutputKey), limitResultOutput);
s->setValue(QLatin1String(autoScrollKey), autoScroll);
s->setValue(QLatin1String(alwaysParseKey), alwaysParse);
s->setValue(QLatin1String(gtestRunDisabledKey), gtestRunDisabled);
s->setValue(QLatin1String(gtestRepeatKey), gtestRepeat);
s->setValue(QLatin1String(gtestShuffleKey), gtestShuffle);
s->setValue(QLatin1String(gtestIterationsKey), gtestIterations);
s->setValue(QLatin1String(gtestSeedKey), gtestSeed);
s->endGroup();
}
......@@ -87,6 +97,11 @@ void TestSettings::fromSettings(const QSettings *s)
limitResultOutput = s->value(root + QLatin1String(limitResultOutputKey), true).toBool();
autoScroll = s->value(root + QLatin1String(autoScrollKey), true).toBool();
alwaysParse = s->value(root + QLatin1String(alwaysParseKey), false).toBool();
gtestRunDisabled = s->value(root + QLatin1String(gtestRunDisabledKey), false).toBool();
gtestRepeat = s->value(root + QLatin1String(gtestRepeatKey), false).toBool();
gtestShuffle = s->value(root + QLatin1String(gtestShuffleKey), false).toBool();
gtestIterations = s->value(root + QLatin1String(gtestIterationsKey), 1).toInt();
gtestSeed = s->value(root + QLatin1String(gtestSeedKey), 0).toInt();
}
bool TestSettings::equals(const TestSettings &rhs) const
......@@ -96,7 +111,10 @@ bool TestSettings::equals(const TestSettings &rhs) const
&& omitRunConfigWarn == rhs.omitRunConfigWarn
&& limitResultOutput == rhs.limitResultOutput
&& autoScroll == rhs.autoScroll
&& alwaysParse == rhs.alwaysParse;
&& alwaysParse == rhs.alwaysParse
&& gtestRunDisabled == rhs.gtestRunDisabled
&& gtestRepeat == rhs.gtestRepeat && gtestIterations == rhs.gtestIterations
&& gtestShuffle == rhs.gtestShuffle && gtestSeed == rhs.gtestSeed;
}
QString TestSettings::metricsTypeToOption(const MetricsType type)
......
......@@ -52,12 +52,17 @@ struct TestSettings
static QString metricsTypeToOption(const MetricsType type);
int timeout;
int gtestIterations;
int gtestSeed;
MetricsType metrics;
bool omitInternalMssg;
bool omitRunConfigWarn;
bool limitResultOutput;
bool autoScroll;
bool alwaysParse;
bool gtestRunDisabled;
bool gtestShuffle;
bool gtestRepeat;
};
inline bool operator==(const TestSettings &s1, const TestSettings &s2) { return s1.equals(s2); }
......
......@@ -41,6 +41,9 @@ TestSettingsWidget::TestSettingsWidget(QWidget *parent)
m_ui.setupUi(this);
m_ui.callgrindRB->setEnabled(Utils::HostOsInfo::isAnyUnixHost()); // valgrind available on UNIX
m_ui.perfRB->setEnabled(Utils::HostOsInfo::isLinuxHost()); // according to docs perf Linux only
connect(m_ui.repeatGTestsCB, &QCheckBox::toggled, m_ui.repetitionSpin, &QSpinBox::setEnabled);
connect(m_ui.shuffleGTestsCB, &QCheckBox::toggled, m_ui.seedSpin, &QSpinBox::setEnabled);
}
void TestSettingsWidget::setSettings(const TestSettings &settings)
......@@ -51,6 +54,11 @@ void TestSettingsWidget::setSettings(const TestSettings &settings)
m_ui.limitResultOutputCB->setChecked(settings.limitResultOutput);
m_ui.autoScrollCB->setChecked(settings.autoScroll);
m_ui.alwaysParseCB->setChecked(settings.alwaysParse);
m_ui.runDisabledGTestsCB->setChecked(settings.gtestRunDisabled);
m_ui.repeatGTestsCB->setChecked(settings.gtestRepeat);
m_ui.shuffleGTestsCB->setChecked(settings.gtestShuffle);
m_ui.repetitionSpin->setValue(settings.gtestIterations);
m_ui.seedSpin->setValue(settings.gtestSeed);
switch (settings.metrics) {
case MetricsType::Walltime:
......@@ -82,6 +90,11 @@ TestSettings TestSettingsWidget::settings() const
result.limitResultOutput = m_ui.limitResultOutputCB->isChecked();
result.autoScroll = m_ui.autoScrollCB->isChecked();
result.alwaysParse = m_ui.alwaysParseCB->isChecked();
result.gtestRunDisabled = m_ui.runDisabledGTestsCB->isChecked();
result.gtestRepeat = m_ui.repeatGTestsCB->isChecked();
result.gtestShuffle = m_ui.shuffleGTestsCB->isChecked();
result.gtestIterations = m_ui.repetitionSpin->value();
result.gtestSeed = m_ui.seedSpin->value();
if (m_ui.walltimeRB->isChecked())
result.metrics = MetricsType::Walltime;
......
......@@ -6,8 +6,8 @@
<rect>
<x>0</x>
<y>0</y>
<width>451</width>
<height>271</height>
<width>485</width>
<height>404</height>
</rect>
</property>
<property name="windowTitle">
......@@ -150,6 +150,134 @@
</layout>
</widget>
</item>
<item>
<widget class="QGroupBox" name="groupBox_3">
<property name="title">
<string>Google Test</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout_6">
<item>
<widget class="QCheckBox" name="runDisabledGTestsCB">
<property name="toolTip">
<string>Executes disabled tests when performing a test run.</string>
</property>
<property name="text">
<string>Run disabled tests</string>
</property>
</widget>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_3">
<item>
<widget class="QCheckBox" name="repeatGTestsCB">
<property name="toolTip">
<string>Repeats a test run (you might be required to increase the timeout to avoid canceling the tests).</string>
</property>
<property name="text">
<string>Repeat tests</string>
</property>
</widget>
</item>
<item>
<widget class="QLabel" name="label_3">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="text">
<string>Iterations:</string>
</property>
</widget>
</item>
<item>
<widget class="QSpinBox" name="repetitionSpin">
<property name="enabled">
<bool>false</bool>
</property>
<property name="minimum">
<number>1</number>
</property>
<property name="maximum">
<number>9999</number>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer_3">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>40</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
</layout>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_4">
<item>
<widget class="QCheckBox" name="shuffleGTestsCB">
<property name="toolTip">
<string>Shuffle tests automatically on every iteration by the given seed.</string>
</property>
<property name="text">
<string>Shuffle tests</string>
</property>
</widget>
</item>
<item>
<widget class="QLabel" name="label_2">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="text">
<string>Seed:</string>
</property>
</widget>
</item>
<item>
<widget class="QSpinBox" name="seedSpin">
<property name="enabled">
<bool>false</bool>
</property>
<property name="toolTip">
<string>A seed of 0 generates a seed based on the current timestamp.</string>
</property>
<property name="specialValueText">
<string/>
</property>
<property name="maximum">
<number>99999</number>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer_4">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>40</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
</layout>
</item>
</layout>
</widget>
</item>
<item>
<spacer name="verticalSpacer">
<property name="orientation">
......
......@@ -388,6 +388,8 @@ GoogleTestTreeItem *GoogleTestTreeItem::createTestItem(const TestParseResult &re
item->setState(Parameterized);
if (result.typed)
item->setState(Typed);
if (result.disabled)
item->setState(Disabled);
foreach (const TestCodeLocationAndType &location, result.dataTagsOrTestSets.first())
item->appendChild(createTestSetItem(result, location));
return item;
......@@ -408,10 +410,14 @@ GoogleTestTreeItem *GoogleTestTreeItem::createTestSetItem(const TestParseResult
QVariant GoogleTestTreeItem::data(int column, int role) const
{
switch (role) {
case Qt::DisplayRole:
if (type() == TestCase)
return QVariant(name() + nameSuffix());
break;
case Qt::DisplayRole: {
if (type() == TestTreeItem::Root)
return TestTreeItem::data(column, role);
const QString &displayName = (m_state & GoogleTestTreeItem::Disabled)
? name().mid(9) : name();
return QVariant(displayName + nameSuffix());
}
case StateRole:
return (int)m_state;
default:
......
......@@ -212,6 +212,7 @@ struct GTestCaseSpec
QString testCaseName;
bool parameterized;
bool typed;
bool disabled;
};
} // namespace Internal
......
......@@ -160,6 +160,7 @@ struct TestParseResult
unsigned column = 0;
bool parameterized = false;
bool typed = false;
bool disabled = false;
QMap<QString, TestCodeLocationAndType> functions;
QMap<QString, TestCodeLocationList> dataTagsOrTestSets;
};
......
......@@ -383,16 +383,17 @@ bool GTestVisitor::visit(CPlusPlus::FunctionDefinitionAST *ast)
m_document->translationUnit()->getTokenStartPosition(token, &line, &column);
TestCodeLocationAndType locationAndType;
locationAndType.m_name = disabled ? testName.mid(9) : testName;
locationAndType.m_name = testName;
locationAndType.m_line = line;
locationAndType.m_column = column - 1;
locationAndType.m_type = TestTreeItem::TestFunctionOrSet;
locationAndType.m_state = (disabled || disabledCase) ? GoogleTestTreeItem::Disabled
: GoogleTestTreeItem::Enabled;
locationAndType.m_state = disabled ? GoogleTestTreeItem::Disabled
: GoogleTestTreeItem::Enabled;
GTestCaseSpec spec;
spec.testCaseName = disabledCase ? testCaseName.mid(9) : testCaseName;
spec.testCaseName = testCaseName;
spec.parameterized = TestUtils::isGTestParameterized(prettyName);
spec.typed = TestUtils::isGTestTyped(prettyName);
spec.disabled = disabledCase;
m_gtestFunctions[spec].append(locationAndType);
}
......
......@@ -134,10 +134,14 @@ inline bool operator<(const GTestCaseSpec &spec1, const GTestCaseSpec &spec2)
if (spec1.testCaseName != spec2.testCaseName)
return spec1.testCaseName < spec2.testCaseName;
if (spec1.parameterized == spec2.parameterized) {
if (spec1.typed == spec2.typed)
return false;
else
if (spec1.typed == spec2.typed) {
if (spec1.disabled == spec2.disabled)
return false;
else
return !spec1.disabled;
} else {
return !spec1.typed;
}
} else {
return !spec1.parameterized;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment