Commit ca1d1dfb authored by Nikolai Kosjar's avatar Nikolai Kosjar

Clang: Add possibility to "pgo-train" libclang with a batch file

This allows to start Qt Creator in batch processing mode:

  $ export QTC_CLANG_BATCH=/path/to/file
  $ export QT_LOGGING_RULES=qtc.clangcodemodel.batch=true
  $ ./qtcreator -load ClangCodeModel

The batch file will be executed and Qt Creator will exit. Advanced
logging output can be activated as stated above.

Note that it is required that the project was already configured/set up
properly with the used settingspath, otherwise the wrong configuration
will be taken or a pop-up dialog will block the execution.

A small example follows that covers all the understood and so far needed
batch file commands in order to train libclang for profile guided
optimization. ${PWD} expands to the directory of the batch file.

    openProject "${PWD}/calendarwidget.pro"

    # Initial parsing
    openDocument "${PWD}/window.cpp"
    closeAllDocuments
    openDocument "${PWD}/window.cpp"

    # Reparse
    setCursor 478 1
    insertText " "
    insertText " "
    insertText " "

    # Completion
    complete
    complete
    complete

    # Member completion
    insertText "comboBox->"
    complete
    complete
    complete

    # Wait in order to inspect the result
    processEvents 3000

Change-Id: I7dc5dddc6752272ecc2fb4f30497b17cee3f9a9f
Reviewed-by: default avatarhjk <hjk@qt.io>
Reviewed-by: David Schulz's avatarDavid Schulz <david.schulz@qt.io>
parent f8ad72de
/****************************************************************************
**
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "clangautomationutils.h"
#include "clangcompletionassistinterface.h"
#include "clangcompletionassistprovider.h"
#include <texteditor/codeassist/assistinterface.h>
#include <texteditor/codeassist/assistproposalitem.h>
#include <texteditor/codeassist/completionassistprovider.h>
#include <texteditor/codeassist/genericproposalmodel.h>
#include <texteditor/codeassist/iassistprocessor.h>
#include <texteditor/codeassist/iassistproposal.h>
#include <texteditor/textdocument.h>
#include <texteditor/texteditor.h>
#include <utils/qtcassert.h>
#include <QCoreApplication>
#include <QElapsedTimer>
namespace ClangCodeModel {
namespace Internal {
class WaitForAsyncCompletions
{
public:
enum WaitResult { GotResults, GotInvalidResults, Timeout };
WaitResult wait(TextEditor::IAssistProcessor *processor,
TextEditor::AssistInterface *assistInterface,
int timeoutInMs)
{
QTC_ASSERT(processor, return Timeout);
QTC_ASSERT(assistInterface, return Timeout);
bool gotResults = false;
processor->setAsyncCompletionAvailableHandler(
[this, &gotResults] (TextEditor::IAssistProposal *proposal) {
QTC_ASSERT(proposal, return);
proposalModel = proposal->model();
delete proposal;
gotResults = true;
});
// Are there any immediate results?
if (TextEditor::IAssistProposal *proposal = processor->perform(assistInterface)) {
delete processor;
proposalModel = proposal->model();
delete proposal;
QTC_ASSERT(proposalModel, return GotInvalidResults);
return GotResults;
}
// There are not any, so wait for async results.
QElapsedTimer timer;
timer.start();
while (!gotResults) {
if (timer.elapsed() >= timeoutInMs)
return Timeout;
QCoreApplication::processEvents();
}
return proposalModel ? GotResults : GotInvalidResults;
}
public:
TextEditor::IAssistProposalModel *proposalModel;
};
static const CppTools::ProjectPartHeaderPaths toHeaderPaths(const QStringList &paths)
{
using namespace CppTools;
ProjectPartHeaderPaths result;
foreach (const QString &path, paths)
result << ProjectPartHeaderPath(path, ProjectPartHeaderPath::IncludePath);
return result;
}
ProposalModel completionResults(TextEditor::BaseTextEditor *textEditor,
const QStringList &includePaths,
int timeOutInMs)
{
using namespace TextEditor;
TextEditorWidget *textEditorWidget = qobject_cast<TextEditorWidget *>(textEditor->widget());
QTC_ASSERT(textEditorWidget, return ProposalModel());
AssistInterface *assistInterface = textEditorWidget->createAssistInterface(
TextEditor::Completion, TextEditor::ExplicitlyInvoked);
QTC_ASSERT(assistInterface, return ProposalModel());
if (!includePaths.isEmpty()) {
auto clangAssistInterface = static_cast<ClangCompletionAssistInterface *>(assistInterface);
clangAssistInterface->setHeaderPaths(toHeaderPaths(includePaths));
}
CompletionAssistProvider *assistProvider
= textEditor->textDocument()->completionAssistProvider();
QTC_ASSERT(qobject_cast<ClangCompletionAssistProvider *>(assistProvider),
return ProposalModel());
QTC_ASSERT(assistProvider, return ProposalModel());
QTC_ASSERT(assistProvider->runType() == IAssistProvider::Asynchronous, return ProposalModel());
IAssistProcessor *processor = assistProvider->createProcessor();
QTC_ASSERT(processor, return ProposalModel());
WaitForAsyncCompletions waitForCompletions;
const WaitForAsyncCompletions::WaitResult result = waitForCompletions.wait(processor,
assistInterface,
timeOutInMs);
QTC_ASSERT(result == WaitForAsyncCompletions::GotResults, return ProposalModel());
return QSharedPointer<TextEditor::IAssistProposalModel>(waitForCompletions.proposalModel);
}
} // namespace Internal
} // namespace ClangCodeModel
/****************************************************************************
**
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include <QString>
#include <QSharedPointer>
namespace TextEditor {
class BaseTextEditor;
class IAssistProposalModel;
}
namespace ClangCodeModel {
namespace Internal {
using ProposalModel = QSharedPointer<TextEditor::IAssistProposalModel>;
ProposalModel completionResults(TextEditor::BaseTextEditor *textEditor,
const QStringList &includePaths = QStringList(),
int timeOutInMs = 10000);
} // namespace Internal
} // namespace ClangCodeModel
/****************************************************************************
**
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "clangbatchfileprocessor.h"
#include "clangautomationutils.h"
#include <clangcodemodel/clangeditordocumentprocessor.h>
#include <coreplugin/editormanager/editormanager.h>
#include <coreplugin/editormanager/ieditor.h>
#include <coreplugin/icore.h>
#include <cpptools/cpptoolsreuse.h>
#include <cpptools/cpptoolstestcase.h>
#include <cpptools/modelmanagertesthelper.h>
#include <cpptools/projectinfo.h>
#include <projectexplorer/projectexplorer.h>
#include <texteditor/codeassist/assistinterface.h>
#include <texteditor/codeassist/assistproposalitem.h>
#include <texteditor/codeassist/completionassistprovider.h>
#include <texteditor/codeassist/genericproposalmodel.h>
#include <texteditor/codeassist/iassistprocessor.h>
#include <texteditor/codeassist/iassistproposal.h>
#include <texteditor/textdocument.h>
#include <texteditor/texteditor.h>
#include <utils/executeondestruction.h>
#include <utils/qtcassert.h>
#include <QDebug>
#include <QFileInfo>
#include <QLoggingCategory>
#include <QSharedPointer>
#include <QString>
#include <QtTest>
using namespace ClangBackEnd;
using namespace ClangCodeModel;
using namespace ClangCodeModel::Internal;
using namespace ProjectExplorer;
static Q_LOGGING_CATEGORY(debug, "qtc.clangcodemodel.batch");
static int timeOutFromEnvironmentVariable()
{
const QByteArray timeoutAsByteArray = qgetenv("QTC_CLANG_BATCH_TIMEOUT");
bool isConversionOk = false;
const int intervalAsInt = timeoutAsByteArray.toInt(&isConversionOk);
if (!isConversionOk) {
qCDebug(debug, "Environment variable QTC_CLANG_BATCH_TIMEOUT is not set, assuming 30000.");
return 30000;
}
return intervalAsInt;
}
static int timeOutInMs()
{
static int timeOut = timeOutFromEnvironmentVariable();
return timeOut;
}
namespace {
class BatchFileLineTokenizer
{
public:
BatchFileLineTokenizer(const QString &line);
QString nextToken();
private:
const QChar *advanceToTokenBegin();
const QChar *advanceToTokenEnd();
bool atEnd() const;
bool atWhiteSpace() const;
bool atQuotationMark() const;
private:
bool m_isWithinQuotation = false;
QString m_line;
const QChar *m_currentChar;
};
BatchFileLineTokenizer::BatchFileLineTokenizer(const QString &line)
: m_line(line)
, m_currentChar(m_line.unicode())
{
}
QString BatchFileLineTokenizer::nextToken()
{
if (const QChar *tokenBegin = advanceToTokenBegin()) {
if (const QChar *tokenEnd = advanceToTokenEnd()) {
const int length = tokenEnd - tokenBegin;
return QString(tokenBegin, length);
}
}
return QString();
}
const QChar *BatchFileLineTokenizer::advanceToTokenBegin()
{
m_isWithinQuotation = false;
forever {
if (atEnd())
return 0;
if (atQuotationMark()) {
m_isWithinQuotation = true;
++m_currentChar;
return m_currentChar;
}
if (!atWhiteSpace())
return m_currentChar;
++m_currentChar;
}
}
const QChar *BatchFileLineTokenizer::advanceToTokenEnd()
{
forever {
if (m_isWithinQuotation) {
if (atEnd()) {
qWarning("ClangBatchFileProcessor: error: unfinished quotation.");
return 0;
}
if (atQuotationMark())
return m_currentChar++;
} else if (atWhiteSpace() || atEnd()) {
return m_currentChar;
}
++m_currentChar;
}
}
bool BatchFileLineTokenizer::atEnd() const
{
return *m_currentChar == QLatin1Char('\0');
}
bool BatchFileLineTokenizer::atWhiteSpace() const
{
return *m_currentChar == ' '
|| *m_currentChar == '\t'
|| *m_currentChar == '\n';
}
bool BatchFileLineTokenizer::atQuotationMark() const
{
return *m_currentChar == '"';
}
struct CommandContext {
QString filePath;
int lineNumber = -1;
};
class Command
{
public:
using Ptr = QSharedPointer<Command>;
public:
Command(const CommandContext &context) : m_commandContext(context) {}
virtual ~Command() {}
const CommandContext &context() const { return m_commandContext; }
virtual bool run() { return true; }
private:
const CommandContext m_commandContext;
};
class OpenProjectCommand : public Command
{
public:
OpenProjectCommand(const CommandContext &context,
const QString &projectFilePath);
bool run() override;
static Command::Ptr parse(BatchFileLineTokenizer &arguments,
const CommandContext &context);
private:
QString m_projectFilePath;
};
OpenProjectCommand::OpenProjectCommand(const CommandContext &context,
const QString &projectFilePath)
: Command(context)
, m_projectFilePath(projectFilePath)
{
}
bool OpenProjectCommand::run()
{
qCDebug(debug) << "line" << context().lineNumber << "OpenProjectCommand" << m_projectFilePath;
const ProjectExplorerPlugin::OpenProjectResult openProjectSucceeded
= ProjectExplorerPlugin::openProject(m_projectFilePath);
QTC_ASSERT(openProjectSucceeded, return false);
Project *project = openProjectSucceeded.project();
project->configureAsExampleProject({});
return CppTools::Tests::TestCase::waitUntilCppModelManagerIsAwareOf(project, timeOutInMs());
}
Command::Ptr OpenProjectCommand::parse(BatchFileLineTokenizer &arguments,
const CommandContext &context)
{
const QString projectFilePath = arguments.nextToken();
if (projectFilePath.isEmpty()) {
qWarning("%s:%d: error: No project file path given.",
qPrintable(context.filePath),
context.lineNumber);
return Command::Ptr();
}
const QString absoluteProjectFilePath = QFileInfo(projectFilePath).absoluteFilePath();
return Command::Ptr(new OpenProjectCommand(context, absoluteProjectFilePath));
}
class OpenDocumentCommand : public Command
{
public:
OpenDocumentCommand(const CommandContext &context,
const QString &documentFilePath);
bool run() override;
static Command::Ptr parse(BatchFileLineTokenizer &arguments, const CommandContext &context);
private:
QString m_documentFilePath;
};
OpenDocumentCommand::OpenDocumentCommand(const CommandContext &context,
const QString &documentFilePath)
: Command(context)
, m_documentFilePath(documentFilePath)
{
}
class WaitForUpdatedCodeWarnings : public QObject
{
Q_OBJECT
public:
WaitForUpdatedCodeWarnings(ClangEditorDocumentProcessor *processor);
bool wait(int timeOutInMs) const;
private:
void onCodeWarningsUpdated() { m_gotResults = true; }
private:
bool m_gotResults = false;
};
WaitForUpdatedCodeWarnings::WaitForUpdatedCodeWarnings(ClangEditorDocumentProcessor *processor)
{
connect(processor,
&ClangEditorDocumentProcessor::codeWarningsUpdated,
this, &WaitForUpdatedCodeWarnings::onCodeWarningsUpdated);
}
bool WaitForUpdatedCodeWarnings::wait(int timeOutInMs) const
{
QTime time;
time.start();
forever {
if (time.elapsed() > timeOutInMs) {
qWarning("WaitForUpdatedCodeWarnings: timeout of %d ms reached.", timeOutInMs);
return false;
}
if (m_gotResults)
return true;
QCoreApplication::processEvents();
QThread::msleep(20);
}
}
bool OpenDocumentCommand::run()
{
qCDebug(debug) << "line" << context().lineNumber << "OpenDocumentCommand" << m_documentFilePath;
const bool openEditorSucceeded = Core::EditorManager::openEditor(m_documentFilePath);
QTC_ASSERT(openEditorSucceeded, return false);
auto *processor = ClangEditorDocumentProcessor::get(m_documentFilePath);
QTC_ASSERT(processor, return false);
WaitForUpdatedCodeWarnings waiter(processor);
return waiter.wait(timeOutInMs());
}
Command::Ptr OpenDocumentCommand::parse(BatchFileLineTokenizer &arguments,
const CommandContext &context)
{
const QString documentFilePath = arguments.nextToken();
if (documentFilePath.isEmpty()) {
qWarning("%s:%d: error: No document file path given.",
qPrintable(context.filePath),
context.lineNumber);
return Command::Ptr();
}
const QString absoluteDocumentFilePath = QFileInfo(documentFilePath).absoluteFilePath();
return Command::Ptr(new OpenDocumentCommand(context, absoluteDocumentFilePath));
}
class CloseAllDocuments : public Command
{
public:
CloseAllDocuments(const CommandContext &context);
bool run() override;
static Command::Ptr parse(BatchFileLineTokenizer &arguments, const CommandContext &context);
};
CloseAllDocuments::CloseAllDocuments(const CommandContext &context)
: Command(context)
{
}
bool CloseAllDocuments::run()
{
qCDebug(debug) << "line" << context().lineNumber << "CloseAllDocuments";
return Core::EditorManager::closeAllEditors(/*askAboutModifiedEditors=*/ false);
}
Command::Ptr CloseAllDocuments::parse(BatchFileLineTokenizer &arguments,
const CommandContext &context)
{
const QString argument = arguments.nextToken();
if (!argument.isEmpty()) {
qWarning("%s:%d: error: Unexpected argument.",
qPrintable(context.filePath),
context.lineNumber);
return Command::Ptr();
}
return Command::Ptr(new CloseAllDocuments(context));
}
class InsertTextCommand : public Command
{
public:
// line and column are 1-based
InsertTextCommand(const CommandContext &context, const QString &text);
bool run() override;
static Command::Ptr parse(BatchFileLineTokenizer &arguments,
const CommandContext &context);
private:
const QString m_textToInsert;
};
InsertTextCommand::InsertTextCommand(const CommandContext &context, const QString &text)
: Command(context)
, m_textToInsert(text)
{
}
TextEditor::BaseTextEditor *currentTextEditor()
{
return qobject_cast<TextEditor::BaseTextEditor*>(Core::EditorManager::currentEditor());
}
bool InsertTextCommand::run()
{
qCDebug(debug) << "line" << context().lineNumber << "InsertTextCommand" << m_textToInsert;
TextEditor::BaseTextEditor *editor = currentTextEditor();
QTC_ASSERT(editor, return false);
const QString documentFilePath = editor->document()->filePath().toString();
auto *processor = ClangEditorDocumentProcessor::get(documentFilePath);
QTC_ASSERT(processor, return false);
editor->insert(m_textToInsert);
WaitForUpdatedCodeWarnings waiter(processor);
return waiter.wait(timeOutInMs());
}
Command::Ptr InsertTextCommand::parse(BatchFileLineTokenizer &arguments,
const CommandContext &context)
{
const QString textToInsert = arguments.nextToken();
if (textToInsert.isEmpty()) {
qWarning("%s:%d: error: No text to insert given.",
qPrintable(context.filePath),