Commit 5efd8246 authored by Tobias Hunger's avatar Tobias Hunger

SynchronousProcess: Store raw bytes from stdout/stderr of the process

Only convert the raw output later in a stdOut() and stdErr() method of
the SynchronousProcessResponse.

This is necessary since we have processes that use different encodings
for different sections of the file (I am looking at you, git).

Also remove the signals for raw data on stdout/stderr, leaving only the
signals returning buffered QString lines. This should be safe, even
with UTF-16 output.

Change-Id: Ida613fa86d1468cbd33bc6b3a1506a849c2d1c0a
Reviewed-by: Tobias Hunger's avatarTobias Hunger <tobias.hunger@qt.io>
parent 86882018
......@@ -46,7 +46,7 @@ QString BuildableHelperLibrary::qtChooserToQmakePath(const QString &path)
SynchronousProcessResponse response = proc.runBlocking(path, QStringList(QLatin1String("-print-env")));
if (response.result != SynchronousProcessResponse::Finished)
return QString();
const QString output = response.stdOut;
const QString output = response.stdOut();
int pos = output.indexOf(toolDir);
if (pos == -1)
return QString();
......
......@@ -277,8 +277,8 @@ void ShellCommand::run(QFutureInterface<void> &future)
Utils::SynchronousProcessResponse resp
= runCommand(job.binary, job.arguments, job.timeoutS, job.workingDirectory,
job.exitCodeInterpreter);
stdOut += resp.stdOut;
stdErr += resp.stdErr;
stdOut += resp.stdOut();
stdErr += resp.stdErr();
d->m_lastExecExitCode = resp.exitCode;
d->m_lastExecSuccess = resp.result == Utils::SynchronousProcessResponse::Finished;
if (!d->m_lastExecSuccess)
......@@ -427,21 +427,20 @@ Utils::SynchronousProcessResponse ShellCommand::runSynchronous(const Utils::File
&stdOut, &stdErr, true);
if (!d->m_aborted) {
response.codec = d->m_codec ? d->m_codec : QTextCodec::codecForLocale();
if (!stdErr.isEmpty()) {
response.stdErr = Utils::SynchronousProcess::normalizeNewlines(
d->m_codec ? d->m_codec->toUnicode(stdErr) : QString::fromLocal8Bit(stdErr));
response.rawStdErr = stdErr;
if (!(d->m_flags & SuppressStdErr))
proxy->append(response.stdErr);
proxy->append(response.stdErr());
}
if (!stdOut.isEmpty()) {
response.stdOut = Utils::SynchronousProcess::normalizeNewlines(
d->m_codec ? d->m_codec->toUnicode(stdOut) : QString::fromLocal8Bit(stdOut));
response.rawStdOut = stdOut;
if (d->m_flags & ShowStdOut) {
if (d->m_flags & SilentOutput)
proxy->appendSilently(response.stdOut);
proxy->appendSilently(response.stdOut());
else
proxy->append(response.stdOut);
proxy->append(response.stdOut());
}
}
}
......
......@@ -37,6 +37,7 @@
#include <QApplication>
#include <limits.h>
#include <memory>
#ifdef Q_OS_UNIX
# include <unistd.h>
......@@ -115,8 +116,8 @@ void SynchronousProcessResponse::clear()
{
result = StartFailed;
exitCode = -1;
stdOut.clear();
stdErr.clear();
rawStdOut.clear();
rawStdErr.clear();
}
QString SynchronousProcessResponse::exitMessage(const QString &binary, int timeoutS) const
......@@ -137,22 +138,49 @@ QString SynchronousProcessResponse::exitMessage(const QString &binary, int timeo
return QString();
}
QString SynchronousProcessResponse::allOutput() const
QByteArray SynchronousProcessResponse::allRawOutput() const
{
if (!stdOut.isEmpty() && !stdErr.isEmpty()) {
if (stdOut.endsWith(QLatin1Char('\n')))
return stdOut + stdErr;
else
return stdOut + QLatin1Char('\n') + stdErr;
if (!rawStdOut.isEmpty() && !rawStdErr.isEmpty()) {
QByteArray result;
result.reserve(rawStdOut.size() + rawStdErr.size() + 1);
result = rawStdOut;
if (!result.endsWith('\n'))
result += '\n';
result += rawStdErr;
return result;
}
return !stdOut.isEmpty() ? stdOut : stdErr;
return !rawStdOut.isEmpty() ? rawStdOut : rawStdErr;
}
QString SynchronousProcessResponse::allOutput() const
{
const QString out = stdOut();
const QString err = stdErr();
QString result;
result.reserve(out.size() + err.size() + 1);
result = out;
if (!result.endsWith('\n'))
result += '\n';
result += err;
return result;
}
QString SynchronousProcessResponse::stdOut() const
{
return SynchronousProcess::normalizeNewlines(codec->toUnicode(rawStdOut));
}
QString SynchronousProcessResponse::stdErr() const
{
return SynchronousProcess::normalizeNewlines(codec->toUnicode(rawStdErr));
}
QTCREATOR_UTILS_EXPORT QDebug operator<<(QDebug str, const SynchronousProcessResponse& r)
{
QDebug nsp = str.nospace();
nsp << "SynchronousProcessResponse: result=" << r.result << " ex=" << r.exitCode << '\n'
<< r.stdOut.size() << " bytes stdout, stderr=" << r.stdErr << '\n';
<< r.rawStdOut.size() << " bytes stdout, stderr=" << r.rawStdErr << '\n';
return str;
}
......@@ -169,52 +197,62 @@ class ChannelBuffer : public QObject
public:
void clearForRun();
QString linesRead();
void append(const QString &text, bool emitSignals);
void append(const QByteArray &text, bool emitSignals);
QString data;
int bufferPos = 0;
bool firstData = true;
QByteArray rawData;
QString incompleteLineBuffer; // lines not yet signaled
QTextCodec *codec = nullptr; // Not owner
std::unique_ptr<QTextCodec::ConverterState> codecState;
int rawDataPos = 0;
bool bufferedSignalsEnabled = false;
bool firstBuffer = true;
signals:
void output(const QString &text, bool firstTime);
void outputBuffered(const QString &text, bool firstTime);
};
void ChannelBuffer::clearForRun()
{
firstData = true;
firstBuffer = true;
bufferPos = 0;
rawDataPos = 0;
rawData.clear();
codecState.reset(new QTextCodec::ConverterState);
incompleteLineBuffer.clear();
}
/* Check for complete lines read from the device and return them, moving the
* buffer position. */
QString ChannelBuffer::linesRead()
{
// Any new lines?
const int lastLineIndex = qMax(data.lastIndexOf(QLatin1Char('\n')),
data.lastIndexOf(QLatin1Char('\r')));
if (lastLineIndex == -1 || lastLineIndex <= bufferPos)
// Convert and append the new input to the buffer of incomplete lines
const char *start = rawData.constData() + rawDataPos;
const int len = rawData.size() - rawDataPos;
incompleteLineBuffer.append(codec->toUnicode(start, len, codecState.get()));
rawDataPos = rawData.size();
// Any completed lines in the incompleteLineBuffer?
const int lastLineIndex = qMax(incompleteLineBuffer.lastIndexOf('\n'),
incompleteLineBuffer.lastIndexOf('\r'));
if (lastLineIndex == -1)
return QString();
const int nextBufferPos = lastLineIndex + 1;
const QString lines = data.mid(bufferPos, nextBufferPos - bufferPos);
bufferPos = nextBufferPos;
// Get completed lines and remove them from the incompleteLinesBuffer:
const QString lines = SynchronousProcess::normalizeNewlines(incompleteLineBuffer.left(lastLineIndex));
incompleteLineBuffer = incompleteLineBuffer.mid(lastLineIndex + 1);
return lines;
}
void ChannelBuffer::append(const QString &text, bool emitSignals)
void ChannelBuffer::append(const QByteArray &text, bool emitSignals)
{
if (text.isEmpty())
return;
data += text;
rawData += text;
if (!emitSignals)
return;
// Emit binary signals
emit output(text, firstData);
firstData = false;
// Buffered. Emit complete lines?
if (bufferedSignalsEnabled) {
const QString lines = linesRead();
......@@ -231,8 +269,6 @@ struct SynchronousProcessPrivate {
void clearForRun();
QTextCodec *m_codec;
QTextCodec::ConverterState m_stdOutState;
QTextCodec::ConverterState m_stdErrState;
TerminalControllingProcess m_process;
QTimer m_timer;
QEventLoop m_eventLoop;
......@@ -258,8 +294,11 @@ void SynchronousProcessPrivate::clearForRun()
{
m_hangTimerCount = 0;
m_stdOut.clearForRun();
m_stdOut.codec = m_codec;
m_stdErr.clearForRun();
m_stdErr.codec = m_codec;
m_result.clear();
m_result.codec = m_codec;
m_startFailure = false;
m_binary.clear();
}
......@@ -276,12 +315,16 @@ SynchronousProcess::SynchronousProcess() :
connect(&d->m_process, static_cast<void (QProcess::*)(QProcess::ProcessError)>(&QProcess::error),
this, &SynchronousProcess::error);
connect(&d->m_process, &QProcess::readyReadStandardOutput,
this, &SynchronousProcess::stdOutReady);
this, [this]() {
d->m_hangTimerCount = 0;
processStdOut(true);
});
connect(&d->m_process, &QProcess::readyReadStandardError,
this, &SynchronousProcess::stdErrReady);
connect(&d->m_stdOut, &ChannelBuffer::output, this, &SynchronousProcess::stdOut);
this, [this]() {
d->m_hangTimerCount = 0;
processStdErr(true);
});
connect(&d->m_stdOut, &ChannelBuffer::outputBuffered, this, &SynchronousProcess::stdOutBuffered);
connect(&d->m_stdErr, &ChannelBuffer::output, this, &SynchronousProcess::stdErr);
connect(&d->m_stdErr, &ChannelBuffer::outputBuffered, this, &SynchronousProcess::stdErrBuffered);
}
......@@ -436,8 +479,8 @@ SynchronousProcessResponse SynchronousProcess::run(const QString &binary,
processStdErr(false);
}
d->m_result.stdOut = d->m_stdOut.data;
d->m_result.stdErr = d->m_stdErr.data;
d->m_result.rawStdOut = d->m_stdOut.rawData;
d->m_result.rawStdErr = d->m_stdErr.rawData;
d->m_timer.stop();
if (isGuiThread())
......@@ -485,8 +528,8 @@ SynchronousProcessResponse SynchronousProcess::runBlocking(const QString &binary
processStdOut(false);
processStdErr(false);
d->m_result.stdOut = d->m_stdOut.data;
d->m_result.stdErr = d->m_stdErr.data;
d->m_result.rawStdOut = d->m_stdOut.rawData;
d->m_result.rawStdErr = d->m_stdErr.rawData;
return d->m_result;
}
......@@ -569,42 +612,16 @@ void SynchronousProcess::error(QProcess::ProcessError e)
d->m_eventLoop.quit();
}
void SynchronousProcess::stdOutReady()
{
d->m_hangTimerCount = 0;
processStdOut(true);
}
void SynchronousProcess::stdErrReady()
{
d->m_hangTimerCount = 0;
processStdErr(true);
}
QString SynchronousProcess::convertOutput(const QByteArray &ba,
QTextCodec::ConverterState *state) const
{
return normalizeNewlines(d->m_codec->toUnicode(ba, ba.size(), state));
}
void SynchronousProcess::processStdOut(bool emitSignals)
{
// Handle binary data
const QString stdOutput = convertOutput(d->m_process.readAllStandardOutput(),
&d->m_stdOutState);
if (debug > 1)
qDebug() << Q_FUNC_INFO << emitSignals << stdOutput;
d->m_stdOut.append(stdOutput, emitSignals);
d->m_stdOut.append(d->m_process.readAllStandardOutput(), emitSignals);
}
void SynchronousProcess::processStdErr(bool emitSignals)
{
// Handle binary data
const QString stdError = convertOutput(d->m_process.readAllStandardError(),
&d->m_stdErrState);
if (debug > 1)
qDebug() << Q_FUNC_INFO << emitSignals << stdError;
d->m_stdErr.append(stdError, emitSignals);
d->m_stdErr.append(d->m_process.readAllStandardError(), emitSignals);
}
QSharedPointer<QProcess> SynchronousProcess::createProcess(unsigned flags)
......
......@@ -59,12 +59,19 @@ struct QTCREATOR_UTILS_EXPORT SynchronousProcessResponse
// Helper to format an exit message.
QString exitMessage(const QString &binary, int timeoutS) const;
QByteArray allRawOutput() const;
QString allOutput() const;
QString stdOut() const;
QString stdErr() const;
Result result;
int exitCode;
QString stdOut;
QString stdErr;
QTextCodec *codec = nullptr;
QByteArray rawStdOut;
QByteArray rawStdErr;
};
QTCREATOR_UTILS_EXPORT QDebug operator<<(QDebug str, const SynchronousProcessResponse &);
......@@ -132,7 +139,7 @@ public:
// occurs on stderr/stdout as opposed to waitForFinished()). Returns false if a timeout
// occurs. Checking of the process' exit state/code still has to be done.
static bool readDataFromProcess(QProcess &p, int timeoutS,
QByteArray *stdOut = 0, QByteArray *stdErr = 0,
QByteArray *rawStdOut = 0, QByteArray *rawStdErr = 0,
bool timeOutMessageBox = false);
// Stop a process by first calling terminate() (allowing for signal handling) and
// then kill().
......@@ -146,11 +153,8 @@ public:
static QString normalizeNewlines(const QString &text);
signals:
void stdOut(const QString &text, bool firstTime);
void stdErr(const QString &text, bool firstTime);
void stdOutBuffered(const QString &data, bool firstTime);
void stdErrBuffered(const QString &data, bool firstTime);
void stdOutBuffered(const QString &lines, bool firstTime);
void stdErrBuffered(const QString &lines, bool firstTime);
public slots:
bool terminate();
......@@ -159,11 +163,8 @@ private:
void slotTimeout();
void finished(int exitCode, QProcess::ExitStatus e);
void error(QProcess::ProcessError);
void stdOutReady();
void stdErrReady();
void processStdOut(bool emitSignals);
void processStdErr(bool emitSignals);
QString convertOutput(const QByteArray &, QTextCodec::ConverterState *state) const;
SynchronousProcessPrivate *d;
};
......
......@@ -305,7 +305,7 @@ QAbstractItemModel *AndroidBuildApkStep::keystoreCertificates()
QMessageBox::critical(0, tr("Error"), tr("Invalid password."));
m_keystorePasswd.clear();
}
rawCerts = response.stdOut;
rawCerts = response.stdOut();
}
return new CertificatesModel(rawCerts, this);
}
......
......@@ -85,10 +85,10 @@ static int updateVersionHelper(const QString &command)
return 0;
// Astyle prints the version on stdout or stderr, depending on platform
const int version = parseVersion(response.stdOut.trimmed());
const int version = parseVersion(response.stdOut().trimmed());
if (version != 0)
return version;
return parseVersion(response.stdErr.trimmed());
return parseVersion(response.stdErr().trimmed());
}
void ArtisticStyleSettings::updateVersion()
......@@ -176,7 +176,7 @@ void ArtisticStyleSettings::createDocumentationFile() const
stream.writeStartElement(Constants::DOCUMENTATION_XMLROOT);
// astyle writes its output to 'error'...
const QStringList lines = response.stdErr.split(QLatin1Char('\n'));
const QStringList lines = response.stdErr().split(QLatin1Char('\n'));
QStringList keys;
QStringList docu;
for (QString line : lines) {
......
......@@ -104,7 +104,7 @@ FormatTask format(FormatTask task)
task.error = BeautifierPlugin::tr("Failed to format: %1.").arg(response.exitMessage(executable, 5));
return task;
}
const QString output = response.stdErr;
const QString output = response.stdErr();
if (!output.isEmpty())
task.error = executable + QLatin1String(": ") + output;
......
......@@ -1480,8 +1480,8 @@ ClearCasePlugin::runCleartool(const QString &workingDir,
response.error = sp_resp.result != SynchronousProcessResponse::Finished;
if (response.error)
response.message = sp_resp.exitMessage(executable, timeOutS);
response.stdErr = sp_resp.stdErr;
response.stdOut = sp_resp.stdOut;
response.stdErr = sp_resp.stdErr();
response.stdOut = sp_resp.stdOut();
return response;
}
......
......@@ -153,7 +153,7 @@ QStringList CMakeTool::supportedGenerators() const
Utils::SynchronousProcessResponse response = run(QLatin1String("--help"));
if (response.result == Utils::SynchronousProcessResponse::Finished) {
bool inGeneratorSection = false;
const QStringList lines = response.stdOut.split(QLatin1Char('\n'));
const QStringList lines = response.stdOut().split(QLatin1Char('\n'));
foreach (const QString &line, lines) {
if (line.isEmpty())
continue;
......@@ -188,19 +188,19 @@ TextEditor::Keywords CMakeTool::keywords()
Utils::SynchronousProcessResponse response;
response = run(QLatin1String("--help-command-list"));
if (response.result == Utils::SynchronousProcessResponse::Finished)
m_functions = response.stdOut.split(QLatin1Char('\n'));
m_functions = response.stdOut().split(QLatin1Char('\n'));
response = run(QLatin1String("--help-commands"));
if (response.result == Utils::SynchronousProcessResponse::Finished)
parseFunctionDetailsOutput(response.stdOut);
parseFunctionDetailsOutput(response.stdOut());
response = run(QLatin1String("--help-property-list"));
if (response.result == Utils::SynchronousProcessResponse::Finished)
m_variables = parseVariableOutput(response.stdOut);
m_variables = parseVariableOutput(response.stdOut());
response = run(QLatin1String("--help-variable-list"));
if (response.result == Utils::SynchronousProcessResponse::Finished) {
m_variables.append(parseVariableOutput(response.stdOut));
m_variables.append(parseVariableOutput(response.stdOut()));
m_variables = Utils::filteredUnique(m_variables);
Utils::sort(m_variables);
}
......
......@@ -1140,8 +1140,8 @@ CvsResponse CvsPlugin::runCvs(const QString &workingDirectory,
timeOutS, flags, outputCodec);
response.result = CvsResponse::OtherError;
response.stdErr = sp_resp.stdErr;
response.stdOut = sp_resp.stdOut;
response.stdErr = sp_resp.stdErr();
response.stdOut = sp_resp.stdOut();
switch (sp_resp.result) {
case SynchronousProcessResponse::Finished:
response.result = CvsResponse::Ok;
......
......@@ -142,7 +142,7 @@ GdbCoreEngine::readExecutableNameFromCore(const QString &debuggerCommand, const
SynchronousProcessResponse response = proc.runBlocking(debuggerCommand, args);
if (response.result == SynchronousProcessResponse::Finished) {
QString output = response.stdOut;
QString output = response.stdOut();
// Core was generated by `/data/dev/creator-2.6/bin/qtcreator'.
// Program terminated with signal 11, Segmentation fault.
int pos1 = output.indexOf("Core was generated by");
......
......@@ -473,8 +473,8 @@ public:
// No conflicts => do nothing
if (response.result == SynchronousProcessResponse::Finished)
return;
handler.readStdOut(response.stdOut);
handler.readStdErr(response.stdErr);
handler.readStdOut(response.stdOut());
handler.readStdErr(response.stdErr());
}
private:
......@@ -1543,7 +1543,7 @@ bool GitClient::executeSynchronousStash(const QString &workingDirectory,
const SynchronousProcessResponse response = vcsSynchronousExec(workingDirectory, arguments, flags);
const bool rc = response.result == SynchronousProcessResponse::Finished;
if (!rc)
msgCannotRun(arguments, workingDirectory, response.stdErr.toLocal8Bit(), errorMessage);
msgCannotRun(arguments, workingDirectory, response.rawStdErr, errorMessage);
return rc;
}
......@@ -2119,7 +2119,7 @@ QStringList GitClient::synchronousRepositoryBranches(const QString &repositoryUR
QString headSha;
// split "82bfad2f51d34e98b18982211c82220b8db049b<tab>refs/heads/master"
bool headFound = false;
foreach (const QString &line, resp.stdOut.split('\n')) {
foreach (const QString &line, resp.stdOut().split('\n')) {
if (line.endsWith("\tHEAD")) {
QTC_CHECK(headSha.isNull());
headSha = line.left(line.indexOf('\t'));
......
......@@ -104,7 +104,7 @@ void IosProbe::detectDeveloperPaths()
if (response.result != Utils::SynchronousProcessResponse::Finished) {
qCWarning(probeLog) << QString::fromLatin1("Could not detect selected xcode with /usr/bin/xcode-select");
} else {
QString path = response.stdOut;
QString path = response.stdOut();
path.chop(1);
addDeveloperPath(path);
}
......
......@@ -157,7 +157,7 @@ bool MercurialClient::synchronousPull(const QString &workingDir, const QString &
workingDir, vcsBinary(), args, vcsTimeoutS(), flags, 0, env);
const bool ok = resp.result == SynchronousProcessResponse::Finished;
parsePullOutput(resp.stdOut.trimmed());
parsePullOutput(resp.stdOut().trimmed());
return ok;
}
......
......@@ -1016,8 +1016,8 @@ PerforceResponse PerforcePlugin::synchronousProcess(const QString &workingDir,
PerforceResponse response;
response.error = true;
response.exitCode = sp_resp.exitCode;
response.stdErr = sp_resp.stdErr;
response.stdOut = sp_resp.stdOut;
response.stdErr = sp_resp.stdErr();
response.stdOut = sp_resp.stdOut();
switch (sp_resp.result) {
case SynchronousProcessResponse::Finished:
response.error = false;
......
......@@ -286,7 +286,7 @@ bool AbstractMsvcToolChain::generateEnvironmentSettings(Utils::Environment &env,
}
// The SDK/MSVC scripts do not return exit codes != 0. Check on stdout.
const QString stdOut = response.stdOut;
const QString stdOut = response.stdOut();
//
// Now parse the file to get the environment settings
......
......@@ -116,15 +116,15 @@ static bool
if (response.result != Utils::SynchronousProcessResponse::Finished) {
*errorMessage = QString::fromLatin1("Generator script failed: %1")
.arg(response.exitMessage(binary, 30));
if (!response.stdErr.isEmpty()) {
const QString stdErr = response.stdErr();
if (!stdErr.isEmpty()) {
errorMessage->append(QLatin1Char('\n'));
errorMessage->append(response.stdErr);
errorMessage->append(stdErr);
}
return false;
}
if (stdOut) {
*stdOut = response.stdOut;
stdOut->remove(QLatin1Char('\r'));
*stdOut = response.stdOut();
if (CustomWizard::verbose())
qDebug("Output: '%s'\n", qPrintable(*stdOut));
}
......
......@@ -253,18 +253,17 @@ QByteArray MsvcToolChain::msvcPredefinedMacros(const QStringList cxxflags,
response.exitCode != 0)
return predefinedMacros;
const QStringList output = response.stdOut.split('\n');
const QStringList output = Utils::filtered(response.stdOut().split('\n'),
[](const QString &s) { return s.startsWith('V'); });
foreach (const QString& line, output) {
if (line.startsWith('V')) {
QStringList split = line.split('=');
const QString key = split.at(0).mid(1);
QString value = split.at(1);
predefinedMacros += "#define ";
predefinedMacros += key.toUtf8();
predefinedMacros += ' ';
predefinedMacros += value.toUtf8();
predefinedMacros += '\n';
}
QStringList split = line.split('=');
const QString key = split.at(0).mid(1);
QString value = split.at(1);
predefinedMacros += "#define ";
predefinedMacros += key.toUtf8();
predefinedMacros += ' ';
predefinedMacros += value.toUtf8();
predefinedMacros += '\n';
}
if (debug)
qDebug() << "msvcPredefinedMacros" << predefinedMacros;
......
......@@ -892,8 +892,8 @@ SubversionResponse SubversionPlugin::runSvn(const QString &workingDir,
response.error = sp_resp.result != SynchronousProcessResponse::Finished;
if (response.error)
response.message = sp_resp.exitMessage(executable.toString(), timeOutS);
response.stdErr = sp_resp.stdErr;
response.stdOut = sp_resp.stdOut;
response.stdErr = sp_resp.stdErr();
response.stdOut = sp_resp.stdOut();
return response;
}
......
......@@ -74,7 +74,7 @@ QString findFallbackDefinitionsLocation()