Commit f7c68f6b authored by Erik Verbruggen's avatar Erik Verbruggen

C++: change working-copy to work on UTF-8 encoded QByteArrays.

These not only take less space than UTF-16 encoded QStrings, but due to
the caching in the CppEditorSupport also take less time to build.

This patch also fixes a number of possible encoding issues, where files
and constant strings were (falsely) assumed to be UTF-8.

Change-Id: Ib6f91c9a94ebed5b5dfbd4eb2998825c62c72784
Reviewed-by: default avatarNikolai Kosjar <nikolai.kosjar@digia.com>
Reviewed-by: default avatarhjk <hjk121@nokiamail.com>
parent eebb1dfc
......@@ -736,7 +736,8 @@ void Snapshot::insert(Document::Ptr doc)
_documents.insert(doc->fileName(), doc);
}
Document::Ptr Snapshot::preprocessedDocument(const QString &source, const QString &fileName) const
Document::Ptr Snapshot::preprocessedDocument(const QByteArray &source,
const QString &fileName) const
{
Document::Ptr newDoc = Document::create(fileName);
if (Document::Ptr thisDocument = document(fileName)) {
......
......@@ -398,7 +398,7 @@ public:
Snapshot simplified(Document::Ptr doc) const;
Document::Ptr preprocessedDocument(const QString &source,
Document::Ptr preprocessedDocument(const QByteArray &source,
const QString &fileName) const;
Document::Ptr documentFromSource(const QByteArray &preprocessedDocument,
......
......@@ -41,7 +41,7 @@ FastPreprocessor::FastPreprocessor(const Snapshot &snapshot)
, _preproc(this, &_env)
{ }
QByteArray FastPreprocessor::run(Document::Ptr newDoc, const QString &source)
QByteArray FastPreprocessor::run(Document::Ptr newDoc, const QByteArray &source)
{
std::swap(newDoc, _currentDoc);
const QString fileName = _currentDoc->fileName();
......
......@@ -54,7 +54,7 @@ class CPLUSPLUS_EXPORT FastPreprocessor: public Client
public:
FastPreprocessor(const Snapshot &snapshot);
QByteArray run(Document::Ptr newDoc, const QString &source);
QByteArray run(Document::Ptr newDoc, const QByteArray &source);
// CPlusPlus::Client
virtual void sourceNeeded(unsigned line, const QString &fileName, IncludeType mode);
......
......@@ -271,6 +271,39 @@ TextFileFormat::ReadResult
return result;
}
TextFileFormat::ReadResult TextFileFormat::readFileUTF8(const QString &fileName,
QByteArray *plainText, QString *errorString)
{
QByteArray data;
try {
Utils::FileReader reader;
if (!reader.fetch(fileName, errorString))
return TextFileFormat::ReadIOError;
data = reader.data();
} catch (const std::bad_alloc &) {
*errorString = QCoreApplication::translate("Utils::TextFileFormat", "Out of memory.");
return TextFileFormat::ReadMemoryAllocationError;
}
TextFileFormat format = TextFileFormat::detect(data);
if (!format.codec)
format.codec = QTextCodec::codecForLocale();
if (format.codec->name() == "UTF-8") {
if (format.hasUtf8Bom)
data.remove(0, 3);
*plainText = data;
return TextFileFormat::ReadSuccess;
}
QString target;
if (!format.decode(data, &target)) {
*errorString = QCoreApplication::translate("Utils::TextFileFormat", "An encoding error was encountered.");
return TextFileFormat::ReadEncodingError;
}
*plainText = target.toUtf8();
return TextFileFormat::ReadSuccess;
}
/*!
\brief Write out a text file.
*/
......
......@@ -76,6 +76,8 @@ public:
static ReadResult readFile(const QString &fileName, const QTextCodec *defaultCodec,
QString *plainText, TextFileFormat *format, QString *errorString,
QByteArray *decodingErrorSample = 0);
static ReadResult readFileUTF8(const QString &fileName, QByteArray *plainText,
QString *errorString);
bool writeFile(const QString &fileName, QString plainText, QString *errorString) const;
......
......@@ -196,7 +196,7 @@ void TestActionsTestCase::run(const Actions &tokenActions, const Actions &fileAc
Snapshot snapshot = mm->snapshot();
Document::Ptr document = snapshot.preprocessedDocument(
editorWidget->document()->toPlainText(), filePath);
editorWidget->document()->toPlainText().toUtf8(), filePath);
QVERIFY(document);
document->parse();
TranslationUnit *translationUnit = document->translationUnit();
......
......@@ -45,6 +45,7 @@ public:
explicit AbstractEditorSupport(CppModelManagerInterface *modelmanager);
virtual ~AbstractEditorSupport();
/// \returns the contents, encoded as UTF-8
virtual QByteArray contents() const = 0;
virtual QString fileName() const = 0;
......
......@@ -53,8 +53,8 @@ using namespace CppTools::Internal;
using namespace CppTools;
using namespace CPlusPlus;
static QString getSource(const QString &fileName,
const CppModelManagerInterface::WorkingCopy &workingCopy)
static QByteArray getSource(const QString &fileName,
const CppModelManagerInterface::WorkingCopy &workingCopy)
{
if (workingCopy.contains(fileName)) {
return workingCopy.source(fileName);
......@@ -68,7 +68,7 @@ static QString getSource(const QString &fileName,
if (result != Utils::TextFileFormat::ReadSuccess)
qWarning() << "Could not read " << fileName << ". Error: " << error;
return fileContents;
return fileContents.toUtf8();
}
}
......@@ -110,7 +110,7 @@ public:
return usages; // skip this document, it's not using symbolId.
}
Document::Ptr doc;
const QString unpreprocessedSource = getSource(fileName, workingCopy);
const QByteArray unpreprocessedSource = getSource(fileName, workingCopy);
if (symbolDocument && fileName == symbolDocument->fileName()) {
doc = symbolDocument;
......@@ -124,7 +124,7 @@ public:
if (doc != symbolDocument)
doc->check();
FindUsages process(unpreprocessedSource.toUtf8(), doc, snapshot);
FindUsages process(unpreprocessedSource, doc, snapshot);
process(symbol);
usages = process.usages();
......@@ -439,7 +439,7 @@ bool CppFindReferences::findSymbol(CppFindReferencesParameters *parameters,
Document::Ptr newSymbolDocument = snapshot.document(symbolFile);
// document is not parsed and has no bindings yet, do it
QString source = getSource(newSymbolDocument->fileName(), _modelManager->workingCopy());
QByteArray source = getSource(newSymbolDocument->fileName(), _modelManager->workingCopy());
Document::Ptr doc =
snapshot.preprocessedDocument(source, newSymbolDocument->fileName());
doc->check();
......@@ -541,7 +541,7 @@ public:
{
QList<Usage> usages;
Document::Ptr doc = snapshot.document(fileName);
QString source;
QByteArray source;
restart_search:
if (future->isPaused())
......@@ -566,8 +566,8 @@ restart_search:
if (macro.name() == useMacro.name()) {
unsigned lineStart;
const QString &lineSource = matchingLine(use.begin(), source, &lineStart);
usages.append(Usage(fileName, lineSource, use.beginLine(),
const QByteArray &lineSource = matchingLine(use.begin(), source, &lineStart);
usages.append(Usage(fileName, QString::fromUtf8(lineSource), use.beginLine(),
use.begin() - lineStart, useMacro.name().length()));
}
}
......@@ -578,18 +578,18 @@ restart_search:
return usages;
}
static QString matchingLine(unsigned position, const QString &source,
unsigned *lineStart = 0)
static QByteArray matchingLine(unsigned position, const QByteArray &source,
unsigned *lineStart = 0)
{
int lineBegin = source.lastIndexOf(QLatin1Char('\n'), position) + 1;
int lineEnd = source.indexOf(QLatin1Char('\n'), position);
int lineBegin = source.lastIndexOf('\n', position) + 1;
int lineEnd = source.indexOf('\n', position);
if (lineEnd == -1)
lineEnd = source.length();
if (lineStart)
*lineStart = lineBegin;
const QString matchingLine = source.mid(lineBegin, lineEnd - lineBegin);
const QByteArray matchingLine = source.mid(lineBegin, lineEnd - lineBegin);
return matchingLine;
}
};
......@@ -652,10 +652,11 @@ void CppFindReferences::findMacroUses(const Macro &macro, const QString &replace
// add the macro definition itself
{
const QString &source = getSource(macro.fileName(), workingCopy);
const QByteArray &source = getSource(macro.fileName(), workingCopy);
unsigned lineStart;
const QString line = FindMacroUsesInFile::matchingLine(macro.offset(), source, &lineStart);
search->addResult(macro.fileName(), macro.line(), line,
const QByteArray line = FindMacroUsesInFile::matchingLine(macro.offset(), source,
&lineStart);
search->addResult(macro.fileName(), macro.line(), QString::fromUtf8(line),
macro.offset() - lineStart, macro.name().length());
}
......
......@@ -541,13 +541,13 @@ CppModelManager::WorkingCopy CppModelManager::buildWorkingCopyList()
QSetIterator<AbstractEditorSupport *> it(m_extraEditorSupports);
while (it.hasNext()) {
AbstractEditorSupport *es = it.next();
workingCopy.insert(es->fileName(), QString::fromUtf8(es->contents()));
workingCopy.insert(es->fileName(), es->contents());
}
// Add the project configuration file
QByteArray conf(pp_configuration);
QByteArray conf = QByteArray::fromRawData(pp_configuration, qstrlen(pp_configuration));
conf += definedMacros();
workingCopy.insert(configurationFileName(), QString::fromLocal8Bit(conf));
workingCopy.insert(configurationFileName(), conf);
return workingCopy;
}
......
......@@ -673,7 +673,7 @@ void CppToolsPlugin::test_modelmanager_extraeditorsupport_uiFiles()
QCOMPARE(workingCopy.size(), 2); // mm->configurationFileName() and "ui_*.h"
QStringList fileNamesInWorkinCopy;
QHashIterator<QString, QPair<QString, unsigned> > it = workingCopy.iterator();
QHashIterator<QString, QPair<QByteArray, unsigned> > it = workingCopy.iterator();
while (it.hasNext()) {
it.next();
fileNamesInWorkinCopy << QFileInfo(it.key()).fileName();
......
......@@ -176,29 +176,38 @@ public:
QByteArray m_defines;
};
/// The working-copy stores all files that are stored on disk in their current state.
///
/// So, currently the working copy holds:
/// - unsaved content of editors
/// - uic-ed UI files (through \c AbstractEditorSupport)
/// - the preprocessor configuration
///
/// Contents are keyed on filename, and hold the revision in the editor and the editor's
/// contents encoded as UTF-8.
class CPPTOOLS_EXPORT WorkingCopy
{
public:
void insert(const QString &fileName, const QString &source, unsigned revision = 0)
void insert(const QString &fileName, const QByteArray &source, unsigned revision = 0)
{ _elements.insert(fileName, qMakePair(source, revision)); }
bool contains(const QString &fileName) const
{ return _elements.contains(fileName); }
QString source(const QString &fileName) const
QByteArray source(const QString &fileName) const
{ return _elements.value(fileName).first; }
QPair<QString, unsigned> get(const QString &fileName) const
QPair<QByteArray, unsigned> get(const QString &fileName) const
{ return _elements.value(fileName); }
QHashIterator<QString, QPair<QString, unsigned> > iterator() const
{ return QHashIterator<QString, QPair<QString, unsigned> >(_elements); }
QHashIterator<QString, QPair<QByteArray, unsigned> > iterator() const
{ return QHashIterator<QString, QPair<QByteArray, unsigned> >(_elements); }
int size() const
{ return _elements.size(); }
private:
typedef QHash<QString, QPair<QString, unsigned> > Table;
typedef QHash<QString, QPair<QByteArray, unsigned> > Table;
Table _elements;
};
......
......@@ -3,6 +3,7 @@
#include "cppmodelmanager.h"
#include <utils/hostosinfo.h>
#include <utils/textfileformat.h>
#include <QCoreApplication>
......@@ -154,14 +155,14 @@ void CppPreprocessor::resetEnvironment()
}
void CppPreprocessor::getFileContents(const QString &absoluteFilePath,
QString *contents,
QByteArray *contents,
unsigned *revision) const
{
if (absoluteFilePath.isEmpty())
return;
if (m_workingCopy.contains(absoluteFilePath)) {
const QPair<QString, unsigned> entry = m_workingCopy.get(absoluteFilePath);
const QPair<QByteArray, unsigned> entry = m_workingCopy.get(absoluteFilePath);
if (contents)
*contents = entry.first;
if (revision)
......@@ -169,16 +170,11 @@ void CppPreprocessor::getFileContents(const QString &absoluteFilePath,
return;
}
QFile file(absoluteFilePath);
if (file.open(QFile::ReadOnly | QFile::Text)) {
QTextStream stream(&file);
stream.setCodec(Core::EditorManager::defaultTextCodec());
if (contents)
*contents = stream.readAll();
if (revision)
*revision = 0;
file.close();
}
QString errStr;
if (contents)
Utils::TextFileFormat::readFileUTF8(absoluteFilePath, contents, &errStr);
if (revision)
*revision = 0;
}
bool CppPreprocessor::checkFile(const QString &absoluteFilePath) const
......@@ -370,7 +366,7 @@ void CppPreprocessor::sourceNeeded(unsigned line, const QString &fileName, Inclu
m_included.insert(absoluteFileName);
unsigned editorRevision = 0;
QString contents;
QByteArray contents;
getFileContents(absoluteFileName, &contents, &editorRevision);
if (m_currentDoc) {
if (contents.isEmpty() && !QFileInfo(absoluteFileName).isAbsolute()) {
......
......@@ -44,7 +44,7 @@ public:
protected:
CPlusPlus::Document::Ptr switchDocument(CPlusPlus::Document::Ptr doc);
void getFileContents(const QString &absoluteFilePath, QString *contents, unsigned *revision) const;
void getFileContents(const QString &absoluteFilePath, QByteArray *contents, unsigned *revision) const;
bool checkFile(const QString &absoluteFilePath) const;
QString resolveFile(const QString &fileName, IncludeType type);
QString resolveFile_helper(const QString &fileName, IncludeType type);
......
......@@ -109,7 +109,7 @@ CppRefactoringFileConstPtr CppRefactoringChanges::fileNoEditor(const QString &fi
{
QTextDocument *document = 0;
if (data()->m_workingCopy.contains(fileName))
document = new QTextDocument(data()->m_workingCopy.source(fileName));
document = new QTextDocument(QString::fromUtf8(data()->m_workingCopy.source(fileName)));
CppRefactoringFilePtr result(new CppRefactoringFile(document, fileName));
result->m_data = m_data;
......@@ -144,7 +144,7 @@ Document::Ptr CppRefactoringFile::cppDocument() const
{
if (!m_cppDocument || !m_cppDocument->translationUnit() ||
!m_cppDocument->translationUnit()->ast()) {
const QString source = document()->toPlainText();
const QByteArray source = document()->toPlainText().toUtf8();
const QString name = fileName();
const Snapshot &snapshot = data()->m_snapshot;
......
......@@ -47,7 +47,7 @@ public:
{
const CPlusPlus::Snapshot snapshot;
const QString fileName;
const QString code;
const QByteArray code;
const int line;
const int column;
const unsigned revision;
......@@ -59,7 +59,7 @@ public:
Source(const CPlusPlus::Snapshot &snapshot,
const QString &fileName,
const QString &code,
const QByteArray &code,
int line, int column,
unsigned revision,
bool force)
......
......@@ -166,12 +166,12 @@ QString CppEditorSupport::fileName() const
return m_textEditor->document()->filePath();
}
QString CppEditorSupport::contents() const
QByteArray CppEditorSupport::contents() const
{
const int editorRev = editorRevision();
if (m_cachedContentsEditorRevision != editorRev && !m_fileIsBeingReloaded) {
m_cachedContentsEditorRevision = editorRev;
m_cachedContents = m_textEditor->textDocument()->contents();
m_cachedContents = m_textEditor->textDocument()->contents().toUtf8();
}
return m_cachedContents;
......@@ -430,7 +430,7 @@ SemanticInfo::Source CppEditorSupport::currentSource(bool force)
const Snapshot snapshot = m_modelManager->snapshot();
QString code;
QByteArray code;
if (force || m_lastSemanticInfo.revision != editorRevision())
code = contents(); // get the source code only when needed.
......
......@@ -96,7 +96,7 @@ public:
QString fileName() const;
QString contents() const;
QByteArray contents() const;
unsigned editorRevision() const;
void setExtraDiagnostics(const QString &key,
......@@ -173,7 +173,7 @@ private:
QFuture<void> m_documentParser;
// content caching
mutable QString m_cachedContents;
mutable QByteArray m_cachedContents;
mutable int m_cachedContentsEditorRevision;
bool m_fileIsBeingReloaded;
......
......@@ -2702,13 +2702,13 @@ static CPlusPlus::Document::Ptr getParsedDocument(const QString &fileName,
const CppTools::CppModelManagerInterface::WorkingCopy &workingCopy,
const CPlusPlus::Snapshot &snapshot)
{
QString src;
QByteArray src;
if (workingCopy.contains(fileName)) {
src = workingCopy.source(fileName);
} else {
FileReader reader;
if (reader.fetch(fileName)) // ### FIXME error reporting
src = QString::fromLocal8Bit(reader.data()); // ### FIXME encoding
src = QString::fromLocal8Bit(reader.data()).toUtf8();
}
CPlusPlus::Document::Ptr doc = snapshot.preprocessedDocument(src, fileName);
......
......@@ -493,13 +493,13 @@ static Document::Ptr getParsedDocument(const QString &fileName,
CppTools::CppModelManagerInterface::WorkingCopy &workingCopy,
Snapshot &snapshot)
{
QString src;
QByteArray src;
if (workingCopy.contains(fileName)) {
src = workingCopy.source(fileName);
} else {
Utils::FileReader reader;
if (reader.fetch(fileName)) // ### FIXME error reporting
src = QString::fromLocal8Bit(reader.data()); // ### FIXME encoding
src = QString::fromLocal8Bit(reader.data()).toUtf8();
}
Document::Ptr doc = snapshot.preprocessedDocument(src, fileName);
......@@ -547,7 +547,7 @@ bool QtCreatorIntegration::navigateToSlot(const QString &objectName,
} else {
const CppTools::CppModelManagerInterface::WorkingCopy workingCopy =
CppTools::CppModelManagerInterface::instance()->workingCopy();
QHashIterator<QString, QPair<QString, unsigned> > it = workingCopy.iterator();
QHashIterator<QString, QPair<QByteArray, unsigned> > it = workingCopy.iterator();
while (it.hasNext()) {
it.next();
const QString fileName = it.key();
......
......@@ -221,8 +221,11 @@ void GenericProject::parseProject(RefreshOptions options)
m_defines.clear();
QFile configFile(configFileName());
if (configFile.open(QFile::ReadOnly))
m_defines = configFile.readAll();
if (configFile.open(QFile::ReadOnly)) {
// convert from local/file encoding to UTF-8
QTextStream configStream(&configFile);
m_defines = configStream.readAll().toUtf8();
}
}
if (options & Files)
......
......@@ -223,14 +223,15 @@ QByteArray AbstractMsvcToolChain::msvcPredefinedMacros(const QStringList cxxflag
Q_UNUSED(cxxflags);
Q_UNUSED(env);
QByteArray predefinedMacros = "#define __MSVCRT__\n"
static const QByteArray predefinedMacros(
"#define __MSVCRT__\n"
"#define __w64\n"
"#define __int64 long long\n"
"#define __int32 long\n"
"#define __int16 short\n"
"#define __int8 char\n"
"#define __ptr32\n"
"#define __ptr64\n";
"#define __ptr64\n");
return predefinedMacros;
}
......
......@@ -123,14 +123,14 @@ void UiCodeModelSupport::init() const
if (debug)
qDebug()<<"uic run wasn't succesfull";
m_cacheTime = QDateTime ();
m_contents = QByteArray();
m_contents.clear();
m_state = FINISHED;
return;
}
} else {
if (debug)
qDebug()<<"Could open "<<m_uiFileName<<"needed for the cpp model";
m_contents = QByteArray();
m_contents.clear();
m_state = FINISHED;
}
}
......@@ -296,7 +296,10 @@ bool UiCodeModelSupport::finishProcess() const
if (m_state != RUNNING) // waitForFinished can recurse into finishProcess
return true;
m_contents = m_process.readAllStandardOutput();
// As far as I can discover in the UIC sources, it writes out local 8-bit encoding. The
// conversion below is to normalize both the encoding, and the line terminators.
QString normalized = QString::fromLocal8Bit(m_process.readAllStandardOutput());
m_contents = normalized.toUtf8();
m_cacheTime = QDateTime::currentDateTime();
if (debug)
qDebug() << "ok" << m_contents.size() << "bytes.";
......
......@@ -59,6 +59,7 @@ public:
~UiCodeModelSupport();
void setHeaderFileName(const QString &name);
/// \returns the contents encoded in UTF-8.
QByteArray contents() const;
QString uiFileName() const; // The .ui-file
QString fileName() const; // The header file
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment