cppsourceprocessor.cpp 17.7 KB
Newer Older
Nikolai Kosjar's avatar
Nikolai Kosjar committed
1 2
/****************************************************************************
**
3 4
** Copyright (C) 2016 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
Nikolai Kosjar's avatar
Nikolai Kosjar committed
5 6 7 8 9 10 11
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
12 13 14
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
Nikolai Kosjar's avatar
Nikolai Kosjar committed
15
**
16 17 18 19 20 21 22
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
Nikolai Kosjar's avatar
Nikolai Kosjar committed
23 24 25
**
****************************************************************************/

26
#include "cppsourceprocessor.h"
27

Nikolai Kosjar's avatar
Nikolai Kosjar committed
28
#include "cppmodelmanager.h"
29
#include "cpptoolsreuse.h"
Nikolai Kosjar's avatar
Nikolai Kosjar committed
30

31 32 33
#include <coreplugin/editormanager/editormanager.h>

#include <utils/fileutils.h>
34
#include <utils/hostosinfo.h>
35
#include <utils/qtcassert.h>
36
#include <utils/textfileformat.h>
37 38

#include <QCoreApplication>
39
#include <QCryptographicHash>
40
#include <QDir>
41
#include <QLoggingCategory>
42
#include <QTextCodec>
43

44
/*!
45 46
 * \class CppTools::Internal::CppSourceProcessor
 * \brief The CppSourceProcessor class updates set of indexed C++ files.
47
 *
48 49
 * Working copy ensures that documents with most recent copy placed in memory will be parsed
 * correctly.
50 51
 *
 * \sa CPlusPlus::Document
52
 * \sa CppTools::WorkingCopy
53 54
 */

55 56 57 58
using namespace CPlusPlus;
using namespace CppTools;
using namespace CppTools::Internal;

59 60
typedef Document::DiagnosticMessage Message;

61 62
static Q_LOGGING_CATEGORY(log, "qtc.cpptools.sourceprocessor")

63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
namespace {

inline QByteArray generateFingerPrint(const QList<Macro> &definedMacros, const QByteArray &code)
{
    QCryptographicHash hash(QCryptographicHash::Sha1);
    hash.addData(code);
    foreach (const Macro &macro, definedMacros) {
        if (macro.isHidden()) {
            static const QByteArray undef("#undef ");
            hash.addData(undef);
            hash.addData(macro.name());
        } else {
            static const QByteArray def("#define ");
            hash.addData(macro.name());
            hash.addData(" ", 1);
            hash.addData(def);
            hash.addData(macro.definitionText());
        }
        hash.addData("\n", 1);
    }
    return hash.result();
}

inline Message messageNoSuchFile(Document::Ptr &document, const QString &fileName, unsigned line)
{
    const QString text = QCoreApplication::translate(
        "CppSourceProcessor", "%1: No such file or directory").arg(fileName);
    return Message(Message::Warning, document->fileName(), line, /*column =*/ 0, text);
}

inline Message messageNoFileContents(Document::Ptr &document, const QString &fileName,
                                     unsigned line)
{
    const QString text = QCoreApplication::translate(
        "CppSourceProcessor", "%1: Could not get file contents").arg(fileName);
    return Message(Message::Warning, document->fileName(), line, /*column =*/ 0, text);
}

101
inline const Macro revision(const WorkingCopy &workingCopy,
102 103 104 105 106 107 108 109 110
                            const Macro &macro)
{
    Macro newMacro(macro);
    newMacro.setFileRevision(workingCopy.get(macro.fileName()).second);
    return newMacro;
}

} // anonymous namespace

111
CppSourceProcessor::CppSourceProcessor(const Snapshot &snapshot, DocumentCallback documentFinished)
112
    : m_snapshot(snapshot),
113
      m_documentFinished(documentFinished),
114
      m_preprocess(this, &m_env),
115
      m_languageFeatures(LanguageFeatures::defaultFeatures()),
116
      m_defaultCodec(Core::EditorManager::defaultTextCodec())
117 118 119 120
{
    m_preprocess.setKeepComments(true);
}

121
CppSourceProcessor::~CppSourceProcessor()
122 123
{ }

124 125 126 127 128
void CppSourceProcessor::setCancelChecker(const CppSourceProcessor::CancelChecker &cancelChecker)
{
    m_preprocess.setCancelChecker(cancelChecker);
}

129
void CppSourceProcessor::setWorkingCopy(const WorkingCopy &workingCopy)
130 131
{ m_workingCopy = workingCopy; }

132
void CppSourceProcessor::setHeaderPaths(const ProjectPartHeaderPaths &headerPaths)
133
{
134
    m_headerPaths.clear();
135

136
    for (int i = 0, ei = headerPaths.size(); i < ei; ++i) {
137
        const ProjectPartHeaderPath &path = headerPaths.at(i);
138

139 140
        if (path.type == ProjectPartHeaderPath::IncludePath)
            m_headerPaths.append(ProjectPartHeaderPath(cleanPath(path.path), path.type));
141 142 143
        else
            addFrameworkPath(path);
    }
144 145
}

146 147 148 149 150
void CppSourceProcessor::setLanguageFeatures(const LanguageFeatures languageFeatures)
{
    m_languageFeatures = languageFeatures;
}

151 152 153 154 155 156 157
// Add the given framework path, and expand private frameworks.
//
// Example:
//  <framework-path>/ApplicationServices.framework
// has private frameworks in:
//  <framework-path>/ApplicationServices.framework/Frameworks
// if the "Frameworks" folder exists inside the top level framework.
158
void CppSourceProcessor::addFrameworkPath(const ProjectPartHeaderPath &frameworkPath)
159
{
160 161
    QTC_ASSERT(frameworkPath.isFrameworkPath(), return);

162 163 164
    // The algorithm below is a bit too eager, but that's because we're not getting
    // in the frameworks we're linking against. If we would have that, then we could
    // add only those private frameworks.
165 166
    const ProjectPartHeaderPath cleanFrameworkPath(cleanPath(frameworkPath.path),
                                                   frameworkPath.type);
167 168
    if (!m_headerPaths.contains(cleanFrameworkPath))
        m_headerPaths.append(cleanFrameworkPath);
169

170
    const QDir frameworkDir(cleanFrameworkPath.path);
171
    const QStringList filter = QStringList("*.framework");
172 173 174
    foreach (const QFileInfo &framework, frameworkDir.entryInfoList(filter)) {
        if (!framework.isDir())
            continue;
Nikolai Kosjar's avatar
Nikolai Kosjar committed
175 176
        const QFileInfo privateFrameworks(framework.absoluteFilePath(),
                                          QLatin1String("Frameworks"));
177
        if (privateFrameworks.exists() && privateFrameworks.isDir())
178 179
            addFrameworkPath(ProjectPartHeaderPath(privateFrameworks.absoluteFilePath(),
                                                   frameworkPath.type));
180 181 182
    }
}

183 184 185 186
void CppSourceProcessor::setTodo(const QSet<QString> &files)
{
    m_todo = files;
}
187

188 189
void CppSourceProcessor::run(const QString &fileName,
                             const QStringList &initialIncludes)
190
{
191
    sourceNeeded(0, fileName, IncludeGlobal, initialIncludes);
192 193
}

194
void CppSourceProcessor::removeFromCache(const QString &fileName)
195 196 197 198
{
    m_snapshot.remove(fileName);
}

199
void CppSourceProcessor::resetEnvironment()
200 201 202
{
    m_env.reset();
    m_processed.clear();
203
    m_included.clear();
204 205
}

206 207 208
bool CppSourceProcessor::getFileContents(const QString &absoluteFilePath,
                                         QByteArray *contents,
                                         unsigned *revision) const
209
{
210 211
    if (absoluteFilePath.isEmpty() || !contents || !revision)
        return false;
212

213
    // Get from working copy
214
    if (m_workingCopy.contains(absoluteFilePath)) {
215
        const QPair<QByteArray, unsigned> entry = m_workingCopy.get(absoluteFilePath);
216 217 218
        *contents = entry.first;
        *revision = entry.second;
        return true;
219 220
    }

221 222 223 224 225 226 227 228
    // Get from file
    *revision = 0;
    QString error;
    if (Utils::TextFileFormat::readFileUTF8(absoluteFilePath, m_defaultCodec, contents, &error)
            != Utils::TextFileFormat::ReadSuccess) {
        qWarning("Error reading file \"%s\": \"%s\".", qPrintable(absoluteFilePath),
                 qPrintable(error));
        return false;
229
    }
230
    return true;
231 232
}

233
bool CppSourceProcessor::checkFile(const QString &absoluteFilePath) const
234
{
235 236 237
    if (absoluteFilePath.isEmpty()
            || m_included.contains(absoluteFilePath)
            || m_workingCopy.contains(absoluteFilePath)) {
238
        return true;
239
    }
240

Nikolai Kosjar's avatar
Nikolai Kosjar committed
241
    const QFileInfo fileInfo(absoluteFilePath);
242 243 244
    return fileInfo.isFile() && fileInfo.isReadable();
}

245
QString CppSourceProcessor::cleanPath(const QString &path)
246 247 248 249 250 251 252 253
{
    QString result = QDir::cleanPath(path);
    const QChar slash(QLatin1Char('/'));
    if (!result.endsWith(slash))
        result.append(slash);
    return result;
}

254 255
/// Resolve the given file name to its absolute path w.r.t. the include type.
QString CppSourceProcessor::resolveFile(const QString &fileName, IncludeType type)
256
{
257
    if (isInjectedFile(fileName))
258 259
        return fileName;

260 261 262
    if (QFileInfo(fileName).isAbsolute())
        return checkFile(fileName) ? fileName : QString();

263 264 265 266 267 268 269 270 271 272 273 274
    if (m_currentDoc) {
        if (type == IncludeLocal) {
            const QFileInfo currentFileInfo(m_currentDoc->fileName());
            const QString path = cleanPath(currentFileInfo.absolutePath()) + fileName;
            if (checkFile(path))
                return path;
            // Fall through! "16.2 Source file inclusion" from the standard states to continue
            // searching as if this would be a global include.

        } else if (type == IncludeNext) {
            const QFileInfo currentFileInfo(m_currentDoc->fileName());
            const QString currentDirPath = cleanPath(currentFileInfo.dir().path());
275 276
            auto headerPathsEnd = m_headerPaths.end();
            auto headerPathsIt = m_headerPaths.begin();
277 278 279
            for (; headerPathsIt != headerPathsEnd; ++headerPathsIt) {
                if (headerPathsIt->path == currentDirPath) {
                    ++headerPathsIt;
280
                    return resolveFile_helper(fileName, headerPathsIt);
281 282 283
                }
            }
        }
284 285
    }

286 287 288 289 290 291 292 293 294 295 296 297 298
    QHash<QString, QString>::ConstIterator it = m_fileNameCache.constFind(fileName);
    if (it != m_fileNameCache.constEnd())
        return it.value();
    const QString fn = resolveFile_helper(fileName, m_headerPaths.begin());
    if (!fn.isEmpty())
        m_fileNameCache.insert(fileName, fn);
    return fn;
}

QString CppSourceProcessor::resolveFile_helper(const QString &fileName,
                                               ProjectPartHeaderPaths::Iterator headerPathsIt)
{
    auto headerPathsEnd = m_headerPaths.end();
Nikolai Kosjar's avatar
Nikolai Kosjar committed
299
    const int index = fileName.indexOf(QLatin1Char('/'));
300 301 302 303 304 305 306 307 308 309 310 311
    for (; headerPathsIt != headerPathsEnd; ++headerPathsIt) {
        if (headerPathsIt->isValid()) {
            QString path;
            if (headerPathsIt->isFrameworkPath()) {
                if (index == -1)
                    continue;
                path = headerPathsIt->path + fileName.left(index)
                       + QLatin1String(".framework/Headers/") + fileName.mid(index + 1);
            } else {
                path = headerPathsIt->path + fileName;
            }
            if (m_workingCopy.contains(path) || checkFile(path))
312 313 314 315 316 317 318
                return path;
        }
    }

    return QString();
}

319
void CppSourceProcessor::macroAdded(const Macro &macro)
320
{
321
    if (!m_currentDoc)
322 323 324 325 326
        return;

    m_currentDoc->appendMacro(macro);
}

327 328
void CppSourceProcessor::passedMacroDefinitionCheck(unsigned bytesOffset, unsigned utf16charsOffset,
                                                    unsigned line, const Macro &macro)
329
{
330
    if (!m_currentDoc)
331 332
        return;

333 334 335 336
    m_currentDoc->addMacroUse(revision(m_workingCopy, macro),
                              bytesOffset, macro.name().length(),
                              utf16charsOffset, macro.nameToQString().size(),
                              line, QVector<MacroArgumentReference>());
337 338
}

339 340
void CppSourceProcessor::failedMacroDefinitionCheck(unsigned bytesOffset, unsigned utf16charOffset,
                                                    const ByteArrayRef &name)
341
{
342
    if (!m_currentDoc)
343 344
        return;

345 346
    m_currentDoc->addUndefinedMacroUse(QByteArray(name.start(), name.size()),
                                       bytesOffset, utf16charOffset);
347 348
}

349 350
void CppSourceProcessor::notifyMacroReference(unsigned bytesOffset, unsigned utf16charOffset,
                                              unsigned line, const Macro &macro)
351
{
352
    if (!m_currentDoc)
353 354
        return;

355 356 357 358
    m_currentDoc->addMacroUse(revision(m_workingCopy, macro),
                              bytesOffset, macro.name().length(),
                              utf16charOffset, macro.nameToQString().size(),
                              line, QVector<MacroArgumentReference>());
359 360
}

361 362 363
void CppSourceProcessor::startExpandingMacro(unsigned bytesOffset, unsigned utf16charOffset,
                                             unsigned line, const Macro &macro,
                                             const QVector<MacroArgumentReference> &actuals)
364
{
365
    if (!m_currentDoc)
366 367
        return;

368 369 370 371
    m_currentDoc->addMacroUse(revision(m_workingCopy, macro),
                              bytesOffset, macro.name().length(),
                              utf16charOffset, macro.nameToQString().size(),
                              line, actuals);
372 373
}

374
void CppSourceProcessor::stopExpandingMacro(unsigned, const Macro &)
375
{
376
    if (!m_currentDoc)
377 378 379
        return;
}

380
void CppSourceProcessor::markAsIncludeGuard(const QByteArray &macroName)
381 382 383 384 385 386 387
{
    if (!m_currentDoc)
        return;

    m_currentDoc->setIncludeGuardMacroName(macroName);
}

388
void CppSourceProcessor::mergeEnvironment(Document::Ptr doc)
389
{
390
    if (!doc)
391 392 393 394 395 396 397 398 399
        return;

    const QString fn = doc->fileName();

    if (m_processed.contains(fn))
        return;

    m_processed.insert(fn);

400
    foreach (const Document::Include &incl, doc->resolvedIncludes()) {
Nikolai Kosjar's avatar
Nikolai Kosjar committed
401
        const QString includedFile = incl.resolvedFileName();
402 403 404

        if (Document::Ptr includedDoc = m_snapshot.document(includedFile))
            mergeEnvironment(includedDoc);
Nikolai Kosjar's avatar
Nikolai Kosjar committed
405
        else if (!m_included.contains(includedFile))
406 407 408 409 410 411
            run(includedFile);
    }

    m_env.addMacros(doc->definedMacros());
}

412
void CppSourceProcessor::startSkippingBlocks(unsigned utf16charsOffset)
413 414
{
    if (m_currentDoc)
415
        m_currentDoc->startSkippingBlocks(utf16charsOffset);
416 417
}

418
void CppSourceProcessor::stopSkippingBlocks(unsigned utf16charsOffset)
419 420
{
    if (m_currentDoc)
421
        m_currentDoc->stopSkippingBlocks(utf16charsOffset);
422 423
}

424 425
void CppSourceProcessor::sourceNeeded(unsigned line, const QString &fileName, IncludeType type,
                                      const QStringList &initialIncludes)
426 427 428 429 430 431
{
    if (fileName.isEmpty())
        return;

    QString absoluteFileName = resolveFile(fileName, type);
    absoluteFileName = QDir::cleanPath(absoluteFileName);
432
    if (m_currentDoc) {
433
        m_currentDoc->addIncludeFile(Document::Include(fileName, absoluteFileName, line, type));
434
        if (absoluteFileName.isEmpty()) {
435
            m_currentDoc->addDiagnosticMessage(messageNoSuchFile(m_currentDoc, fileName, line));
436 437 438
            return;
        }
    }
439
    if (m_included.contains(absoluteFileName))
440
        return; // We've already seen this file.
441
    if (!isInjectedFile(absoluteFileName))
442 443
        m_included.insert(absoluteFileName);

444
    // Already in snapshot? Use it!
445 446
    if (Document::Ptr document = m_snapshot.document(absoluteFileName)) {
        mergeEnvironment(document);
447 448 449
        return;
    }

450
    const QFileInfo info(absoluteFileName);
451
    if (fileSizeExceedsLimit(info, m_fileSizeLimitInMb))
452 453
        return; // TODO: Add diagnostic message

454
    // Otherwise get file contents
455
    unsigned editorRevision = 0;
456
    QByteArray contents;
457 458
    const bool gotFileContents = getFileContents(absoluteFileName, &contents, &editorRevision);
    if (m_currentDoc && !gotFileContents) {
459
        m_currentDoc->addDiagnosticMessage(messageNoFileContents(m_currentDoc, fileName, line));
460
        return;
461 462
    }

463
    qCDebug(log) << "Parsing:" << absoluteFileName << "contents:" << contents.size() << "bytes";
464

465 466
    Document::Ptr document = Document::create(absoluteFileName);
    document->setEditorRevision(editorRevision);
467
    document->setLanguageFeatures(m_languageFeatures);
468 469 470 471 472
    foreach (const QString &include, initialIncludes) {
        m_included.insert(include);
        Document::Include inc(include, include, 0, IncludeLocal);
        document->addIncludeFile(inc);
    }
473
    if (info.exists())
474
        document->setLastModified(info.lastModified());
475

476
    const Document::Ptr previousDocument = switchCurrentDocument(document);
477
    const QByteArray preprocessedCode = m_preprocess.run(absoluteFileName, contents);
Nikolai Kosjar's avatar
Nikolai Kosjar committed
478
//    {
479 480
//        QByteArray b(preprocessedCode); b.replace("\n", "<<<\n");
//        qDebug("Preprocessed code for \"%s\": [[%s]]", fileName.toUtf8().constData(), b.constData());
Nikolai Kosjar's avatar
Nikolai Kosjar committed
481
//    }
482 483 484 485 486 487 488 489
    document->setFingerprint(generateFingerPrint(document->definedMacros(), preprocessedCode));

    // Re-use document from global snapshot if possible
    Document::Ptr globalDocument = m_globalSnapshot.document(absoluteFileName);
    if (globalDocument && globalDocument->fingerprint() == document->fingerprint()) {
        switchCurrentDocument(previousDocument);
        mergeEnvironment(globalDocument);
        m_snapshot.insert(globalDocument);
490 491 492 493
        m_todo.remove(absoluteFileName);
        return;
    }

494 495 496 497 498 499
    // Otherwise process the document
    document->setUtf8Source(preprocessedCode);
    document->keepSourceAndAST();
    document->tokenize();
    document->check(m_workingCopy.contains(document->fileName()) ? Document::FullCheck
                                                                 : Document::FastCheck);
500

501
    m_documentFinished(document);
502

503 504 505
    m_snapshot.insert(document);
    m_todo.remove(absoluteFileName);
    switchCurrentDocument(previousDocument);
506 507
}

508 509 510 511 512
void CppSourceProcessor::setFileSizeLimitInMb(int fileSizeLimitInMb)
{
    m_fileSizeLimitInMb = fileSizeLimitInMb;
}

513
Document::Ptr CppSourceProcessor::switchCurrentDocument(Document::Ptr doc)
514
{
Nikolai Kosjar's avatar
Nikolai Kosjar committed
515
    const Document::Ptr previousDoc = m_currentDoc;
516 517 518
    m_currentDoc = doc;
    return previousDoc;
}