cpppreprocessor.cpp 15.8 KB
Newer Older
1
2
#include "cpppreprocessor.h"

Nikolai Kosjar's avatar
Nikolai Kosjar committed
3
4
#include "cppmodelmanager.h"

5
6
7
#include <coreplugin/editormanager/editormanager.h>

#include <utils/fileutils.h>
8
#include <utils/hostosinfo.h>
9
#include <utils/textfileformat.h>
10
11

#include <QCoreApplication>
12
#include <QCryptographicHash>
13
#include <QTextCodec>
14

15
16
17
18
19
20
21
22
23
24
25
26
/*!
 * \class CppTools::Internal::CppPreprocessor
 * \brief The CppPreprocessor class updates set of indexed C++ files.
 *
 * Indexed file is truncated version of fully parsed document: copy of source
 * code and full AST will be dropped when indexing is done. Working copy ensures
 * that documents with most recent copy placed in memory will be parsed correctly.
 *
 * \sa CPlusPlus::Document
 * \sa CppTools::CppModelManagerInterface::WorkingCopy
 */

27
28
29
30
using namespace CPlusPlus;
using namespace CppTools;
using namespace CppTools::Internal;

Nikolai Kosjar's avatar
Nikolai Kosjar committed
31
32
CppPreprocessor::CppPreprocessor(QPointer<CppModelManager> modelManager,
                                 bool dumpFileNameWhileParsing)
33
34
35
36
    : m_snapshot(modelManager->snapshot()),
      m_modelManager(modelManager),
      m_dumpFileNameWhileParsing(dumpFileNameWhileParsing),
      m_preprocess(this, &m_env),
37
38
      m_revision(0),
      m_defaultCodec(Core::EditorManager::defaultTextCodec())
39
40
41
42
{
    m_preprocess.setKeepComments(true);
}

43
44
45
46
47
48
CppPreprocessor::CppPreprocessor(QPointer<CppModelManager> modelManager, const Snapshot &snapshot,
                                 bool dumpFileNameWhileParsing)
    : m_snapshot(snapshot),
      m_modelManager(modelManager),
      m_dumpFileNameWhileParsing(dumpFileNameWhileParsing),
      m_preprocess(this, &m_env),
49
50
      m_revision(0),
      m_defaultCodec(Core::EditorManager::defaultTextCodec())
51
52
53
54
{
    m_preprocess.setKeepComments(true);
}

55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
CppPreprocessor::~CppPreprocessor()
{ }

void CppPreprocessor::setRevision(unsigned revision)
{ m_revision = revision; }

void CppPreprocessor::setWorkingCopy(const CppModelManagerInterface::WorkingCopy &workingCopy)
{ m_workingCopy = workingCopy; }

void CppPreprocessor::setIncludePaths(const QStringList &includePaths)
{
    m_includePaths.clear();

    for (int i = 0; i < includePaths.size(); ++i) {
        const QString &path = includePaths.at(i);

        if (Utils::HostOsInfo::isMacHost()) {
            if (i + 1 < includePaths.size() && path.endsWith(QLatin1String(".framework/Headers"))) {
                const QFileInfo pathInfo(path);
                const QFileInfo frameworkFileInfo(pathInfo.path());
                const QString frameworkName = frameworkFileInfo.baseName();

                const QFileInfo nextIncludePath = includePaths.at(i + 1);
                if (nextIncludePath.fileName() == frameworkName) {
                    // We got a QtXXX.framework/Headers followed by $QTDIR/include/QtXXX.
                    // In this case we prefer to include files from $QTDIR/include/QtXXX.
                    continue;
                }
            }
        }
        m_includePaths.append(cleanPath(path));
    }
}

void CppPreprocessor::setFrameworkPaths(const QStringList &frameworkPaths)
{
    m_frameworkPaths.clear();
Nikolai Kosjar's avatar
Nikolai Kosjar committed
92
    foreach (const QString &frameworkPath, frameworkPaths)
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
        addFrameworkPath(frameworkPath);
}

// Add the given framework path, and expand private frameworks.
//
// Example:
//  <framework-path>/ApplicationServices.framework
// has private frameworks in:
//  <framework-path>/ApplicationServices.framework/Frameworks
// if the "Frameworks" folder exists inside the top level framework.
void CppPreprocessor::addFrameworkPath(const QString &frameworkPath)
{
    // The algorithm below is a bit too eager, but that's because we're not getting
    // in the frameworks we're linking against. If we would have that, then we could
    // add only those private frameworks.
Nikolai Kosjar's avatar
Nikolai Kosjar committed
108
    const QString cleanFrameworkPath = cleanPath(frameworkPath);
109
110
111
112
113
114
115
116
    if (!m_frameworkPaths.contains(cleanFrameworkPath))
        m_frameworkPaths.append(cleanFrameworkPath);

    const QDir frameworkDir(cleanFrameworkPath);
    const QStringList filter = QStringList() << QLatin1String("*.framework");
    foreach (const QFileInfo &framework, frameworkDir.entryInfoList(filter)) {
        if (!framework.isDir())
            continue;
Nikolai Kosjar's avatar
Nikolai Kosjar committed
117
118
        const QFileInfo privateFrameworks(framework.absoluteFilePath(),
                                          QLatin1String("Frameworks"));
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
        if (privateFrameworks.exists() && privateFrameworks.isDir())
            addFrameworkPath(privateFrameworks.absoluteFilePath());
    }
}

void CppPreprocessor::setTodo(const QStringList &files)
{ m_todo = QSet<QString>::fromList(files); }

namespace {
class Process: public std::unary_function<Document::Ptr, void>
{
    QPointer<CppModelManager> _modelManager;
    Document::Ptr _doc;
    Document::CheckMode _mode;

public:
    Process(QPointer<CppModelManager> modelManager,
            Document::Ptr doc,
            const CppModelManager::WorkingCopy &workingCopy)
        : _modelManager(modelManager),
          _doc(doc),
          _mode(Document::FastCheck)
    {
        if (workingCopy.contains(_doc->fileName()))
            _mode = Document::FullCheck;
    }

    void operator()()
    {
        _doc->check(_mode);

150
        if (_modelManager) {
151
            _modelManager->emitDocumentUpdated(_doc);
152
153
            _doc->releaseSourceAndAST();
        }
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
    }
};
} // end of anonymous namespace

void CppPreprocessor::run(const QString &fileName)
{
    sourceNeeded(0, fileName, IncludeGlobal);
}

void CppPreprocessor::removeFromCache(const QString &fileName)
{
    m_snapshot.remove(fileName);
}

void CppPreprocessor::resetEnvironment()
{
    m_env.reset();
    m_processed.clear();
172
    m_included.clear();
173
174
}

175
bool CppPreprocessor::getFileContents(const QString &absoluteFilePath,
176
                                      QByteArray *contents,
177
178
                                      unsigned *revision) const
{
179
180
    if (absoluteFilePath.isEmpty() || !contents || !revision)
        return false;
181

182
    // Get from working copy
183
    if (m_workingCopy.contains(absoluteFilePath)) {
184
        const QPair<QByteArray, unsigned> entry = m_workingCopy.get(absoluteFilePath);
185
186
187
        *contents = entry.first;
        *revision = entry.second;
        return true;
188
189
    }

190
191
192
193
194
195
196
197
    // Get from file
    *revision = 0;
    QString error;
    if (Utils::TextFileFormat::readFileUTF8(absoluteFilePath, m_defaultCodec, contents, &error)
            != Utils::TextFileFormat::ReadSuccess) {
        qWarning("Error reading file \"%s\": \"%s\".", qPrintable(absoluteFilePath),
                 qPrintable(error));
        return false;
198
    }
199
    return true;
200
201
202
203
}

bool CppPreprocessor::checkFile(const QString &absoluteFilePath) const
{
204
205
206
    if (absoluteFilePath.isEmpty()
            || m_included.contains(absoluteFilePath)
            || m_workingCopy.contains(absoluteFilePath)) {
207
        return true;
208
    }
209

Nikolai Kosjar's avatar
Nikolai Kosjar committed
210
    const QFileInfo fileInfo(absoluteFilePath);
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
    return fileInfo.isFile() && fileInfo.isReadable();
}

/// Resolve the given file name to its absolute path w.r.t. the include type.
QString CppPreprocessor::resolveFile(const QString &fileName, IncludeType type)
{
    if (type == IncludeGlobal) {
        QHash<QString, QString>::ConstIterator it = m_fileNameCache.find(fileName);
        if (it != m_fileNameCache.end())
            return it.value();
        const QString fn = resolveFile_helper(fileName, type);
        m_fileNameCache.insert(fileName, fn);
        return fn;
    }

    // IncludeLocal, IncludeNext
    return resolveFile_helper(fileName, type);
}

QString CppPreprocessor::cleanPath(const QString &path)
{
    QString result = QDir::cleanPath(path);
    const QChar slash(QLatin1Char('/'));
    if (!result.endsWith(slash))
        result.append(slash);
    return result;
}

QString CppPreprocessor::resolveFile_helper(const QString &fileName, IncludeType type)
{
241
    if (isInjectedFile(fileName))
242
243
        return fileName;

244
245
246
    if (QFileInfo(fileName).isAbsolute())
        return checkFile(fileName) ? fileName : QString();

247
    if (type == IncludeLocal && m_currentDoc) {
Nikolai Kosjar's avatar
Nikolai Kosjar committed
248
249
        const QFileInfo currentFileInfo(m_currentDoc->fileName());
        const QString path = cleanPath(currentFileInfo.absolutePath()) + fileName;
250
251
        if (checkFile(path))
            return path;
252
253
        // Fall through! "16.2 Source file inclusion" from the standard states to continue
        // searching as if this would be a global include.
254
255
256
    }

    foreach (const QString &includePath, m_includePaths) {
Nikolai Kosjar's avatar
Nikolai Kosjar committed
257
        const QString path = includePath + fileName;
258
        if (m_workingCopy.contains(path) || checkFile(path))
259
260
261
            return path;
    }

Nikolai Kosjar's avatar
Nikolai Kosjar committed
262
    const int index = fileName.indexOf(QLatin1Char('/'));
263
    if (index != -1) {
Nikolai Kosjar's avatar
Nikolai Kosjar committed
264
265
266
        const QString frameworkName = fileName.left(index);
        const QString name = frameworkName + QLatin1String(".framework/Headers/")
            + fileName.mid(index + 1);
267
268

        foreach (const QString &frameworkPath, m_frameworkPaths) {
Nikolai Kosjar's avatar
Nikolai Kosjar committed
269
            const QString path = frameworkPath + name;
270
271
272
273
274
275
276
277
278
279
            if (checkFile(path))
                return path;
        }
    }

    return QString();
}

void CppPreprocessor::macroAdded(const Macro &macro)
{
280
    if (!m_currentDoc)
281
282
283
284
285
        return;

    m_currentDoc->appendMacro(macro);
}

Nikolai Kosjar's avatar
Nikolai Kosjar committed
286
287
static inline const Macro revision(const CppModelManagerInterface::WorkingCopy &s,
                                   const Macro &macro)
288
289
290
291
292
293
294
295
{
    Macro newMacro(macro);
    newMacro.setFileRevision(s.get(macro.fileName()).second);
    return newMacro;
}

void CppPreprocessor::passedMacroDefinitionCheck(unsigned offset, unsigned line, const Macro &macro)
{
296
    if (!m_currentDoc)
297
298
299
300
301
302
303
304
        return;

    m_currentDoc->addMacroUse(revision(m_workingCopy, macro), offset, macro.name().length(), line,
                              QVector<MacroArgumentReference>());
}

void CppPreprocessor::failedMacroDefinitionCheck(unsigned offset, const ByteArrayRef &name)
{
305
    if (!m_currentDoc)
306
307
308
309
310
311
312
        return;

    m_currentDoc->addUndefinedMacroUse(QByteArray(name.start(), name.size()), offset);
}

void CppPreprocessor::notifyMacroReference(unsigned offset, unsigned line, const Macro &macro)
{
313
    if (!m_currentDoc)
314
315
316
317
318
319
320
321
322
323
        return;

    m_currentDoc->addMacroUse(revision(m_workingCopy, macro), offset, macro.name().length(), line,
                              QVector<MacroArgumentReference>());
}

void CppPreprocessor::startExpandingMacro(unsigned offset, unsigned line,
                                          const Macro &macro,
                                          const QVector<MacroArgumentReference> &actuals)
{
324
    if (!m_currentDoc)
325
326
        return;

Nikolai Kosjar's avatar
Nikolai Kosjar committed
327
328
    m_currentDoc->addMacroUse(revision(m_workingCopy, macro), offset, macro.name().length(), line,
                              actuals);
329
330
331
332
}

void CppPreprocessor::stopExpandingMacro(unsigned, const Macro &)
{
333
    if (!m_currentDoc)
334
335
336
337
338
339
340
341
342
343
344
345
346
        return;
}

void CppPreprocessor::markAsIncludeGuard(const QByteArray &macroName)
{
    if (!m_currentDoc)
        return;

    m_currentDoc->setIncludeGuardMacroName(macroName);
}

void CppPreprocessor::mergeEnvironment(Document::Ptr doc)
{
347
    if (!doc)
348
349
350
351
352
353
354
355
356
        return;

    const QString fn = doc->fileName();

    if (m_processed.contains(fn))
        return;

    m_processed.insert(fn);

357
    foreach (const Document::Include &incl, doc->resolvedIncludes()) {
Nikolai Kosjar's avatar
Nikolai Kosjar committed
358
        const QString includedFile = incl.resolvedFileName();
359
360
361

        if (Document::Ptr includedDoc = m_snapshot.document(includedFile))
            mergeEnvironment(includedDoc);
Nikolai Kosjar's avatar
Nikolai Kosjar committed
362
        else if (!m_included.contains(includedFile))
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
            run(includedFile);
    }

    m_env.addMacros(doc->definedMacros());
}

void CppPreprocessor::startSkippingBlocks(unsigned offset)
{
    if (m_currentDoc)
        m_currentDoc->startSkippingBlocks(offset);
}

void CppPreprocessor::stopSkippingBlocks(unsigned offset)
{
    if (m_currentDoc)
        m_currentDoc->stopSkippingBlocks(offset);
}

381
382
383
384
385
386
387
388
389
390
391
392
// This is a temporary fix to handle non-ascii characters. This can be removed when the lexer can
// handle multi-byte characters.
static QByteArray convertToLatin1(const QByteArray &contents)
{
    const char *p = contents.constData();
    while (char ch = *p++)
        if (ch & 0x80)
            return QString::fromUtf8(contents).toLatin1();

    return contents;
}

393
394
void CppPreprocessor::sourceNeeded(unsigned line, const QString &fileName, IncludeType type)
{
395
    typedef Document::DiagnosticMessage Message;
396
397
398
399
400
    if (fileName.isEmpty())
        return;

    QString absoluteFileName = resolveFile(fileName, type);
    absoluteFileName = QDir::cleanPath(absoluteFileName);
401
    if (m_currentDoc) {
402
        m_currentDoc->addIncludeFile(Document::Include(fileName, absoluteFileName, line, type));
403
404
405
406
407
408
409
410
        if (absoluteFileName.isEmpty()) {
            const QString text = QCoreApplication::translate(
                "CppPreprocessor", "%1: No such file or directory").arg(fileName);
            Message message(Message::Warning, m_currentDoc->fileName(), line, /*column =*/ 0, text);
            m_currentDoc->addDiagnosticMessage(message);
            return;
        }
    }
411
412
413
414
415
    if (m_included.contains(absoluteFileName))
        return; // we've already seen this file.
    if (absoluteFileName != modelManager()->configurationFileName())
        m_included.insert(absoluteFileName);

416
417
418
419
420
421
422
423
    // Already in snapshot? Use it!
    Document::Ptr doc = m_snapshot.document(absoluteFileName);
    if (doc) {
        mergeEnvironment(doc);
        return;
    }

    // Otherwise get file contents
424
    unsigned editorRevision = 0;
425
    QByteArray contents;
426
    const bool gotFileContents = getFileContents(absoluteFileName, &contents, &editorRevision);
427
    contents = convertToLatin1(contents);
428
429
430
431
432
433
    if (m_currentDoc && !gotFileContents) {
        const QString text = QCoreApplication::translate(
            "CppPreprocessor", "%1: Could not get file contents").arg(fileName);
        Message message(Message::Warning, m_currentDoc->fileName(), line, /*column =*/ 0, text);
        m_currentDoc->addDiagnosticMessage(message);
        return;
434
435
436
437
    }

    if (m_dumpFileNameWhileParsing) {
        qDebug() << "Parsing file:" << absoluteFileName
438
                 << "contents:" << contents.size() << "bytes";
439
440
441
442
443
444
    }

    doc = Document::create(absoluteFileName);
    doc->setRevision(m_revision);
    doc->setEditorRevision(editorRevision);

Nikolai Kosjar's avatar
Nikolai Kosjar committed
445
    const QFileInfo info(absoluteFileName);
446
447
448
    if (info.exists())
        doc->setLastModified(info.lastModified());

Nikolai Kosjar's avatar
Nikolai Kosjar committed
449
    const Document::Ptr previousDoc = switchDocument(doc);
450
451

    const QByteArray preprocessedCode = m_preprocess.run(absoluteFileName, contents);
Nikolai Kosjar's avatar
Nikolai Kosjar committed
452
453
454
455
456
457
//    {
//        QByteArray b(preprocessedCode);
//        b.replace("\n", "<<<\n");
//        qDebug("Preprocessed code for \"%s\": [[%s]]", fileName.toUtf8().constData(),
//               b.constData());
//    }
458

459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
    QCryptographicHash hash(QCryptographicHash::Sha1);
    hash.addData(preprocessedCode);
    foreach (const Macro &macro, doc->definedMacros()) {
        if (macro.isHidden()) {
            static const QByteArray undef("#undef ");
            hash.addData(undef);
            hash.addData(macro.name());
        } else {
            static const QByteArray def("#define ");
            hash.addData(macro.name());
            hash.addData(" ", 1);
            hash.addData(def);
            hash.addData(macro.definitionText());
        }
        hash.addData("\n", 1);
    }
    doc->setFingerprint(hash.result());

    Document::Ptr anotherDoc = m_globalSnapshot.document(absoluteFileName);
    if (anotherDoc && anotherDoc->fingerprint() == doc->fingerprint()) {
        switchDocument(previousDoc);
        mergeEnvironment(anotherDoc);
        m_snapshot.insert(anotherDoc);
        m_todo.remove(absoluteFileName);
        return;
    }

486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
    doc->setUtf8Source(preprocessedCode);
    doc->keepSourceAndAST();
    doc->tokenize();

    m_snapshot.insert(doc);
    m_todo.remove(absoluteFileName);

    Process process(m_modelManager, doc, m_workingCopy);
    process();

    (void) switchDocument(previousDoc);
}

Document::Ptr CppPreprocessor::switchDocument(Document::Ptr doc)
{
Nikolai Kosjar's avatar
Nikolai Kosjar committed
501
    const Document::Ptr previousDoc = m_currentDoc;
502
503
504
    m_currentDoc = doc;
    return previousDoc;
}