pp-engine.cpp 47.9 KB
Newer Older
1
2
/**************************************************************************
**
con's avatar
con committed
3
4
** This file is part of Qt Creator
**
hjk's avatar
hjk committed
5
** Copyright (c) 2012 Nokia Corporation and/or its subsidiary(-ies).
con's avatar
con committed
6
**
7
** Contact: Nokia Corporation (qt-info@nokia.com)
con's avatar
con committed
8
**
9
**
10
** GNU Lesser General Public License Usage
11
**
hjk's avatar
hjk committed
12
13
14
15
16
17
** This file may be used under the terms of the GNU Lesser General Public
** License version 2.1 as published by the Free Software Foundation and
** appearing in the file LICENSE.LGPL included in the packaging of this file.
** Please review the following information to ensure the GNU Lesser General
** Public License version 2.1 requirements will be met:
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
18
**
con's avatar
con committed
19
** In addition, as a special exception, Nokia gives you certain additional
hjk's avatar
hjk committed
20
** rights. These rights are described in the Nokia Qt LGPL Exception
con's avatar
con committed
21
22
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
hjk's avatar
hjk committed
23
24
25
26
27
** Other Usage
**
** Alternatively, this file may be used in accordance with the terms and
** conditions contained in a signed written agreement between you and Nokia.
**
con's avatar
con committed
28
** If you have questions regarding the use of this file, please contact
29
** Nokia at qt-info@nokia.com.
con's avatar
con committed
30
**
31
**************************************************************************/
con's avatar
con committed
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
/*
  Copyright 2005 Roberto Raggi <roberto@kdevelop.org>

  Permission to use, copy, modify, distribute, and sell this software and its
  documentation for any purpose is hereby granted without fee, provided that
  the above copyright notice appear in all copies and that both that
  copyright notice and this permission notice appear in supporting
  documentation.

  The above copyright notice and this permission notice shall be included in
  all copies or substantial portions of the Software.

  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
  KDEVELOP TEAM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

#include "pp.h"
53
#include "pp-cctype.h"
hjk's avatar
hjk committed
54

55
#include <Control.h>
con's avatar
con committed
56
57
#include <Lexer.h>
#include <Token.h>
58
#include <Literals.h>
59
#include <cctype>
60

con's avatar
con committed
61
#include <QtDebug>
62
#include <algorithm>
63
#include <list>
64
#include <QList>
65
66
67
68
69
70
71
72
73
#include <QDate>
#include <QTime>

#define NO_DEBUG

#ifndef NO_DEBUG
#  include <iostream>
#endif // NO_DEBUG

74
75
#include <deque>

76
77
78
namespace {
enum {
    eagerExpansion = 1,
79
80
    MAX_TOKEN_EXPANSION_COUNT = 5000,
    MAX_TOKEN_BUFFER_DEPTH = 16000, // for when macros are using some kind of right-folding, this is the list of "delayed" buffers waiting to be expanded after the current one.
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
};
}

namespace {
template<typename _T>
class ScopedSwap
{
    _T oldValue;
    _T &ref;

public:
    ScopedSwap(_T &var, _T newValue)
        : oldValue(newValue)
        , ref(var)
    {
        std::swap(ref, oldValue);
    }

    ~ScopedSwap()
    {
        std::swap(ref, oldValue);
    }
};
typedef ScopedSwap<bool> ScopedBoolSwap;
typedef ScopedSwap<unsigned> ScopedUnsignedSwap;
} // anonymous namespace
con's avatar
con committed
107

Roberto Raggi's avatar
Roberto Raggi committed
108
109
namespace CPlusPlus {

110
111
112
namespace Internal {
struct TokenBuffer
{
113
    std::deque<PPToken> tokens;
114
115
116
    const Macro *macro;
    TokenBuffer *next;

117
118
    TokenBuffer(const PPToken *start, const PPToken *end, const Macro *macro, TokenBuffer *next)
        : tokens(start, end), macro(macro), next(next)
119
120
    {}

121
122
123
124
    bool isBlocked(const Macro *macro) const {
        if (!macro)
            return false;

125
        for (const TokenBuffer *it = this; it; it = it->next)
126
            if (it->macro == macro && it->macro->name() == macro->name())
127
128
129
130
131
                return true;
        return false;
    }
};

Roberto Raggi's avatar
Roberto Raggi committed
132
133
134
135
struct Value
{
    enum Kind {
        Kind_Long,
136
        Kind_ULong
Roberto Raggi's avatar
Roberto Raggi committed
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
    };

    Kind kind;

    union {
        long l;
        unsigned long ul;
    };


    Value()
        : kind(Kind_Long), l(0)
    { }

    inline bool is_ulong () const
    { return kind == Kind_ULong; }

    inline void set_ulong (unsigned long v)
    {
        ul = v;
        kind = Kind_ULong;
    }

    inline void set_long (long v)
    {
        l = v;
        kind = Kind_Long;
    }

    inline bool is_zero () const
    { return l == 0; }

#define PP_DEFINE_BIN_OP(name, op) \
    inline Value operator op(const Value &other) const \
    { \
        Value v = *this; \
        if (v.is_ulong () || other.is_ulong ()) \
            v.set_ulong (v.ul op other.ul); \
        else \
            v.set_long (v.l op other.l); \
        return v; \
    }

    PP_DEFINE_BIN_OP(op_add, +)
    PP_DEFINE_BIN_OP(op_sub, -)
    PP_DEFINE_BIN_OP(op_mult, *)
    PP_DEFINE_BIN_OP(op_div, /)
    PP_DEFINE_BIN_OP(op_mod, %)
    PP_DEFINE_BIN_OP(op_lhs, <<)
    PP_DEFINE_BIN_OP(op_rhs, >>)
    PP_DEFINE_BIN_OP(op_lt, <)
    PP_DEFINE_BIN_OP(op_gt, >)
    PP_DEFINE_BIN_OP(op_le, <=)
    PP_DEFINE_BIN_OP(op_ge, >=)
    PP_DEFINE_BIN_OP(op_eq, ==)
    PP_DEFINE_BIN_OP(op_ne, !=)
    PP_DEFINE_BIN_OP(op_bit_and, &)
    PP_DEFINE_BIN_OP(op_bit_or, |)
    PP_DEFINE_BIN_OP(op_bit_xor, ^)
    PP_DEFINE_BIN_OP(op_and, &&)
    PP_DEFINE_BIN_OP(op_or, ||)

#undef PP_DEFINE_BIN_OP
};

202
} // namespace Internal
203
} // namespace CPlusPlus
Roberto Raggi's avatar
Roberto Raggi committed
204

con's avatar
con committed
205
using namespace CPlusPlus;
206
using namespace CPlusPlus::Internal;
Roberto Raggi's avatar
Roberto Raggi committed
207

con's avatar
con committed
208
209
namespace {

210
211
212
213
214
inline bool isValidToken(const PPToken &tk)
{
    return tk.isNot(T_EOF_SYMBOL) && (! tk.newline() || tk.joined());
}

215
Macro *macroDefinition(const ByteArrayRef &name, unsigned offset, Environment *env, Client *client)
Christian Kamm's avatar
Christian Kamm committed
216
217
218
219
220
221
222
223
{
    Macro *m = env->resolve(name);
    if (client) {
        if (m)
            client->passedMacroDefinitionCheck(offset, *m);
        else
            client->failedMacroDefinitionCheck(offset, name);
    }
224
    return m;
Christian Kamm's avatar
Christian Kamm committed
225
226
}

con's avatar
con committed
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
class RangeLexer
{
    const Token *first;
    const Token *last;
    Token trivial;

public:
    inline RangeLexer(const Token *first, const Token *last)
        : first(first), last(last)
    {
        // WARN: `last' must be a valid iterator.
        trivial.offset = last->offset;
    }

    inline operator bool() const
    { return first != last; }

    inline bool isValid() const
    { return first != last; }

    inline int size() const
    { return std::distance(first, last); }

    inline const Token *dot() const
    { return first; }

    inline const Token &operator*() const
    {
        if (first != last)
            return *first;

        return trivial;
    }

    inline const Token *operator->() const
    {
        if (first != last)
            return first;

        return &trivial;
    }

    inline RangeLexer &operator++()
    {
        ++first;
        return *this;
    }
};

class ExpressionEvaluator
{
    ExpressionEvaluator(const ExpressionEvaluator &other);
    void operator = (const ExpressionEvaluator &other);

public:
Christian Kamm's avatar
Christian Kamm committed
282
283
    ExpressionEvaluator(Client *client, Environment *env)
        : client(client), env(env), _lex(0)
con's avatar
con committed
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
    { }

    Value operator()(const Token *firstToken, const Token *lastToken,
                     const QByteArray &source)
    {
        this->source = source;
        const Value previousValue = switchValue(Value());
        RangeLexer tmp(firstToken, lastToken);
        RangeLexer *previousLex = _lex;
        _lex = &tmp;
        process_expression();
        _lex = previousLex;
        return switchValue(previousValue);
    }

protected:
    Value switchValue(const Value &value)
    {
        Value previousValue = _value;
        _value = value;
        return previousValue;
    }

    bool isTokenDefined() const
    {
        if ((*_lex)->isNot(T_IDENTIFIER))
            return false;
311
        const ByteArrayRef spell = tokenSpell();
con's avatar
con committed
312
313
314
315
316
        if (spell.size() != 7)
            return false;
        return spell == "defined";
    }

317
318
319
320
321
322
323
324
325
326
    const char *tokenPosition() const
    {
        return source.constData() + (*_lex)->offset;
    }

    int tokenLength() const
    {
        return (*_lex)->f.length;
    }

327
    ByteArrayRef tokenSpell() const
con's avatar
con committed
328
    {
329
        return ByteArrayRef(tokenPosition(), tokenLength());
con's avatar
con committed
330
331
    }

332
333
    inline void process_expression()
    { process_constant_expression(); }
con's avatar
con committed
334

335
    void process_primary()
con's avatar
con committed
336
    {
337
        if ((*_lex)->is(T_NUMERIC_LITERAL)) {
338
339
340
341
            const char *spell = tokenPosition();
            int len = tokenLength();
            while (len) {
                const char ch = spell[len - 1];
342

343
                if (! (ch == 'u' || ch == 'U' || ch == 'l' || ch == 'L'))
344
                    break;
345
                --len;
346
347
            }

348
349
350
            const char *end = spell + len;
            char *vend = const_cast<char *>(end);
            _value.set_long(strtol(spell, &vend, 0));
con's avatar
con committed
351
352
353
354
            ++(*_lex);
        } else if (isTokenDefined()) {
            ++(*_lex);
            if ((*_lex)->is(T_IDENTIFIER)) {
355
                _value.set_long(macroDefinition(tokenSpell(), (*_lex)->offset, env, client) != 0);
con's avatar
con committed
356
357
358
359
                ++(*_lex);
            } else if ((*_lex)->is(T_LPAREN)) {
                ++(*_lex);
                if ((*_lex)->is(T_IDENTIFIER)) {
360
                    _value.set_long(macroDefinition(tokenSpell(), (*_lex)->offset, env, client) != 0);
con's avatar
con committed
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
                    ++(*_lex);
                    if ((*_lex)->is(T_RPAREN)) {
                        ++(*_lex);
                    }
                }
            }
        } else if ((*_lex)->is(T_IDENTIFIER)) {
            _value.set_long(0);
            ++(*_lex);
        } else if ((*_lex)->is(T_MINUS)) {
            ++(*_lex);
            process_primary();
            _value.set_long(- _value.l);
        } else if ((*_lex)->is(T_PLUS)) {
            ++(*_lex);
            process_primary();
Roberto Raggi's avatar
Roberto Raggi committed
377
378
379
380
        } else if ((*_lex)->is(T_TILDE)) {
            ++(*_lex);
            process_primary();
            _value.set_long(~ _value.l);
con's avatar
con committed
381
382
383
384
385
386
387
388
389
390
391
392
        } else if ((*_lex)->is(T_EXCLAIM)) {
            ++(*_lex);
            process_primary();
            _value.set_long(_value.is_zero());
        } else if ((*_lex)->is(T_LPAREN)) {
            ++(*_lex);
            process_expression();
            if ((*_lex)->is(T_RPAREN))
                ++(*_lex);
        }
    }

393
    Value process_expression_with_operator_precedence(const Value &lhs, int minPrecedence)
con's avatar
con committed
394
    {
395
        Value result = lhs;
con's avatar
con committed
396

397
398
399
        while (precedence((*_lex)->kind()) >= minPrecedence) {
            const int oper = (*_lex)->kind();
            const int operPrecedence = precedence(oper);
con's avatar
con committed
400
401
            ++(*_lex);
            process_primary();
402
            Value rhs = _value;
con's avatar
con committed
403

404
            for (int LA_token_kind = (*_lex)->kind(), LA_precedence = precedence(LA_token_kind);
405
                    LA_precedence > operPrecedence && isBinaryOperator(LA_token_kind);
406
407
                    LA_token_kind = (*_lex)->kind(), LA_precedence = precedence(LA_token_kind)) {
                rhs = process_expression_with_operator_precedence(rhs, LA_precedence);
con's avatar
con committed
408
409
            }

410
            result = evaluate_expression(oper, result, rhs);
con's avatar
con committed
411
412
        }

413
        return result;
con's avatar
con committed
414
415
    }

416
    void process_constant_expression()
con's avatar
con committed
417
    {
418
419
        process_primary();
        _value = process_expression_with_operator_precedence(_value, precedence(T_PIPE_PIPE));
con's avatar
con committed
420

421
422
        if ((*_lex)->is(T_QUESTION)) {
            const Value cond = _value;
con's avatar
con committed
423
            ++(*_lex);
424
425
426
427
428
429
430
431
            process_constant_expression();
            Value left = _value, right;
            if ((*_lex)->is(T_COLON)) {
                ++(*_lex);
                process_constant_expression();
                right = _value;
            }
            _value = ! cond.is_zero() ? left : right;
con's avatar
con committed
432
433
434
        }
    }

435
436
private:
    inline int precedence(int tokenKind) const
con's avatar
con committed
437
    {
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
        switch (tokenKind) {
        case T_PIPE_PIPE:       return 0;
        case T_AMPER_AMPER:     return 1;
        case T_PIPE:            return 2;
        case T_CARET:           return 3;
        case T_AMPER:           return 4;
        case T_EQUAL_EQUAL:
        case T_EXCLAIM_EQUAL:   return 5;
        case T_GREATER:
        case T_LESS:
        case T_LESS_EQUAL:
        case T_GREATER_EQUAL:   return 6;
        case T_LESS_LESS:
        case T_GREATER_GREATER: return 7;
        case T_PLUS:
        case T_MINUS:           return 8;
        case T_STAR:
        case T_SLASH:
        case T_PERCENT:         return 9;

        default:
            return -1;
con's avatar
con committed
460
461
462
        }
    }

463
    static inline bool isBinaryOperator(int tokenKind)
con's avatar
con committed
464
    {
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
        switch (tokenKind) {
        case T_PIPE_PIPE:
        case T_AMPER_AMPER:
        case T_PIPE:
        case T_CARET:
        case T_AMPER:
        case T_EQUAL_EQUAL:
        case T_EXCLAIM_EQUAL:
        case T_GREATER:
        case T_LESS:
        case T_LESS_EQUAL:
        case T_GREATER_EQUAL:
        case T_LESS_LESS:
        case T_GREATER_GREATER:
        case T_PLUS:
        case T_MINUS:
        case T_STAR:
        case T_SLASH:
        case T_PERCENT:
            return true;
con's avatar
con committed
485

486
487
        default:
            return false;
con's avatar
con committed
488
489
490
        }
    }

491
    static inline Value evaluate_expression(int tokenKind, const Value &lhs, const Value &rhs)
con's avatar
con committed
492
    {
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
        switch (tokenKind) {
        case T_PIPE_PIPE:       return lhs || rhs;
        case T_AMPER_AMPER:     return lhs && rhs;
        case T_PIPE:            return lhs | rhs;
        case T_CARET:           return lhs ^ rhs;
        case T_AMPER:           return lhs & rhs;
        case T_EQUAL_EQUAL:     return lhs == rhs;
        case T_EXCLAIM_EQUAL:   return lhs != rhs;
        case T_GREATER:         return lhs > rhs;
        case T_LESS:            return lhs < rhs;
        case T_LESS_EQUAL:      return lhs <= rhs;
        case T_GREATER_EQUAL:   return lhs >= rhs;
        case T_LESS_LESS:       return lhs << rhs;
        case T_GREATER_GREATER: return lhs >> rhs;
        case T_PLUS:            return lhs + rhs;
        case T_MINUS:           return lhs - rhs;
        case T_STAR:            return lhs * rhs;
        case T_SLASH:           return rhs.is_zero() ? Value() : lhs / rhs;
        case T_PERCENT:         return rhs.is_zero() ? Value() : lhs % rhs;

        default:
            return Value();
con's avatar
con committed
515
516
517
518
        }
    }

private:
Christian Kamm's avatar
Christian Kamm committed
519
    Client *client;
con's avatar
con committed
520
521
522
523
524
525
526
527
    Environment *env;
    QByteArray source;
    RangeLexer *_lex;
    Value _value;
};

} // end of anonymous namespace

528
529
530
531
532
533
Preprocessor::State::State()
    : m_lexer(0)
    , m_skipping(MAX_LEVEL)
    , m_trueTest(MAX_LEVEL)
    , m_ifLevel(0)
    , m_tokenBuffer(0)
534
    , m_tokenBufferDepth(0)
535
536
537
538
539
540
    , m_inPreprocessorDirective(false)
    , m_result(0)
    , m_markGeneratedTokens(true)
    , m_noLines(false)
    , m_inCondition(false)
    , m_inDefine(false)
con's avatar
con committed
541
{
542
543
    m_skipping[m_ifLevel] = false;
    m_trueTest[m_ifLevel] = false;
con's avatar
con committed
544
545
}

546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
void Preprocessor::State::pushTokenBuffer(const PPToken *start, const PPToken *end, const Macro *macro)
{
    if (m_tokenBufferDepth <= MAX_TOKEN_BUFFER_DEPTH) {
#ifdef COMPRESS_TOKEN_BUFFER
        // This does not work correctly for boost's preprocessor library, or that library exposes a bug in the code.
        if (macro || !m_tokenBuffer) {
            m_tokenBuffer = new TokenBuffer(start, end, macro, m_tokenBuffer);
            ++m_tokenBufferDepth;
        } else {
            m_tokenBuffer->tokens.insert(m_tokenBuffer->tokens.begin(), start, end);
        }
//        qDebug()<<"New depth:" << m_tokenBufferDepth << "with buffer size:" << m_tokenBuffer->tokens.size();
#else
        m_tokenBuffer = new TokenBuffer(start, end, macro, m_tokenBuffer);
        ++m_tokenBufferDepth;
#endif
    } else {
        //### Should we tell the user that his source is insane?
//        qDebug() << "Macro insanity level reached in" << m_currentFileName;
    }
}

void Preprocessor::State::popTokenBuffer()
{
    TokenBuffer *r = m_tokenBuffer;
    m_tokenBuffer = m_tokenBuffer->next;
    delete r;

    if (m_tokenBufferDepth)
        --m_tokenBufferDepth;
}
577
578
579
580
581
582

Preprocessor::Preprocessor(Client *client, Environment *env)
    : m_client(client)
    , m_env(env)
    , m_expandMacros(true)
    , m_keepComments(false)
con's avatar
con committed
583
584
585
{
}

586
QByteArray Preprocessor::run(const QString &fileName, const QString &source)
587
{
588
    return run(fileName, source.toLatin1());
589
590
}

591
592
593
594
QByteArray Preprocessor::run(const QString &fileName,
                             const QByteArray &source,
                             bool noLines,
                             bool markGeneratedTokens)
595
596
{
    QByteArray preprocessed;
597
598
//    qDebug()<<"running" << fileName<<"with"<<source.count('\n')<<"lines...";
    preprocess(fileName, source, &preprocessed, noLines, markGeneratedTokens, false);
599
600
601
    return preprocessed;
}

602
603
bool Preprocessor::expandMacros() const
{
604
    return m_expandMacros;
605
606
607
608
}

void Preprocessor::setExpandMacros(bool expandMacros)
{
609
    m_expandMacros = expandMacros;
610
611
}

612
613
bool Preprocessor::keepComments() const
{
614
    return m_keepComments;
615
616
617
618
}

void Preprocessor::setKeepComments(bool keepComments)
{
619
    m_keepComments = keepComments;
620
621
}

622
623
624
625
626
627
628
629
630
631
void Preprocessor::genLine(unsigned lineno, const QByteArray &fileName) const
{
    startNewOutputLine();
    out("# ");
    out(QByteArray::number(lineno));
    out(" \"");
    out(fileName);
    out("\"\n");
}

632
void Preprocessor::handleDefined(PPToken *tk)
633
{
634
    ScopedBoolSwap s(m_state.m_inPreprocessorDirective, true);
635
636
637
638
639
640
641
    unsigned lineno = tk->lineno;
    lex(tk); // consume "defined" token
    bool lparenSeen = tk->is(T_LPAREN);
    if (lparenSeen)
        lex(tk); // consume "(" token
    if (tk->isNot(T_IDENTIFIER))
        //### TODO: generate error message
642
        return;
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
    PPToken idToken = *tk;
    do {
        lex(tk);
        if (tk->isNot(T_POUND_POUND))
            break;
        lex(tk);
        if (tk->is(T_IDENTIFIER))
            idToken = generateConcatenated(idToken, *tk);
        else
            break;
    } while (isValidToken(*tk));
    pushToken(tk);
    QByteArray result(1, '0');
    if (m_env->resolve(idToken.asByteArrayRef()))
        result[0] = '1';
658
    *tk = generateToken(T_NUMERIC_LITERAL, result.constData(), result.size(), lineno, false);
659
660
}

661
void Preprocessor::pushToken(Preprocessor::PPToken *tk)
662
{
663
    const PPToken currentTokenBuffer[] = { *tk };
664
    m_state.pushTokenBuffer(currentTokenBuffer, currentTokenBuffer + 1, 0);
665
}
666

667
668
669
670
671
void Preprocessor::lex(PPToken *tk)
{
_Lagain:
    if (m_state.m_tokenBuffer) {
        if (m_state.m_tokenBuffer->tokens.empty()) {
672
            m_state.popTokenBuffer();
673
674
675
676
677
678
679
680
            goto _Lagain;
        }
        *tk = m_state.m_tokenBuffer->tokens.front();
        m_state.m_tokenBuffer->tokens.pop_front();
    } else {
        tk->setSource(m_state.m_source);
        m_state.m_lexer->scan(tk);
    }
681

682
683
//    if (tk->isValid() && !tk->generated() && !tk->is(T_EOF_SYMBOL))
//        m_env->currentLine = tk->lineno;
684
685
686
687
688
689
690
691
692
693
694
695
696

_Lclassify:
    if (! m_state.m_inPreprocessorDirective) {
        if (tk->newline() && tk->is(T_POUND)) {
            handlePreprocessorDirective(tk);
            goto _Lclassify;
        } else if (tk->newline() && skipping()) {
            ScopedBoolSwap s(m_state.m_inPreprocessorDirective, true);
            do {
                lex(tk);
            } while (isValidToken(*tk));
            goto _Lclassify;
        } else if (tk->is(T_IDENTIFIER) && !isQtReservedWord(tk->asByteArrayRef())) {
697
            if (m_state.m_inCondition && tk->asByteArrayRef() == "defined")
698
699
700
                handleDefined(tk);
            else if (handleIdentifier(tk))
                goto _Lagain;
701
702
703
704
        }
    }
}

705
void Preprocessor::skipPreprocesorDirective(PPToken *tk)
706
{
707
708
709
710
    ScopedBoolSwap s(m_state.m_inPreprocessorDirective, true);

    while (isValidToken(*tk)) {
        lex(tk);
711
712
713
    }
}

714
bool Preprocessor::handleIdentifier(PPToken *tk)
715
{
716
717
718
719
720
721
722
723
724
725
726
727
728
729
    ScopedBoolSwap s(m_state.m_inPreprocessorDirective, true);

    static const QByteArray ppLine("__LINE__");
    static const QByteArray ppFile("__FILE__");
    static const QByteArray ppDate("__DATE__");
    static const QByteArray ppTime("__TIME__");

    ByteArrayRef macroNameRef = tk->asByteArrayRef();
    bool newline = tk->newline();

    if (!m_state.m_inDefine && macroNameRef.size() == 8 && macroNameRef[0] == '_' && macroNameRef[1] == '_') {
        PPToken newTk;
        if (macroNameRef == ppLine) {
            QByteArray txt = QByteArray::number(tk->lineno);
730
            newTk = generateToken(T_STRING_LITERAL, txt.constData(), txt.size(), tk->lineno, false);
731
732
733
734
735
        } else if (macroNameRef == ppFile) {
            QByteArray txt;
            txt.append('"');
            txt.append(m_env->currentFile.toUtf8());
            txt.append('"');
736
            newTk = generateToken(T_STRING_LITERAL, txt.constData(), txt.size(), tk->lineno, false);
737
738
739
740
741
        } else if (macroNameRef == ppDate) {
            QByteArray txt;
            txt.append('"');
            txt.append(QDate::currentDate().toString().toUtf8());
            txt.append('"');
742
            newTk = generateToken(T_STRING_LITERAL, txt.constData(), txt.size(), tk->lineno, false);
743
744
745
746
747
        } else if (macroNameRef == ppTime) {
            QByteArray txt;
            txt.append('"');
            txt.append(QTime::currentTime().toString().toUtf8());
            txt.append('"');
748
            newTk = generateToken(T_STRING_LITERAL, txt.constData(), txt.size(), tk->lineno, false);
749
750
        }

751
752
753
754
755
        if (newTk.isValid()) {
            newTk.f.newline = newline;
            newTk.f.whitespace = tk->whitespace();
            *tk = newTk;
            return false;
756
757
758
        }
    }

759
    Macro *macro = m_env->resolve(macroNameRef);
760
761
    if (!macro)
        return false;
762
763
    if (tk->generated() && m_state.m_tokenBuffer && m_state.m_tokenBuffer->isBlocked(macro))
        return false;
764
//    qDebug() << "expanding" << macro->name() << "on line" << tk->lineno;
765

766
    if (m_client && !tk->generated())
767
        m_client->startExpandingMacro(tk->offset, *macro, macroNameRef);
768
    QVector<PPToken> body = macro->definitionTokens();
769

770
    if (macro->isFunctionLike()) {
771
        if (!expandMacros() || !handleFunctionLikeMacro(tk, macro, body, !m_state.m_inDefine)) {
772
            // the call is not function like or expandMacros() returns false, so stop
773
774
            if (m_client && !tk->generated())
                m_client->stopExpandingMacro(tk->offset, *macro);
775
            return false;
776
        }
777

778
    }
779

780
781
782
783
784
785
786
787
788
    if (body.isEmpty()) {
        if (!m_state.m_inDefine) {
            // macro expanded to empty, so characters disappeared, hence force a re-indent.
            PPToken forceWhitespacingToken;
            // special case: for a macro that expanded to empty, we do not want
            // to generate a new #line and re-indent, but just generate the
            // amount of spaces that the macro name took up.
            forceWhitespacingToken.f.length = tk->length() + (tk->whitespace() ? 1 : 0);
            body.push_front(forceWhitespacingToken);
789
        }
790
791
792
793
794
    } else {
        PPToken &firstNewTk = body.first();
        firstNewTk.f.newline = newline;
        firstNewTk.f.whitespace = true; // the macro call is removed, so space the first token correctly.
    }
795

796
    m_state.pushTokenBuffer(body.begin(), body.end(), macro);
797

798
    if (m_client && !tk->generated())
799
800
801
        m_client->stopExpandingMacro(tk->offset, *macro);

    return true;
802
803
}

804
bool Preprocessor::handleFunctionLikeMacro(PPToken *tk, const Macro *macro, QVector<PPToken> &body, bool addWhitespaceMarker)
con's avatar
con committed
805
{
806
807
808
809
810
811
812
    QVector<QVector<PPToken> > actuals;
    PPToken idToken = *tk;
    if (!collectActualArguments(tk, &actuals)) {
        pushToken(tk);
        *tk = idToken;
        return false;
    }
813

814
    QVector<PPToken> expanded;
815
    expanded.reserve(MAX_TOKEN_EXPANSION_COUNT);
816
817
    for (size_t i = 0, bodySize = body.size(); i < bodySize && expanded.size() < MAX_TOKEN_EXPANSION_COUNT; ++i) {
        int expandedSize = expanded.size();
818
        const PPToken &token = body.at(i);
819
820
821
822
823
824
825
826
827
828
829

        if (token.is(T_IDENTIFIER)) {
            const ByteArrayRef id = token.asByteArrayRef();
            const QVector<QByteArray> &formals = macro->formals();
            int j = 0;
            for (; j < formals.size() && expanded.size() < MAX_TOKEN_EXPANSION_COUNT; ++j) {
                if (formals[j] == id) {
                    if (actuals.size() <= j) {
                        // too few actual parameters
                        //### TODO: error message
                        goto exitNicely;
830
                    }
831

832
                    QVector<PPToken> actualsForThisParam = actuals.at(j);
833
                    if (id == "__VA_ARGS__" || (macro->isVariadic() && j + 1 == formals.size())) {
834
                        unsigned lineno = 0;
835
                        const char comma = ',';
836
837
838
                        for (int k = j + 1; k < actuals.size(); ++k) {
                            if (!actualsForThisParam.isEmpty())
                                lineno = actualsForThisParam.last().lineno;
839
                            actualsForThisParam.append(generateToken(T_COMMA, &comma, 1, lineno, true));
840
                            actualsForThisParam += actuals[k];
841
                        }
842
                    }
Roberto Raggi's avatar
Roberto Raggi committed
843

844
845
846
847
848
849
850
851
852
853
854
                    if (i > 0  && body[i - 1].is(T_POUND)) {
                        QByteArray newText;
                        newText.reserve(256);
                        unsigned lineno = 0;
                        for (int i = 0, ei = actualsForThisParam.size(); i < ei; ++i) {
                            const PPToken &t = actualsForThisParam.at(i);
                            if (i == 0)
                                lineno = t.lineno;
                            else if (t.whitespace())
                                newText.append(' ');
                            newText.append(t.start(), t.length());
855
                        }
856
857
                        newText.replace("\\", "\\\\");
                        newText.replace("\"", "\\\"");
858
                        expanded.push_back(generateToken(T_STRING_LITERAL, newText.constData(), newText.size(), lineno, true));
859
                    } else  {
860
861
                        for (int k = 0, kk = actualsForThisParam.size(); k < kk; ++k)
                            expanded += actualsForThisParam.at(k);
862
                    }
863
                    break;
864
                }
con's avatar
con committed
865
866
            }

867
868
869
870
871
            if (j == formals.size())
                expanded.push_back(token);
        } else if (token.isNot(T_POUND) && token.isNot(T_POUND_POUND)) {
            expanded.push_back(token);
        }
Roberto Raggi's avatar
Roberto Raggi committed
872

873
874
875
876
877
878
879
880
881
        if (i > 1 && body[i - 1].is(T_POUND_POUND)) {
            if (expandedSize < 1 || expanded.size() == expandedSize) //### TODO: [cpp.concat] placemarkers
                continue;
            const PPToken &leftTk = expanded[expandedSize - 1];
            const PPToken &rightTk = expanded[expandedSize];
            expanded[expandedSize - 1] = generateConcatenated(leftTk, rightTk);
            expanded.remove(expandedSize);
        }
    }
Roberto Raggi's avatar
Roberto Raggi committed
882

883
884
885
886
887
888
889
exitNicely:
    pushToken(tk);
    if (addWhitespaceMarker) {
        PPToken forceWhitespacingToken;
        expanded.push_front(forceWhitespacingToken);
    }
    body = expanded;
890
    body.squeeze();
891
    return true;
con's avatar
con committed
892
893
}

894
895
896
897
/// invalid pp-tokens are used as markers to force whitespace checks.
void Preprocessor::preprocess(const QString &fileName, const QByteArray &source,
                              QByteArray *result, bool noLines,
                              bool markGeneratedTokens, bool inCondition)
898
{
899
    if (source.isEmpty())
900
        return;
901

902
    const State savedState = m_state;
hjk's avatar
hjk committed
903
904
905
906
907
908
909
910
911
912
913
914
915

    m_state = State();
    m_state.m_currentFileName = fileName;
    m_state.m_source = source;
    m_state.m_lexer = new Lexer(source.constBegin(), source.constEnd());
    m_state.m_lexer->setScanKeywords(false);
    m_state.m_lexer->setScanAngleStringLiteralTokens(false);
    if (m_keepComments)
        m_state.m_lexer->setScanCommentTokens(true);
    m_state.m_result = result;
    m_state.m_noLines = noLines;
    m_state.m_markGeneratedTokens = markGeneratedTokens;
    m_state.m_inCondition = inCondition;
916

917
918
    const QString previousFileName = m_env->currentFile;
    m_env->currentFile = fileName;
919

920
    const unsigned previousCurrentLine = m_env->currentLine;
921
    m_env->currentLine = 1;
922

923
    const QByteArray fn = fileName.toUtf8();
924
925
    if (!m_state.m_noLines)
        genLine(1, fn);
926

927
928
929
930
931
    PPToken tk(m_state.m_source), prevTk;
    do {
_Lrestart:
        bool forceLine = false;
        lex(&tk);
932

933
934
935
936
937
938
        if (!tk.isValid()) {
            bool wasGenerated = prevTk.generated();
            prevTk = tk;
            prevTk.f.generated = wasGenerated;
            goto _Lrestart;
        }
939

940
941
942
        if (m_state.m_markGeneratedTokens && tk.generated() && !prevTk.generated()) {
            startNewOutputLine();
            out("#gen true\n");
943
            ++m_env->currentLine;
944
945
946
947
            forceLine = true;
        } else if (m_state.m_markGeneratedTokens && !tk.generated() && prevTk.generated()) {
            startNewOutputLine();
            out("#gen false\n");
948
            ++m_env->currentLine;
949
950
            forceLine = true;
        }
951

952
953
        if (forceLine || m_env->currentLine != tk.lineno) {
            if (forceLine || m_env->currentLine > tk.lineno || tk.lineno - m_env->currentLine > 3) {
954
955
956
957
                if (m_state.m_noLines) {
                    if (!m_state.m_markGeneratedTokens)
                        out(' ');
                } else {
958
                    genLine(tk.lineno, fn);
959
                }
960
            } else {
961
                for (unsigned i = m_env->currentLine; i < tk.lineno; ++i)
962
                    out('\n');
963
            }
964
965
966
        } else {
            if (tk.newline() && prevTk.isValid())
                out('\n');
967
        }
968

969
970
971
        if (tk.whitespace() || prevTk.generated() != tk.generated() || !prevTk.isValid()) {
            if (prevTk.generated() && tk.generated()) {
                out(' ');
972
            } else if (tk.isValid() && !prevTk.isValid() && tk.lineno == m_env->currentLine) {
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
                out(QByteArray(prevTk.length() + (tk.whitespace() ? 1 : 0), ' '));
            } else if (prevTk.generated() != tk.generated() || !prevTk.isValid()) {
                const char *begin = tk.source().constBegin();
                const char *end = begin + tk.offset;
                const char *it = end - 1;
                for (; it >= begin; --it)
                    if (*it == '\n')
                        break;
                ++it;
                for (; it < end; ++it)
                    out(' ');
            } else {
                const char *begin = tk.source().constBegin();
                const char *end = begin + tk.offset;
                const char *it = end - 1;
                for (; it >= begin; --it)
                    if (!pp_isspace(*it) || *it == '\n')
                        break;
                ++it;
                for (; it < end; ++it)
                    out(*it);
994
995
996
            }
        }

997
998
        const ByteArrayRef tkBytes = tk.asByteArrayRef();
        out(tkBytes);
999
        m_env->currentLine = tk.lineno;
1000
        if (tk.is(T_COMMENT) || tk.is(T_DOXY_COMMENT))
1001
            m_env->currentLine += tkBytes.count('\n');
1002
1003
        prevTk = tk;
    } while (tk.isNot(T_EOF_SYMBOL));
1004

1005
1006
    delete m_state.m_lexer;
    m_state = savedState;
1007

1008
1009
    m_env->currentFile = previousFileName;
    m_env->currentLine = previousCurrentLine;
1010
1011
}

1012
bool Preprocessor::collectActualArguments(PPToken *tk, QVector<QVector<PPToken> > *actuals)
1013
{
1014
1015
    Q_ASSERT(tk);
    Q_ASSERT(actuals);
1016

1017
    lex(tk); // consume the identifier
1018

1019
1020
1021
    if (tk->isNot(T_LPAREN))
        //### TODO: error message
        return false;
1022

1023
1024
1025
    QVector<PPToken> tokens;
    lex(tk);
    scanActualArgument(tk, &tokens);
1026

1027
    actuals->append(tokens);
1028

1029
1030
    while (tk->is(T_COMMA)) {
        lex(tk);
1031

1032
1033
1034
        QVector<PPToken> tokens;
        scanActualArgument(tk, &tokens);
        actuals->append(tokens);
1035
1036
    }

1037
1038
1039
1040
    if (tk->is(T_RPAREN))
        lex(tk);
    //###TODO: else error message
    return true;
con's avatar
con committed
1041
1042
}

1043
void Preprocessor::scanActualArgument(PPToken *tk, QVector<PPToken> *tokens)
con's avatar
con committed
1044
{
1045
    Q_ASSERT(tokens);
con's avatar
con committed
1046

1047
    int count = 0;
con's avatar
con committed
1048

1049
1050
1051
1052
1053
1054
1055
1056
    while (tk->isNot(T_EOF_SYMBOL)) {
        if (tk->is(T_LPAREN)) {
            ++count;
        } else if (tk->is(T_RPAREN)) {
            if (! count)
                break;
            --count;
        } else if (! count && tk->is(T_COMMA)) {
con's avatar
con committed
1057
            break;
1058
        }
con's avatar
con committed
1059

1060
1061
        tokens->append(*tk);
        lex(tk);
con's avatar
con committed
1062
1063
1064
    }
}

1065
void Preprocessor::handlePreprocessorDirective(PPToken *tk)
con's avatar
con committed
1066
{
1067
    ScopedBoolSwap s(m_state.m_inPreprocessorDirective, true);
1068

1069
1070
    PPToken poundToken = *tk;
    lex(tk); // scan the directive
1071

1072
1073
    if (tk->newline() && ! tk->joined())
        return; // nothing to do.
con's avatar
con committed
1074

1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
    static const QByteArray ppDefine("define");
    static const QByteArray ppIf("if");
    static const QByteArray ppIfDef("ifdef");
    static const QByteArray ppIfNDef("ifndef");
    static const QByteArray ppEndIf("endif");
    static const QByteArray ppElse("else");
    static const QByteArray ppUndef("undef");
    static const QByteArray ppElif("elif");
    static const QByteArray ppInclude("include");
    static const QByteArray ppIncludeNext("include_next");
    static const QByteArray ppImport("import");
    //### TODO:
    // line
    // error
    // pragma
con's avatar
con committed
1090

1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
    if (tk->is(T_IDENTIFIER)) {
        const ByteArrayRef directive = tk->asByteArrayRef();

        if (!skipping() && directive == ppDefine)
            handleDefineDirective(tk);
        else if (!skipping() && directive == ppUndef)
            handleUndefDirective(tk);
        else if (!skipping() && (directive == ppInclude
                                 || directive == ppIncludeNext
                                 || directive == ppImport))
            handleIncludeDirective(tk);
        else if (directive == ppIf)
            handleIfDirective(tk);
        else if (directive == ppIfDef)
            handleIfDefDirective(false, tk);
        else if (directive == ppIfNDef)
            handleIfDefDirective(true, tk);
        else if (directive == ppEndIf)
            handleEndIfDirective(tk, poundToken);
        else if (directive == ppElse)
            handleElseDirective(tk, poundToken);
        else if (directive == ppElif)
            handleElifDirective(tk, poundToken);

        skipPreprocesorDirective(tk);
    }
}
1118
1119


1120
1121
1122
1123
1124
void Preprocessor::handleIncludeDirective(PPToken *tk)
{
    m_state.m_lexer->setScanAngleStringLiteralTokens(true);
    lex(tk); // consume "include" token
    m_state.m_lexer->setScanAngleStringLiteralTokens(false);
Erik Verbruggen's avatar
Erik Verbruggen committed
1125
    const unsigned line = tk->lineno;
1126
    QByteArray included;
con's avatar
con committed
1127

1128
1129
    if (tk->is(T_STRING_LITERAL) || tk->is(T_ANGLE_STRING_LITERAL)) {
        included = tk->asByteArrayRef().toByteArray();
Erik Verbruggen's avatar
Erik Verbruggen committed
1130
        lex(tk); // consume string token
1131
1132
    } else {
        included = expand(tk);
con's avatar
con committed
1133
    }
1134
1135
    included = included.trimmed();

1136
1137
1138
1139
1140
    if (included.isEmpty()) {
        //### TODO: error message
        return;
    }

1141
1142
1143
1144
1145
1146
1147
1148
1149
//    qDebug("include [[%s]]", included.toUtf8().constData());
    Client::IncludeType mode;
    if (included.at(0) == '"')
        mode = Client::IncludeLocal;
    else if (included.at(0) == '<')
        mode = Client::IncludeGlobal;
    else
        return; //### TODO: add error message?

hjk's avatar
hjk committed
1150
1151
    if (m_client) {
        QString inc = QString::fromUtf8(included.constData() + 1, included.size() - 2);
1152
        m_client->sourceNeeded(inc, mode, line);
hjk's avatar
hjk committed
1153
    }
con's avatar
con committed
1154
1155
}

1156
void Preprocessor::handleDefineDirective(PPToken *tk)
con's avatar
con committed
1157
{
1158
1159
    const unsigned defineOffset = tk->offset;
    lex(tk); // consume "define" token
con's avatar
con committed
1160

1161
1162
    bool hasIdentifier = false;
    if (tk->isNot(T_IDENTIFIER))
con's avatar
con committed
1163
        return;
1164
1165

    ScopedBoolSwap inDefine(m_state.m_inDefine, true);
con's avatar
con committed
1166
1167

    Macro macro;
1168
    macro.setFileName(m_env->currentFile);
1169
    macro.setLine(tk->lineno);
1170
1171
1172
    QByteArray macroName = tk->asByteArrayRef().toByteArray();
    macro.setName(macroName);
    macro.setOffset(tk->offset);
con's avatar
con committed
1173

Erik Verbruggen's avatar