qmljsscanner.cpp 8.75 KB
Newer Older
Tobias Hunger's avatar
Tobias Hunger committed
1
2
3
4
/**************************************************************************
**
** This file is part of Qt Creator
**
hjk's avatar
hjk committed
5
** Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
Tobias Hunger's avatar
Tobias Hunger committed
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
**
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** Commercial Usage
**
** Licensees holding valid Qt Commercial licenses may use this file in
** accordance with the Qt Commercial License Agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Nokia.
**
** GNU Lesser General Public License Usage
**
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** If you are unsure which license is appropriate for your use, please
** contact the sales department at http://qt.nokia.com/contact.
**
**************************************************************************/

30
#include <qmljs/qmljsscanner.h>
31
32
33

#include <QTextCharFormat>

34
using namespace QmlJS;
35

36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
namespace {
QString js_keywords[] = {
    QLatin1String("break"),
    QString::fromLatin1("case"),
    QString::fromLatin1("catch"),
    QString::fromLatin1("continue"),
    QString::fromLatin1("debugger"),
    QString::fromLatin1("default"),
    QString::fromLatin1("delete"),
    QString::fromLatin1("do"),
    QString::fromLatin1("else"),
    QString::fromLatin1("finally"),
    QString::fromLatin1("for"),
    QString::fromLatin1("function"),
    QString::fromLatin1("if"),
    QString::fromLatin1("in"),
    QString::fromLatin1("instanceof"),
    QString::fromLatin1("new"),
    QString::fromLatin1("return"),
    QString::fromLatin1("switch"),
    QString::fromLatin1("this"),
    QString::fromLatin1("throw"),
    QString::fromLatin1("try"),
    QString::fromLatin1("typeof"),
    QString::fromLatin1("var"),
    QString::fromLatin1("void"),
    QString::fromLatin1("while"),
    QString::fromLatin1("with")
};
} // end of anonymous namespace

template <typename _Tp, int N>
const _Tp *begin(const _Tp (&a)[N])
{
    return a;
}

template <typename _Tp, int N>
const _Tp *end(const _Tp (&a)[N])
{
    return a + N;
}

Roberto Raggi's avatar
Roberto Raggi committed
79
Scanner::Scanner()
80
    : _state(Normal),
81
      _scanComments(true)
82
83
84
{
}

Roberto Raggi's avatar
Roberto Raggi committed
85
Scanner::~Scanner()
86
87
{
}
88

Roberto Raggi's avatar
Roberto Raggi committed
89
bool Scanner::scanComments() const
90
91
92
93
{
    return _scanComments;
}

Roberto Raggi's avatar
Roberto Raggi committed
94
void Scanner::setScanComments(bool scanComments)
95
96
97
98
{
    _scanComments = scanComments;
}

99
static bool isIdentifierChar(QChar ch)
100
{
101
102
103
104
105
106
107
    switch (ch.unicode()) {
    case '$': case '_':
        return true;

    default:
        return ch.isLetterOrNumber();
    }
108
109
}

110
static bool isNumberChar(QChar ch)
111
{
112
113
114
115
116
117
118
119
120
121
    switch (ch.unicode()) {
    case '.':
    case 'e':
    case 'E': // ### more...
        return true;

    default:
        return ch.isLetterOrNumber();
    }
}
122

Roberto Raggi's avatar
Roberto Raggi committed
123
QList<Token> Scanner::operator()(const QString &text, int startState)
124
{
125
    _state = startState;
126
127
128
129
130
131
    QList<Token> tokens;

    // ### handle multi line comment state.

    int index = 0;

132
    if (_state == MultiLineComment) {
133
        int start = -1;
134
135
        while (index < text.length()) {
            const QChar ch = text.at(index);
136
137
138
139

            if (start == -1 && !ch.isSpace())
                start = index;

140
141
142
143
144
            QChar la;
            if (index + 1 < text.length())
                la = text.at(index + 1);

            if (ch == QLatin1Char('*') && la == QLatin1Char('/')) {
145
                _state = Normal;
146
147
148
149
150
151
                index += 2;
                break;
            } else {
                ++index;
            }
        }
152

153
154
        if (_scanComments)
            tokens.append(Token(start, index - start, Token::Comment));
155
156
    }

157
158
159
160
161
162
163
164
165
166
    while (index < text.length()) {
        const QChar ch = text.at(index);

        QChar la; // lookahead char
        if (index + 1 < text.length())
            la = text.at(index + 1);

        switch (ch.unicode()) {
        case '/':
            if (la == QLatin1Char('/')) {
167
168
                if (_scanComments)
                    tokens.append(Token(index, text.length() - index, Token::Comment));
169
170
171
172
                index = text.length();
            } else if (la == QLatin1Char('*')) {
                const int start = index;
                index += 2;
173
                _state = MultiLineComment;
174
175
176
177
178
179
180
                while (index < text.length()) {
                    const QChar ch = text.at(index);
                    QChar la;
                    if (index + 1 < text.length())
                        la = text.at(index + 1);

                    if (ch == QLatin1Char('*') && la == QLatin1Char('/')) {
181
                        _state = Normal;
182
183
                        index += 2;
                        break;
184
                    } else {
185
                        ++index;
186
187
                    }
                }
188
189
                if (_scanComments)
                    tokens.append(Token(start, index - start, Token::Comment));
190
191
            } else {
                tokens.append(Token(index++, 1, Token::Delimiter));
192
            }
193
            break;
194

195
196
197
198
199
200
201
        case '\'':
        case '"': {
            const QChar quote = ch;
            const int start = index;
            ++index;
            while (index < text.length()) {
                const QChar ch = text.at(index);
202

203
204
205
206
207
208
209
                if (ch == quote)
                    break;
                else if (index + 1 < text.length() && ch == QLatin1Char('\\'))
                    index += 2;
                else
                    ++index;
            }
210

211
212
213
214
215
216
            if (index < text.length()) {
                ++index;
                // good one
            } else {
                // unfinished
            }
217

218
219
            tokens.append(Token(start, index - start, Token::String));
        } break;
220

221
222
223
224
225
226
227
        case '.':
            if (la.isDigit()) {
                const int start = index;
                do {
                    ++index;
                } while (index < text.length() && isNumberChar(text.at(index)));
                tokens.append(Token(start, index - start, Token::Number));
228
229
                break;
            }
230
231
            tokens.append(Token(index++, 1, Token::Dot));
            break;
232

233
234
235
         case '(':
            tokens.append(Token(index++, 1, Token::LeftParenthesis));
            break;
236

237
238
         case ')':
            tokens.append(Token(index++, 1, Token::RightParenthesis));
239
240
            break;

241
242
243
         case '[':
            tokens.append(Token(index++, 1, Token::LeftBracket));
            break;
244

245
246
247
         case ']':
            tokens.append(Token(index++, 1, Token::RightBracket));
            break;
248

249
250
251
         case '{':
            tokens.append(Token(index++, 1, Token::LeftBrace));
            break;
Erik Verbruggen's avatar
Erik Verbruggen committed
252

253
254
255
         case '}':
            tokens.append(Token(index++, 1, Token::RightBrace));
            break;
256

257
258
259
         case ';':
            tokens.append(Token(index++, 1, Token::Semicolon));
            break;
260

261
262
263
264
265
266
267
268
269
         case ':':
            tokens.append(Token(index++, 1, Token::Colon));
            break;

         case ',':
            tokens.append(Token(index++, 1, Token::Comma));
            break;

        default:
Roberto Raggi's avatar
Roberto Raggi committed
270
271
272
273
274
            if (ch.isSpace()) {
                do {
                    ++index;
                } while (index < text.length() && text.at(index).isSpace());
            } else if (ch.isNumber()) {
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
                const int start = index;
                do {
                    ++index;
                } while (index < text.length() && isNumberChar(text.at(index)));
                tokens.append(Token(start, index - start, Token::Number));
            } else if (ch.isLetter() || ch == QLatin1Char('_') || ch == QLatin1Char('$')) {
                const int start = index;
                do {
                    ++index;
                } while (index < text.length() && isIdentifierChar(text.at(index)));

                if (isKeyword(text.mid(start, index - start)))
                    tokens.append(Token(start, index - start, Token::Keyword)); // ### fixme
                else
                    tokens.append(Token(start, index - start, Token::Identifier));
            } else {
                tokens.append(Token(index++, 1, Token::Delimiter));
            }
        } // end of switch
294
295
    }

296
    return tokens;
297
298
}

Roberto Raggi's avatar
Roberto Raggi committed
299
int Scanner::state() const
300
{
301
    return _state;
302
303
}

Roberto Raggi's avatar
Roberto Raggi committed
304
bool Scanner::isKeyword(const QString &text) const
305
{
306
307
308
309
    if (qBinaryFind(begin(js_keywords), end(js_keywords), text) != end(js_keywords))
        return true;

    return false;
310
}
311
312
313
314
315
316
317
318
319
320

QStringList Scanner::keywords()
{
    static QStringList words;
    if (words.isEmpty()) {
        for (const QString *word = begin(js_keywords); word != end(js_keywords); ++word)
            words.append(*word);
    }
    return words;
}