Commit f027b1da authored by Christian Kamm's avatar Christian Kamm
Browse files

QmlJS: Make the Scanner recognize regular expression literals.

Task-number: QTCREATORBUG-4566
Change-Id: I48b08f8eee79b25d0ebe186b996cdcb8f1979e3d
Reviewed-on: http://codereview.qt.nokia.com/38

Reviewed-by: default avatarRoberto Raggi <roberto.raggi@nokia.com>
parent 0350314b
...@@ -123,6 +123,67 @@ static bool isNumberChar(QChar ch) ...@@ -123,6 +123,67 @@ static bool isNumberChar(QChar ch)
} }
} }
static int findRegExpEnd(const QString &text, int start)
{
if (start >= text.size() || text.at(start) != QLatin1Char('/'))
return start;
// find the second /
int index = start + 1;
for (; index < text.length(); ++index) {
const QChar ch = text.at(index);
if (ch == QLatin1Char('\\')) {
++index;
} else if (ch == QLatin1Char('[')) {
// find closing ]
for (; index < text.length(); ++index) {
const QChar ch2 = text.at(index);
if (ch2 == QLatin1Char('\\')) {
++index;
} else if (ch2 == QLatin1Char(']'))
break;
}
if (index >= text.size())
return text.size();
} else if (ch == QLatin1Char('/'))
break;
}
if (index >= text.size())
return text.size();
++index;
// find end of reg exp flags
for (; index < text.size(); ++index) {
const QChar ch = text.at(index);
if (!isIdentifierChar(ch))
break;
}
return index;
}
static inline int multiLineState(int state)
{
return state & 0b11;
}
static inline void setMultiLineState(int *state, int s)
{
*state = s | (*state & ~0b11);
}
static inline bool regexpMayFollow(int state)
{
return state & 0b100;
}
static inline void setRegexpMayFollow(int *state, bool on)
{
*state = (on << 2) | (*state & 0b11);
}
QList<Token> Scanner::operator()(const QString &text, int startState) QList<Token> Scanner::operator()(const QString &text, int startState)
{ {
_state = startState; _state = startState;
...@@ -132,7 +193,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState) ...@@ -132,7 +193,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState)
int index = 0; int index = 0;
if (_state == MultiLineComment) { if (multiLineState(_state) == MultiLineComment) {
int start = -1; int start = -1;
while (index < text.length()) { while (index < text.length()) {
const QChar ch = text.at(index); const QChar ch = text.at(index);
...@@ -145,7 +206,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState) ...@@ -145,7 +206,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState)
la = text.at(index + 1); la = text.at(index + 1);
if (ch == QLatin1Char('*') && la == QLatin1Char('/')) { if (ch == QLatin1Char('*') && la == QLatin1Char('/')) {
_state = Normal; setMultiLineState(&_state, Normal);
index += 2; index += 2;
break; break;
} else { } else {
...@@ -155,7 +216,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState) ...@@ -155,7 +216,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState)
if (_scanComments && start != -1) if (_scanComments && start != -1)
tokens.append(Token(start, index - start, Token::Comment)); tokens.append(Token(start, index - start, Token::Comment));
} else if (_state == MultiLineStringDQuote || _state == MultiLineStringSQuote) { } else if (multiLineState(_state) == MultiLineStringDQuote || multiLineState(_state) == MultiLineStringSQuote) {
const QChar quote = (_state == MultiLineStringDQuote ? QLatin1Char('"') : QLatin1Char('\'')); const QChar quote = (_state == MultiLineStringDQuote ? QLatin1Char('"') : QLatin1Char('\''));
const int start = index; const int start = index;
while (index < text.length()) { while (index < text.length()) {
...@@ -170,10 +231,11 @@ QList<Token> Scanner::operator()(const QString &text, int startState) ...@@ -170,10 +231,11 @@ QList<Token> Scanner::operator()(const QString &text, int startState)
} }
if (index < text.length()) { if (index < text.length()) {
++index; ++index;
_state = Normal; setMultiLineState(&_state, Normal);
} }
if (start < index) if (start < index)
tokens.append(Token(start, index - start, Token::String)); tokens.append(Token(start, index - start, Token::String));
setRegexpMayFollow(&_state, false);
} }
while (index < text.length()) { while (index < text.length()) {
...@@ -192,7 +254,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState) ...@@ -192,7 +254,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState)
} else if (la == QLatin1Char('*')) { } else if (la == QLatin1Char('*')) {
const int start = index; const int start = index;
index += 2; index += 2;
_state = MultiLineComment; setMultiLineState(&_state, MultiLineComment);
while (index < text.length()) { while (index < text.length()) {
const QChar ch = text.at(index); const QChar ch = text.at(index);
QChar la; QChar la;
...@@ -200,7 +262,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState) ...@@ -200,7 +262,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState)
la = text.at(index + 1); la = text.at(index + 1);
if (ch == QLatin1Char('*') && la == QLatin1Char('/')) { if (ch == QLatin1Char('*') && la == QLatin1Char('/')) {
_state = Normal; setMultiLineState(&_state, Normal);
index += 2; index += 2;
break; break;
} else { } else {
...@@ -209,8 +271,14 @@ QList<Token> Scanner::operator()(const QString &text, int startState) ...@@ -209,8 +271,14 @@ QList<Token> Scanner::operator()(const QString &text, int startState)
} }
if (_scanComments) if (_scanComments)
tokens.append(Token(start, index - start, Token::Comment)); tokens.append(Token(start, index - start, Token::Comment));
} else if (regexpMayFollow(_state)) {
const int end = findRegExpEnd(text, index);
tokens.append(Token(index, end - index, Token::RegExp));
index = end;
setRegexpMayFollow(&_state, false);
} else { } else {
tokens.append(Token(index++, 1, Token::Delimiter)); tokens.append(Token(index++, 1, Token::Delimiter));
setRegexpMayFollow(&_state, true);
} }
break; break;
...@@ -235,12 +303,13 @@ QList<Token> Scanner::operator()(const QString &text, int startState) ...@@ -235,12 +303,13 @@ QList<Token> Scanner::operator()(const QString &text, int startState)
// good one // good one
} else { } else {
if (quote.unicode() == '"') if (quote.unicode() == '"')
_state = MultiLineStringDQuote; setMultiLineState(&_state, MultiLineStringDQuote);
else else
_state = MultiLineStringSQuote; setMultiLineState(&_state, MultiLineStringSQuote);
} }
tokens.append(Token(start, index - start, Token::String)); tokens.append(Token(start, index - start, Token::String));
setRegexpMayFollow(&_state, false);
} break; } break;
case '.': case '.':
...@@ -253,42 +322,52 @@ QList<Token> Scanner::operator()(const QString &text, int startState) ...@@ -253,42 +322,52 @@ QList<Token> Scanner::operator()(const QString &text, int startState)
break; break;
} }
tokens.append(Token(index++, 1, Token::Dot)); tokens.append(Token(index++, 1, Token::Dot));
setRegexpMayFollow(&_state, false);
break; break;
case '(': case '(':
tokens.append(Token(index++, 1, Token::LeftParenthesis)); tokens.append(Token(index++, 1, Token::LeftParenthesis));
setRegexpMayFollow(&_state, true);
break; break;
case ')': case ')':
tokens.append(Token(index++, 1, Token::RightParenthesis)); tokens.append(Token(index++, 1, Token::RightParenthesis));
setRegexpMayFollow(&_state, false);
break; break;
case '[': case '[':
tokens.append(Token(index++, 1, Token::LeftBracket)); tokens.append(Token(index++, 1, Token::LeftBracket));
setRegexpMayFollow(&_state, true);
break; break;
case ']': case ']':
tokens.append(Token(index++, 1, Token::RightBracket)); tokens.append(Token(index++, 1, Token::RightBracket));
setRegexpMayFollow(&_state, false);
break; break;
case '{': case '{':
tokens.append(Token(index++, 1, Token::LeftBrace)); tokens.append(Token(index++, 1, Token::LeftBrace));
setRegexpMayFollow(&_state, true);
break; break;
case '}': case '}':
tokens.append(Token(index++, 1, Token::RightBrace)); tokens.append(Token(index++, 1, Token::RightBrace));
setRegexpMayFollow(&_state, false);
break; break;
case ';': case ';':
tokens.append(Token(index++, 1, Token::Semicolon)); tokens.append(Token(index++, 1, Token::Semicolon));
setRegexpMayFollow(&_state, true);
break; break;
case ':': case ':':
tokens.append(Token(index++, 1, Token::Colon)); tokens.append(Token(index++, 1, Token::Colon));
setRegexpMayFollow(&_state, true);
break; break;
case ',': case ',':
tokens.append(Token(index++, 1, Token::Comma)); tokens.append(Token(index++, 1, Token::Comma));
setRegexpMayFollow(&_state, true);
break; break;
case '+': case '+':
...@@ -299,6 +378,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState) ...@@ -299,6 +378,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState)
} else { } else {
tokens.append(Token(index++, 1, Token::Delimiter)); tokens.append(Token(index++, 1, Token::Delimiter));
} }
setRegexpMayFollow(&_state, true);
break; break;
default: default:
...@@ -312,6 +392,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState) ...@@ -312,6 +392,7 @@ QList<Token> Scanner::operator()(const QString &text, int startState)
++index; ++index;
} while (index < text.length() && isNumberChar(text.at(index))); } while (index < text.length() && isNumberChar(text.at(index)));
tokens.append(Token(start, index - start, Token::Number)); tokens.append(Token(start, index - start, Token::Number));
setRegexpMayFollow(&_state, false);
} else if (ch.isLetter() || ch == QLatin1Char('_') || ch == QLatin1Char('$')) { } else if (ch.isLetter() || ch == QLatin1Char('_') || ch == QLatin1Char('$')) {
const int start = index; const int start = index;
do { do {
...@@ -322,8 +403,10 @@ QList<Token> Scanner::operator()(const QString &text, int startState) ...@@ -322,8 +403,10 @@ QList<Token> Scanner::operator()(const QString &text, int startState)
tokens.append(Token(start, index - start, Token::Keyword)); // ### fixme tokens.append(Token(start, index - start, Token::Keyword)); // ### fixme
else else
tokens.append(Token(start, index - start, Token::Identifier)); tokens.append(Token(start, index - start, Token::Identifier));
setRegexpMayFollow(&_state, false);
} else { } else {
tokens.append(Token(index++, 1, Token::Delimiter)); tokens.append(Token(index++, 1, Token::Delimiter));
setRegexpMayFollow(&_state, true);
} }
} // end of switch } // end of switch
} }
......
...@@ -59,7 +59,8 @@ public: ...@@ -59,7 +59,8 @@ public:
Colon, Colon,
Comma, Comma,
Dot, Dot,
Delimiter Delimiter,
RegExp
}; };
inline Token(): offset(0), length(0), kind(EndOfFile) {} inline Token(): offset(0), length(0), kind(EndOfFile) {}
...@@ -82,7 +83,8 @@ public: ...@@ -82,7 +83,8 @@ public:
Normal = 0, Normal = 0,
MultiLineComment = 1, MultiLineComment = 1,
MultiLineStringDQuote = 2, MultiLineStringDQuote = 2,
MultiLineStringSQuote = 3 MultiLineStringSQuote = 3,
RegexpMayFollow = 4 // flag that may be combined with the above
}; };
Scanner(); Scanner();
......
...@@ -681,7 +681,7 @@ bool QmlJSCompletionAssistProcessor::acceptsIdleEditor() const ...@@ -681,7 +681,7 @@ bool QmlJSCompletionAssistProcessor::acceptsIdleEditor() const
if (column >= tk.begin() && column <= tk.end()) { if (column >= tk.begin() && column <= tk.end()) {
if (charBeforeCursor == QLatin1Char('/') && tk.is(Token::String)) if (charBeforeCursor == QLatin1Char('/') && tk.is(Token::String))
return true; // path completion inside string literals return true; // path completion inside string literals
if (tk.is(Token::Comment) || tk.is(Token::String)) if (tk.is(Token::Comment) || tk.is(Token::String) || tk.is(Token::RegExp))
return false; return false;
break; break;
} }
......
...@@ -116,6 +116,10 @@ void Highlighter::highlightBlock(const QString &text) ...@@ -116,6 +116,10 @@ void Highlighter::highlightBlock(const QString &text)
setFormat(token.offset, token.length, m_formats[CommentFormat]); setFormat(token.offset, token.length, m_formats[CommentFormat]);
break; break;
case Token::RegExp:
setFormat(token.offset, token.length, m_formats[StringFormat]);
break;
case Token::LeftParenthesis: case Token::LeftParenthesis:
onOpeningParenthesis('(', token.offset, index == 0); onOpeningParenthesis('(', token.offset, index == 0);
break; break;
...@@ -229,7 +233,8 @@ void Highlighter::highlightBlock(const QString &text) ...@@ -229,7 +233,8 @@ void Highlighter::highlightBlock(const QString &text)
switch (token.kind) { switch (token.kind) {
case Token::Comment: case Token::Comment:
case Token::String: { case Token::String:
case Token::RegExp: {
int i = token.begin(), e = token.end(); int i = token.begin(), e = token.end();
while (i < e) { while (i < e) {
const QChar ch = text.at(i); const QChar ch = text.at(i);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment