Editor: Highlight background for whitespaces in strings and comments

The whitespaces are now highlighted with the same backgroundcolor as the characters.
Also added a test for the Lexer.

Task-number: QTCREATORBUG-5802

Change-Id: Ic1bcd8cfe30d6b8a0281b7963eaab310f972b9d2
Reviewed-by: Roberto Raggi <roberto.raggi@nokia.com>

src/libs/3rdparty/cplusplus/Lexer.cpp

@@ -157,10 +157,15 @@ void Lexer::scan(Token *tok)
 void Lexer::scan_helper(Token *tok)
 {
   _Lagain:
+    _tokenStart = _currentChar;
+    tok->offset = _currentChar - _firstChar;
+
     while (_yychar && std::isspace(_yychar)) {
         if (_yychar == '\n') {
             tok->f.joined = false;
             tok->f.newline = true;
+            if (_state == State_MultiLineComment || _state == State_MultiLineDoxyComment)
+                break;
         } else {
             tok->f.whitespace = true;
         }
@@ -170,12 +175,10 @@ void Lexer::scan_helper(Token *tok)
     if (! _translationUnit)
         tok->lineno = _currentLine;
 
-    _tokenStart = _currentChar;
-    tok->offset = _currentChar - _firstChar;
-
     if (_state == State_MultiLineComment || _state == State_MultiLineDoxyComment) {
         const int originalState = _state;
 
+
         if (! _yychar) {
             tok->f.kind = T_EOF_SYMBOL;
             return;
@@ -204,6 +207,9 @@ void Lexer::scan_helper(Token *tok)
         return; // done
     }
 
+    _tokenStart = _currentChar;
+    tok->offset = _currentChar - _firstChar;
+
     if (! _yychar) {
         tok->f.kind = T_EOF_SYMBOL;
         return;

src/plugins/cppeditor/cpphighlighter.cpp

@@ -162,15 +162,15 @@ void CppHighlighter::highlightBlock(const QString &text)
 
         else if (tk.is(T_STRING_LITERAL) || tk.is(T_CHAR_LITERAL) || tk.is(T_ANGLE_STRING_LITERAL) ||
                  tk.is(T_AT_STRING_LITERAL))
-            highlightLine(text, tk.begin(), tk.length(), m_formats[CppStringFormat]);
+            setFormat(tk.begin(), tk.length(), m_formats[CppStringFormat]);
 
         else if (tk.is(T_WIDE_STRING_LITERAL) || tk.is(T_WIDE_CHAR_LITERAL))
-            highlightLine(text, tk.begin(), tk.length(), m_formats[CppStringFormat]);
+            setFormat(tk.begin(), tk.length(), m_formats[CppStringFormat]);
 
         else if (tk.isComment()) {
 
             if (tk.is(T_COMMENT) || tk.is(T_CPP_COMMENT))
-                highlightLine(text, tk.begin(), tk.length(), m_formats[CppCommentFormat]);
+                setFormat(tk.begin(), tk.length(), m_formats[CppCommentFormat]);
 
             else // a doxygen comment
                 highlightDoxygenComment(text, tk.begin(), tk.length());
@@ -212,7 +212,7 @@ void CppHighlighter::highlightBlock(const QString &text)
         const Token tk = tokens.last();
         const int lastTokenEnd = tk.begin() + tk.length();
         if (text.length() > lastTokenEnd)
-            highlightLine(text, lastTokenEnd, text.length() - lastTokenEnd, QTextCharFormat());
+            setFormat(lastTokenEnd, text.length() - lastTokenEnd, QTextCharFormat());
     }
 
     if (! initialState && state && ! tokens.isEmpty()) {
@@ -439,7 +439,7 @@ void CppHighlighter::highlightDoxygenComment(const QString &text, int position,
 
             int k = CppTools::classifyDoxygenTag(start, it - start);
             if (k != CppTools::T_DOXY_IDENTIFIER) {
-                highlightLine(text, initial, start - uc - initial, format);
+                setFormat(initial, start - uc - initial, format);
                 setFormat(start - uc - 1, it - start + 1, kwFormat);
                 initial = it - uc;
             }
@@ -447,6 +447,6 @@ void CppHighlighter::highlightDoxygenComment(const QString &text, int position,
             ++it;
     }
 
-    highlightLine(text, initial, it - uc - initial, format);
+    setFormat(initial, it - uc - initial, format);
 }
 

tests/auto/cplusplus/misc/tst_misc.cpp

@@ -34,6 +34,8 @@
 
 #include <QtTest>
 #include <QtDebug>
+#include <SimpleLexer.h>
+#include <Token.h>
 
 #include <findcdbbreakpoint.h>
 
@@ -50,6 +52,14 @@ private slots:
 
     void findBreakpoints();
     void findBreakpoints2();
+
+    void testLexerComment();
+    void testLexerComment2();
+    void testLexerComment3();
+    void testLexerMultiLineComment();
+    void testLexerMultiLineComment2();
+    void testLexerMultiLineComment3();
+    void testLexerMultiLineComment4();
 };
 
 void tst_Misc::diagnosticClient_error()
@@ -177,5 +187,107 @@ void tst_Misc::findBreakpoints2()
     QCOMPARE(findBreakpoint(7), 7U);
 }
 
+void tst_Misc::testLexerComment() {
+    const QByteArray src("// int a = 42    ");
+
+    SimpleLexer tokenize;
+    tokenize.setQtMocRunEnabled(false);
+    tokenize.setObjCEnabled(false);
+    tokenize.setCxx0xEnabled(true);
+    const QList<Token> tokens = tokenize(src);
+
+    QCOMPARE(tokenize.state(), 0);
+    QCOMPARE(tokens.size(), 1);
+    QCOMPARE(tokens[0].f.kind, 2U);
+    QCOMPARE(tokens[0].f.length, 17U);
+}
+
+void tst_Misc::testLexerComment2() {
+    const QByteArray src("              // int a = 42    ");
+
+    SimpleLexer tokenize;
+    tokenize.setQtMocRunEnabled(false);
+    tokenize.setObjCEnabled(false);
+    tokenize.setCxx0xEnabled(true);
+    const QList<Token> tokens = tokenize(src);
+
+    QCOMPARE(tokenize.state(), 0);
+    QCOMPARE(tokens.size(), 1);
+    QCOMPARE(tokens[0].f.kind, 2U);
+    QCOMPARE(tokens[0].f.length, 17U);
+}
+
+void tst_Misc::testLexerComment3() {
+    const QByteArray src("  int main( int argc, char** argv) { // Foo m_foo = 42  /n");
+
+    SimpleLexer tokenize;
+    tokenize.setQtMocRunEnabled(false);
+    tokenize.setObjCEnabled(false);
+    tokenize.setCxx0xEnabled(true);
+    const QList<Token> tokens = tokenize(src);
+
+    QCOMPARE(tokenize.state(), 0);
+    QCOMPARE(tokens.size(), 13);
+    QCOMPARE(tokens.last().f.kind, 2U);
+    QCOMPARE(tokens.last().f.length, 21U);
+}
+
+void tst_Misc::testLexerMultiLineComment() {
+    const QByteArray src("/* multi /n"
+                         " * line  /n");
+    SimpleLexer tokenize;
+    tokenize.setQtMocRunEnabled(false);
+    tokenize.setObjCEnabled(false);
+    tokenize.setCxx0xEnabled(true);
+    const QList<Token> tokens = tokenize(src);
+
+    QCOMPARE(tokenize.state(), 1);
+    QCOMPARE(tokens.size(), 1);
+    QCOMPARE(tokens[0].f.kind, 4U);
+    QCOMPARE(tokens.last().f.length, 22U);
+}
+
+void tst_Misc::testLexerMultiLineComment2() {
+    const QByteArray src("");
+    SimpleLexer tokenize;
+    tokenize.setQtMocRunEnabled(false);
+    tokenize.setObjCEnabled(false);
+    tokenize.setCxx0xEnabled(true);
+    QList<Token> tokens = tokenize(src,1);
+
+    QCOMPARE(tokenize.state(), 1);
+    QCOMPARE(tokens.size(), 1);
+    QCOMPARE(tokens[0].f.kind, 4U);
+    QCOMPARE(tokens[0].f.length, 1U);
+}
+
+void tst_Misc::testLexerMultiLineComment3() {
+    const QByteArray src("    ");
+    SimpleLexer tokenize;
+    tokenize.setQtMocRunEnabled(false);
+    tokenize.setObjCEnabled(false);
+    tokenize.setCxx0xEnabled(true);
+    const QList<Token> tokens = tokenize(src,1);
+
+    QCOMPARE(tokenize.state(), 1);
+    QCOMPARE(tokens.size(), 1);
+    QCOMPARE(tokens[0].f.kind, 4U);
+    QCOMPARE(tokens[0].f.length, 5U);
+}
+
+void tst_Misc::testLexerMultiLineComment4() {
+    const QByteArray src("int /* integer */ i");
+    SimpleLexer tokenize;
+    tokenize.setQtMocRunEnabled(false);
+    tokenize.setObjCEnabled(false);
+    tokenize.setCxx0xEnabled(true);
+    const QList<Token> tokens = tokenize(src,0);
+
+    QCOMPARE(tokenize.state(), 0);
+    QCOMPARE(tokens.size(), 3);
+    QCOMPARE(tokens[1].f.kind, 4U);
+    QCOMPARE(tokens[1].f.length, 13U);
+}
+
 QTEST_MAIN(tst_Misc)
 #include "tst_misc.moc"