Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Tobias Hunger
qt-creator
Commits
e42ca194
Commit
e42ca194
authored
Jun 07, 2010
by
Erik Verbruggen
Browse files
Introduced token caching to prevent repetetive tokenizing.
Also removed TokenUnderCursor as it's functionality is in the token cache. Reviewed-by: ckamm
parent
2e100162
Changes
25
Hide whitespace changes
Inline
Side-by-side
src/libs/cplusplus/BackwardsScanner.cpp
View file @
e42ca194
...
...
@@ -27,33 +27,52 @@
**
**************************************************************************/
#include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h>
#include <QtGui/QTextCursor>
#include <QTextDocument>
using
namespace
CPlusPlus
;
BackwardsScanner
::
BackwardsScanner
(
const
QTextCursor
&
cursor
,
const
QString
&
suffix
,
int
maxBlockCount
)
:
_offset
(
0
)
BackwardsScanner
::
BackwardsScanner
(
TokenCache
*
tokenCache
,
const
QTextCursor
&
cursor
,
int
maxBlockCount
,
const
QString
&
suffix
)
:
_tokenCache
(
tokenCache
)
,
_offset
(
0
)
,
_blocksTokenized
(
0
)
,
_block
(
cursor
.
block
())
,
_maxBlockCount
(
maxBlockCount
)
{
_tokenize
.
setQtMocRunEnabled
(
true
);
_tokenize
.
setSkipComments
(
true
);
_tokenize
.
setObjCEnabled
(
true
);
_text
=
_block
.
text
().
left
(
cursor
.
position
()
-
cursor
.
block
().
position
());
int
pos
=
cursor
.
position
()
-
cursor
.
block
().
position
();
_text
=
_block
.
text
().
left
(
pos
);
_text
+=
suffix
;
if
(
!
suffix
.
isEmpty
())
_text
+=
suffix
;
_tokens
.
append
(
tokenCache
->
tokensForBlock
(
_block
));
_tokens
.
append
(
_tokenize
(
_text
,
previousBlockState
(
_block
)));
for
(
int
i
=
_tokens
.
size
()
-
1
;
i
>=
0
;
--
i
)
{
const
int
tokenEnd
=
_tokens
.
at
(
i
).
end
();
if
((
tokenEnd
<
pos
)
||
(
tokenEnd
==
pos
&&
suffix
.
isEmpty
()))
{
break
;
}
else
{
_tokens
.
removeAt
(
i
);
}
}
QString
remainingText
;
if
(
!
_tokens
.
isEmpty
())
remainingText
=
_text
.
mid
(
_tokens
.
last
().
end
());
if
(
!
remainingText
.
isEmpty
())
{
SimpleLexer
tokenize
;
tokenize
.
setQtMocRunEnabled
(
true
);
tokenize
.
setSkipComments
(
true
);
tokenize
.
setObjCEnabled
(
true
);
_tokens
.
append
(
tokenize
(
remainingText
,
TokenCache
::
previousBlockState
(
_block
)));
}
_startToken
=
_tokens
.
size
();
}
int
BackwardsScanner
::
state
()
const
{
return
_tokenize
.
state
();
}
SimpleToken
BackwardsScanner
::
LA
(
int
index
)
const
{
return
const_cast
<
BackwardsScanner
*>
(
this
)
->
fetchToken
(
_startToken
-
index
);
}
...
...
@@ -71,6 +90,7 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
}
else
{
++
_blocksTokenized
;
QList
<
SimpleToken
>
newTokens
=
_tokenCache
->
tokensForBlock
(
_block
);
QString
blockText
=
_block
.
text
();
_text
.
prepend
(
QLatin1Char
(
'\n'
));
_text
.
prepend
(
blockText
);
...
...
@@ -79,12 +99,11 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
for
(
int
i
=
0
;
i
<
_tokens
.
size
();
++
i
)
{
SimpleToken
t
=
_tokens
.
at
(
i
);
t
.
setPosition
(
t
.
position
()
+
blockText
.
length
()
+
1
);
t
.
setText
(
_text
.
midRef
(
t
.
position
(),
t
.
length
()));
adaptedTokens
.
append
(
t
);
}
_tokens
=
_tokenize
(
blockText
,
previousBlockState
(
_block
))
;
_offset
+=
_t
okens
.
size
();
_tokens
=
newTokens
;
_offset
+=
newT
okens
.
size
();
_tokens
+=
adaptedTokens
;
}
}
...
...
@@ -119,20 +138,6 @@ QStringRef BackwardsScanner::textRef(int index) const
return
_text
.
midRef
(
firstToken
.
begin
(),
firstToken
.
length
());
}
int
BackwardsScanner
::
previousBlockState
(
const
QTextBlock
&
block
)
{
const
QTextBlock
prevBlock
=
block
.
previous
();
if
(
prevBlock
.
isValid
())
{
int
state
=
prevBlock
.
userState
();
if
(
state
!=
-
1
)
return
state
;
}
return
0
;
}
int
BackwardsScanner
::
size
()
const
{
return
_tokens
.
size
();
...
...
src/libs/cplusplus/BackwardsScanner.h
View file @
e42ca194
...
...
@@ -36,16 +36,18 @@
namespace
CPlusPlus
{
class
TokenCache
;
class
CPLUSPLUS_EXPORT
BackwardsScanner
{
enum
{
MAX_BLOCK_COUNT
=
10
};
public:
BackwardsScanner
(
const
QTextCursor
&
cursor
,
const
QString
&
suffix
=
QString
(),
int
maxBlockCount
=
MAX_BLOCK_COUNT
);
BackwardsScanner
(
TokenCache
*
cache
,
const
QTextCursor
&
cursor
,
int
maxBlockCount
=
MAX_BLOCK_COUNT
,
const
QString
&
suffix
=
QString
());
int
state
()
const
;
int
startToken
()
const
;
int
startPosition
()
const
;
...
...
@@ -67,20 +69,18 @@ public:
int
startOfMatchingBrace
(
int
index
)
const
;
int
startOfBlock
(
int
index
)
const
;
static
int
previousBlockState
(
const
QTextBlock
&
block
);
int
size
()
const
;
private:
const
SimpleToken
&
fetchToken
(
int
tokenIndex
);
private:
TokenCache
*
_tokenCache
;
QList
<
SimpleToken
>
_tokens
;
int
_offset
;
int
_blocksTokenized
;
QTextBlock
_block
;
QString
_text
;
SimpleLexer
_tokenize
;
int
_maxBlockCount
;
int
_startToken
;
};
...
...
src/libs/cplusplus/ExpressionUnderCursor.cpp
View file @
e42ca194
...
...
@@ -30,6 +30,7 @@
#include "ExpressionUnderCursor.h"
#include "SimpleLexer.h"
#include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h>
#include <QTextCursor>
...
...
@@ -37,8 +38,8 @@
using
namespace
CPlusPlus
;
ExpressionUnderCursor
::
ExpressionUnderCursor
()
:
_jumpedComma
(
false
)
ExpressionUnderCursor
::
ExpressionUnderCursor
(
TokenCache
*
tokenCache
)
:
_tokenCache
(
tokenCache
),
_jumpedComma
(
false
)
{
}
ExpressionUnderCursor
::~
ExpressionUnderCursor
()
...
...
@@ -218,7 +219,7 @@ bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk)
QString
ExpressionUnderCursor
::
operator
()(
const
QTextCursor
&
cursor
)
{
BackwardsScanner
scanner
(
cursor
);
BackwardsScanner
scanner
(
_tokenCache
,
cursor
);
_jumpedComma
=
false
;
...
...
@@ -232,7 +233,7 @@ QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
int
ExpressionUnderCursor
::
startOfFunctionCall
(
const
QTextCursor
&
cursor
)
const
{
BackwardsScanner
scanner
(
cursor
);
BackwardsScanner
scanner
(
_tokenCache
,
cursor
);
int
index
=
scanner
.
startToken
();
...
...
src/libs/cplusplus/ExpressionUnderCursor.h
View file @
e42ca194
...
...
@@ -43,11 +43,12 @@ namespace CPlusPlus {
class
BackwardsScanner
;
class
SimpleToken
;
class
TokenCache
;
class
CPLUSPLUS_EXPORT
ExpressionUnderCursor
{
public:
ExpressionUnderCursor
();
ExpressionUnderCursor
(
TokenCache
*
tokenCache
);
~
ExpressionUnderCursor
();
QString
operator
()(
const
QTextCursor
&
cursor
);
...
...
@@ -59,6 +60,7 @@ private:
bool
isAccessToken
(
const
SimpleToken
&
tk
);
private:
TokenCache
*
_tokenCache
;
bool
_jumpedComma
;
};
...
...
src/libs/cplusplus/MatchingText.cpp
View file @
e42ca194
...
...
@@ -28,6 +28,7 @@
**************************************************************************/
#include "MatchingText.h"
#include "BackwardsScanner.h"
#include "TokenCache.h"
#include <Token.h>
...
...
@@ -75,7 +76,8 @@ static bool isCompleteCharLiteral(const BackwardsScanner &tk, int index)
return
false
;
}
MatchingText
::
MatchingText
()
MatchingText
::
MatchingText
(
TokenCache
*
tokenCache
)
:
_tokenCache
(
tokenCache
)
{
}
bool
MatchingText
::
shouldInsertMatchingText
(
const
QTextCursor
&
tc
)
...
...
@@ -151,7 +153,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri
if
(
text
.
isEmpty
()
||
!
shouldInsertMatchingText
(
la
))
return
QString
();
BackwardsScanner
tk
(
tc
,
textToProcess
.
left
(
*
skippedChars
)
,
MAX_NUM_LINES
);
BackwardsScanner
tk
(
_tokenCache
,
tc
,
MAX_NUM_LINES
,
textToProcess
.
left
(
*
skippedChars
));
const
int
startToken
=
tk
.
startToken
();
int
index
=
startToken
;
...
...
@@ -211,7 +213,7 @@ bool MatchingText::shouldInsertNewline(const QTextCursor &tc) const
QString
MatchingText
::
insertParagraphSeparator
(
const
QTextCursor
&
tc
)
const
{
BackwardsScanner
tk
(
tc
,
QString
()
,
MAX_NUM_LINES
);
BackwardsScanner
tk
(
_tokenCache
,
tc
,
MAX_NUM_LINES
);
int
index
=
tk
.
startToken
();
if
(
tk
[
index
-
1
].
isNot
(
T_LBRACE
))
...
...
src/libs/cplusplus/MatchingText.h
View file @
e42ca194
...
...
@@ -35,11 +35,12 @@
namespace
CPlusPlus
{
class
BackwardsScanner
;
class
TokenCache
;
class
CPLUSPLUS_EXPORT
MatchingText
{
public:
MatchingText
();
MatchingText
(
TokenCache
*
tokenCache
);
static
bool
shouldInsertMatchingText
(
const
QTextCursor
&
tc
);
static
bool
shouldInsertMatchingText
(
const
QChar
&
lookAhead
);
...
...
@@ -50,6 +51,8 @@ public:
private:
bool
shouldInsertNewline
(
const
QTextCursor
&
tc
)
const
;
TokenCache
*
_tokenCache
;
};
}
// end of namespace CPlusPlus
...
...
src/libs/cplusplus/SimpleLexer.cpp
View file @
e42ca194
...
...
@@ -37,12 +37,11 @@
using
namespace
CPlusPlus
;
SimpleToken
::
SimpleToken
(
const
Token
&
token
,
const
QStringRef
&
text
)
SimpleToken
::
SimpleToken
(
const
Token
&
token
)
:
_kind
(
token
.
f
.
kind
)
,
_flags
(
0
)
,
_position
(
token
.
begin
())
,
_length
(
token
.
f
.
length
)
,
_text
(
text
)
{
f
.
_whitespace
=
token
.
f
.
whitespace
;
f
.
_newline
=
token
.
f
.
newline
;
...
...
@@ -148,17 +147,17 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
break
;
QStringRef
spell
=
text
.
midRef
(
lex
.
tokenOffset
(),
lex
.
tokenLength
());
SimpleToken
simpleTk
(
tk
,
spell
);
SimpleToken
simpleTk
(
tk
);
lex
.
setScanAngleStringLiteralTokens
(
false
);
if
(
tk
.
f
.
newline
&&
tk
.
is
(
T_POUND
))
inPreproc
=
true
;
else
if
(
inPreproc
&&
tokens
.
size
()
==
1
&&
simpleTk
.
is
(
T_IDENTIFIER
)
&&
s
impleTk
.
text
()
==
QLatin1String
(
"include"
))
s
pell
==
QLatin1String
(
"include"
))
lex
.
setScanAngleStringLiteralTokens
(
true
);
else
if
(
_objCEnabled
&&
inPreproc
&&
tokens
.
size
()
==
1
&&
simpleTk
.
is
(
T_IDENTIFIER
)
&&
s
impleTk
.
text
()
==
QLatin1String
(
"import"
))
s
pell
==
QLatin1String
(
"import"
))
lex
.
setScanAngleStringLiteralTokens
(
true
);
if
(
_objCEnabled
&&
tk
.
is
(
T_IDENTIFIER
))
...
...
src/libs/cplusplus/SimpleLexer.h
View file @
e42ca194
...
...
@@ -42,7 +42,7 @@ class Token;
class
CPLUSPLUS_EXPORT
SimpleToken
{
public:
SimpleToken
(
const
Token
&
token
,
const
QStringRef
&
text
);
SimpleToken
(
const
Token
&
token
);
SimpleToken
()
:
_kind
(
0
)
...
...
@@ -66,9 +66,6 @@ public:
inline
int
end
()
const
{
return
_position
+
_length
;
}
inline
QStringRef
text
()
const
{
return
_text
;
}
inline
bool
followsNewline
()
const
{
return
f
.
_newline
;
}
...
...
@@ -91,10 +88,6 @@ public:
inline
void
setPosition
(
int
position
)
{
_position
=
position
;
}
// internal
inline
void
setText
(
const
QStringRef
&
text
)
{
_text
=
text
;
}
public:
short
_kind
;
union
{
...
...
@@ -109,7 +102,6 @@ public:
int
_position
;
int
_length
;
QStringRef
_text
;
friend
class
SimpleLexer
;
};
...
...
src/libs/cplusplus/TokenCache.cpp
0 → 100644
View file @
e42ca194
#include "SimpleLexer.h"
#include "TokenCache.h"
using
namespace
CPlusPlus
;
TokenCache
::
TokenCache
()
:
m_doc
(
0
)
,
m_revision
(
-
1
)
{}
void
TokenCache
::
setDocument
(
QTextDocument
*
doc
)
{
m_doc
=
doc
;
m_revision
=
-
1
;
}
QList
<
SimpleToken
>
TokenCache
::
tokensForBlock
(
const
QTextBlock
&
block
)
const
{
Q_ASSERT
(
m_doc
);
Q_ASSERT
(
m_doc
==
block
.
document
());
const
int
documentRevision
=
m_doc
->
revision
();
if
(
documentRevision
!=
m_revision
)
{
m_tokensByBlock
.
clear
();
m_revision
=
documentRevision
;
}
const
int
blockNr
=
block
.
blockNumber
();
if
(
m_tokensByBlock
.
contains
(
blockNr
))
{
return
m_tokensByBlock
.
value
(
blockNr
);
}
else
{
SimpleLexer
tokenize
;
tokenize
.
setObjCEnabled
(
true
);
tokenize
.
setQtMocRunEnabled
(
true
);
tokenize
.
setSkipComments
(
false
);
const
int
prevState
=
previousBlockState
(
block
);
QList
<
SimpleToken
>
tokens
=
tokenize
(
block
.
text
(),
prevState
);
m_tokensByBlock
.
insert
(
blockNr
,
tokens
);
return
tokens
;
}
}
SimpleToken
TokenCache
::
tokenUnderCursor
(
const
QTextCursor
&
cursor
)
const
{
const
QTextBlock
block
=
cursor
.
block
();
const
QList
<
SimpleToken
>
tokens
=
tokensForBlock
(
block
);
const
int
column
=
cursor
.
position
()
-
block
.
position
();
for
(
int
index
=
tokens
.
size
()
-
1
;
index
>=
0
;
--
index
)
{
const
SimpleToken
&
tk
=
tokens
.
at
(
index
);
if
(
tk
.
position
()
<
column
)
return
tk
;
}
return
SimpleToken
();
}
QString
TokenCache
::
text
(
const
QTextBlock
&
block
,
int
tokenIndex
)
const
{
SimpleToken
tk
=
tokensForBlock
(
block
).
at
(
tokenIndex
);
return
block
.
text
().
mid
(
tk
.
position
(),
tk
.
length
());
}
int
TokenCache
::
previousBlockState
(
const
QTextBlock
&
block
)
{
const
QTextBlock
prevBlock
=
block
.
previous
();
if
(
prevBlock
.
isValid
())
{
int
state
=
prevBlock
.
userState
();
if
(
state
!=
-
1
)
return
state
;
}
return
0
;
}
src/libs/cplusplus/TokenCache.h
0 → 100644
View file @
e42ca194
#ifndef TOKENCACHE_H
#define TOKENCACHE_H
#include <CPlusPlusForwardDeclarations.h>
#include <cplusplus/SimpleLexer.h>
#include <QtCore/QHash>
#include <QtCore/QList>
#include <QtGui/QTextBlock>
#include <QtGui/QTextCursor>
#include <QtGui/QTextDocument>
namespace
CPlusPlus
{
class
CPLUSPLUS_EXPORT
TokenCache
{
public:
TokenCache
();
void
setDocument
(
QTextDocument
*
doc
);
QList
<
CPlusPlus
::
SimpleToken
>
tokensForBlock
(
const
QTextBlock
&
block
)
const
;
CPlusPlus
::
SimpleToken
tokenUnderCursor
(
const
QTextCursor
&
cursor
)
const
;
QString
text
(
const
QTextBlock
&
block
,
int
tokenIndex
)
const
;
static
int
previousBlockState
(
const
QTextBlock
&
block
);
private:
QTextDocument
*
m_doc
;
mutable
int
m_revision
;
mutable
QHash
<
int
,
QList
<
CPlusPlus
::
SimpleToken
>
>
m_tokensByBlock
;
};
}
// namespace CPlusPlus
#endif // TOKENCACHE_H
src/libs/cplusplus/TokenUnderCursor.cpp
deleted
100644 → 0
View file @
2e100162
/**************************************************************************
**
** This file is part of Qt Creator
**
** Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
**
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** Commercial Usage
**
** Licensees holding valid Qt Commercial licenses may use this file in
** accordance with the Qt Commercial License Agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Nokia.
**
** GNU Lesser General Public License Usage
**
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** If you are unsure which license is appropriate for your use, please
** contact the sales department at http://qt.nokia.com/contact.
**
**************************************************************************/
#include "TokenUnderCursor.h"
#include "BackwardsScanner.h"
#include <Token.h>
#include <QTextCursor>
#include <QTextBlock>
#include <climits>
using
namespace
CPlusPlus
;
TokenUnderCursor
::
TokenUnderCursor
()
{
}
TokenUnderCursor
::~
TokenUnderCursor
()
{
}
SimpleToken
TokenUnderCursor
::
operator
()(
const
QTextCursor
&
cursor
,
QTextBlock
*
b
)
{
SimpleLexer
tokenize
;
tokenize
.
setObjCEnabled
(
true
);
tokenize
.
setSkipComments
(
false
);
QTextBlock
block
=
cursor
.
block
();
int
column
=
cursor
.
position
()
-
cursor
.
block
().
position
();
_text
=
block
.
text
();
_tokens
=
tokenize
(
_text
,
BackwardsScanner
::
previousBlockState
(
block
));
for
(
int
index
=
_tokens
.
size
()
-
1
;
index
!=
-
1
;
--
index
)
{
const
SimpleToken
&
tk
=
_tokens
.
at
(
index
);
if
(
tk
.
position
()
<
column
)
{
if
(
b
)
*
b
=
block
;
return
tk
;
}
}
return
SimpleToken
();
}
src/libs/cplusplus/TokenUnderCursor.h
deleted
100644 → 0
View file @
2e100162
/**************************************************************************
**
** This file is part of Qt Creator
**
** Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
**
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** Commercial Usage
**
** Licensees holding valid Qt Commercial licenses may use this file in
** accordance with the Qt Commercial License Agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Nokia.
**
** GNU Lesser General Public License Usage
**
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** If you are unsure which license is appropriate for your use, please
** contact the sales department at http://qt.nokia.com/contact.
**
**************************************************************************/
#ifndef CPLUSPLUS_TOKENUNDERCURSOR_H
#define CPLUSPLUS_TOKENUNDERCURSOR_H
#include "SimpleLexer.h"
#include <QList>
QT_BEGIN_NAMESPACE
class
QString
;
class
QTextCursor
;
class
QTextBlock
;
QT_END_NAMESPACE
namespace
CPlusPlus
{
class
CPLUSPLUS_EXPORT
TokenUnderCursor
{
public:
TokenUnderCursor
();
~
TokenUnderCursor
();
SimpleToken
operator
()(
const
QTextCursor
&
cursor
,
QTextBlock
*
block
=
0
);
const
QList
<
SimpleToken
>
&
tokens
()
const
{
return
_tokens
;
}
private:
QList
<
SimpleToken
>
_tokens
;
QString
_text
;
};
}
// end of namespace CPlusPlus
#endif // CPLUSPLUS_TOKENUNDERCURSOR_H
src/libs/cplusplus/cplusplus-lib.pri
View file @
e42ca194
...
...
@@ -12,18 +12,18 @@ contains(QT, gui) {
HEADERS += \
$$PWD/Icons.h \
$$PWD/ExpressionUnderCursor.h \
$$PWD/TokenUnderCursor.h \
$$PWD/BackwardsScanner.h \
$$PWD/MatchingText.h \
$$PWD/OverviewModel.h
$$PWD/OverviewModel.h \
$$PWD/TokenCache.h
SOURCES += \
$$PWD/Icons.cpp \
$$PWD/ExpressionUnderCursor.cpp \
$$PWD/TokenUnderCursor.cpp \
$$PWD/BackwardsScanner.cpp \