Skip to content

GitLab

Commit a8d53e0a authored by Oswald Buddenhagen's avatar Oswald Buddenhagen
Browse files

serialize AST into a single string 

this saves quite some mallocs in the parsing pass.
parent dad37b23
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 666 additions and 526 deletions
src/shared/proparser/profileevaluator.cpp
View file @ a8d53e0a
This diff is collapsed.
src/shared/proparser/profileevaluator.h
View file @ a8d53e0a
......@@ -50,9 +50,14 @@ class ProFileEvaluator
class Private;
public:
struct FunctionDef {
QString string;
int offset;
};
struct FunctionDefs {
QHash<ProString, ProFunctionDef *> testFunctions;
QHash<ProString, ProFunctionDef *> replaceFunctions;
QHash<ProString, FunctionDef> testFunctions;
QHash<ProString, FunctionDef> replaceFunctions;
};
enum TemplateType {
......@@ -109,6 +114,8 @@ private:
friend class ProFileCache;
};
Q_DECLARE_TYPEINFO(ProFileEvaluator::FunctionDef, Q_MOVABLE_TYPE);
class ProFileCache
{
public:
......
src/shared/proparser/proitems.cpp
View file @ a8d53e0a
......@@ -37,7 +37,7 @@ QT_BEGIN_NAMESPACE
using namespace ProStringConstants;
// from qhash.cpp
static uint hash(const QChar *p, int n)
uint ProString::hash(const QChar *p, int n)
{
uint h = 0;
......@@ -86,6 +86,22 @@ ProString::ProString(const char *str, OmitPreHashing) :
{
}
ProString::ProString(const QString &str, int offset, int length) :
m_string(str), m_offset(offset), m_length(length)
{
updatedHash();
}
ProString::ProString(const QString &str, int offset, int length, uint hash) :
m_string(str), m_offset(offset), m_length(length), m_hash(hash)
{
}
ProString::ProString(const QString &str, int offset, int length, ProStringConstants::OmitPreHashing) :
m_string(str), m_offset(offset), m_length(length), m_hash(0x80000000)
{
}
uint ProString::updatedHash() const
{
return (m_hash = hash(m_string.constData() + m_offset, m_length));
......@@ -231,44 +247,8 @@ void ProStringList::removeDuplicates()
erase(begin() + j, end());
}
void ProItem::disposeItems(ProItem *nitm)
{
for (ProItem *itm; (itm = nitm); ) {
nitm = itm->next();
switch (itm->kind()) {
case ProItem::ConditionKind: delete static_cast<ProCondition *>(itm); break;
case ProItem::VariableKind: delete static_cast<ProVariable *>(itm); break;
case ProItem::BranchKind: delete static_cast<ProBranch *>(itm); break;
case ProItem::LoopKind: delete static_cast<ProLoop *>(itm); break;
case ProItem::FunctionDefKind: static_cast<ProFunctionDef *>(itm)->deref(); break;
case ProItem::OpNotKind:
case ProItem::OpAndKind:
case ProItem::OpOrKind:
delete itm;
break;
}
}
}
ProBranch::~ProBranch()
{
disposeItems(m_thenItems);
disposeItems(m_elseItems);
}
ProLoop::~ProLoop()
{
disposeItems(m_proitems);
}
ProFunctionDef::~ProFunctionDef()
{
disposeItems(m_proitems);
}
ProFile::ProFile(const QString &fileName)
: m_proitems(0),
m_refCount(1),
: m_refCount(1),
m_fileName(fileName)
{
int nameOff = fileName.lastIndexOf(QLatin1Char('/'));
......@@ -279,7 +259,6 @@ ProFile::ProFile(const QString &fileName)
ProFile::~ProFile()
{
ProItem::disposeItems(m_proitems);
}
QT_END_NAMESPACE
src/shared/proparser/proitems.h
View file @ a8d53e0a
......@@ -62,6 +62,9 @@ public:
ProString(const QString &str, ProStringConstants::OmitPreHashing);
explicit ProString(const char *str);
ProString(const char *str, ProStringConstants::OmitPreHashing);
ProString(const QString &str, int offset, int length);
ProString(const QString &str, int offset, int length, uint hash);
ProString(const QString &str, int offset, int length, ProStringConstants::OmitPreHashing);
QString toQString() const;
QString &toQString(QString &tmp) const;
bool operator==(const ProString &other) const;
......@@ -79,6 +82,8 @@ public:
ProString trimmed() const;
void clear() { m_string.clear(); m_length = 0; }
static uint hash(const QChar *p, int n);
private:
QString m_string;
int m_offset, m_length;
......@@ -104,130 +109,38 @@ public:
void removeDuplicates();
};
class ProItem
{
public:
enum ProItemKind {
ConditionKind,
OpNotKind,
OpAndKind,
OpOrKind,
VariableKind,
BranchKind,
LoopKind,
FunctionDefKind
};
ProItem(ProItemKind kind) : m_kind(kind), m_lineNumber(0) {}
ProItemKind kind() const { return m_kind; }
int lineNumber() const { return m_lineNumber; }
void setLineNumber(int lineNumber) { m_lineNumber = lineNumber; }
ProItem *next() const { return m_next; }
ProItem **nextRef() { return &m_next; }
static void disposeItems(ProItem *nitm);
private:
ProItem *m_next;
ProItemKind m_kind;
int m_lineNumber;
};
class ProVariable : public ProItem
{
public:
enum VariableOperator {
AddOperator = 0,
RemoveOperator = 1,
ReplaceOperator = 2,
SetOperator = 3,
UniqueAddOperator = 4
};
ProVariable(const ProString &name) : ProItem(VariableKind),
m_variableKind(SetOperator), m_variable(name) {}
void setVariableOperator(VariableOperator variableKind) { m_variableKind = variableKind; }
VariableOperator variableOperator() const { return m_variableKind; }
ProString variable() const { return m_variable; }
void setValue(const ProString &value) { m_value = value; }
ProString value() const { return m_value; }
private:
VariableOperator m_variableKind;
ProString m_variable;
ProString m_value;
};
class ProCondition : public ProItem
{
public:
explicit ProCondition(const QString &text) : ProItem(ConditionKind), m_text(text) {}
QString text() const { return m_text; }
private:
QString m_text;
};
class ProBranch : public ProItem
{
public:
ProBranch() : ProItem(BranchKind) {}
~ProBranch();
ProItem *thenItems() const { return m_thenItems; }
ProItem **thenItemsRef() { return &m_thenItems; }
ProItem *elseItems() const { return m_elseItems; }
ProItem **elseItemsRef() { return &m_elseItems; }
private:
ProItem *m_thenItems;
ProItem *m_elseItems;
};
class ProLoop : public ProItem
{
public:
ProLoop(const QString &var, const QString &expr)
: ProItem(LoopKind), m_variable(ProString(var)),
m_expression(ProString(expr, ProStringConstants::NoHash)) {}
~ProLoop();
ProString variable() const { return m_variable; }
ProString expression() const { return m_expression; }
ProItem *items() const { return m_proitems; }
ProItem **itemsRef() { return &m_proitems; }
private:
ProString m_variable;
ProString m_expression;
ProItem *m_proitems;
};
class ProFunctionDef : public ProItem
{
public:
enum FunctionType { TestFunction, ReplaceFunction };
ProFunctionDef(const QString &name, FunctionType type)
: ProItem(FunctionDefKind), m_name(ProString(name)), m_type(type), m_refCount(1) {}
~ProFunctionDef();
ProString name() const { return m_name; }
FunctionType type() const { return m_type; }
ProItem *items() const { return m_proitems; }
ProItem **itemsRef() { return &m_proitems; }
void ref() { m_refCount.ref(); }
void deref() { if (!m_refCount.deref()) delete this; }
private:
ProString m_name;
FunctionType m_type;
ProItemRefCount m_refCount;
ProItem *m_proitems;
// These token definitions affect both ProFileEvaluator and ProWriter
enum ProToken {
TokTerminator = 0, // end of stream (possibly not included in length; must be zero)
TokLine, // line marker: // +1 (2-nl) to 1st token of each line
// - line (1)
TokAssign, // variable = // "A=":2 => 1+4+2=7 (8)
TokAppend, // variable += // "A+=":3 => 1+4+2=7 (8)
TokAppendUnique, // variable *= // "A*=":3 => 1+4+2=7 (8)
TokRemove, // variable -= // "A-=":3 => 1+4+2=7 (8)
TokReplace, // variable ~= // "A~=":3 => 1+4+2=7 (8)
// - variable name: hash (2), length (1), chars (length)
// - expression: length (2), chars (length)
TokCondition, // CONFIG test: // "A":1 => 1+2=3 (4)
// - test name: lenght (1), chars (length)
TokNot, // '!' operator
TokAnd, // ':' operator
TokOr, // '|' operator
TokBranch, // branch point: // "X:A=":4 => [5]+1+4+1+1+[7]=19 (20)
// - then block length (2)
// - then block + TokTerminator (then block length)
// - else block length (2)
// - else block + TokTerminator (else block length)
TokForLoop, // for loop: // "for(A,B)":8 => 1+4+3+2+1=11 (12)
// - variable name: hash (2), length (1), chars (length)
// - expression: length (2), chars (length)
// - body length (2)
// - body + TokTerminator (body length)
TokTestDef, // test function definition: // "defineTest(A):":14 => 1+4+2+1=8 (9)
TokReplaceDef, // replace function definition: // "defineReplace(A):":17 => 1+4+2+1=8 (9)
// - function name: hash (2), length (1), chars (length)
// - body length (2)
// - body + TokTerminator (body length)
};
class ProFile
......@@ -239,15 +152,15 @@ public:
QString displayFileName() const { return m_displayFileName; }
QString fileName() const { return m_fileName; }
QString directoryName() const { return m_directoryName; }
ProItem *items() const { return m_proitems; }
ProItem **itemsRef() { return &m_proitems; }
const QString &items() const { return m_proitems; }
QString *itemsRef() { return &m_proitems; }
void ref() { m_refCount.ref(); }
void deref() { if (!m_refCount.deref()) delete this; }
private:
ProItem *m_proitems;
ProItemRefCount m_refCount;
QString m_proitems;
QString m_fileName;
QString m_displayFileName;
QString m_directoryName;
......
src/shared/proparser/prowriter.cpp
View file @ a8d53e0a
......@@ -35,20 +35,98 @@
using namespace Qt4ProjectManager::Internal;
static uint getBlockLen(const ushort *&tokPtr)
{
uint len = *tokPtr++;
len |= (uint)*tokPtr++ << 16;
return len;
}
static QString &getHashStr(const ushort *&tokPtr, QString &tmp)
{
tokPtr += 2; // ignore hash
uint len = *tokPtr++;
tmp.setRawData((const QChar *)tokPtr, len);
tokPtr += len;
return tmp;
}
static void skipStr(const ushort *&tokPtr)
{
uint len = *tokPtr++;
tokPtr += len;
}
static void skipHashStr(const ushort *&tokPtr)
{
tokPtr += 2;
uint len = *tokPtr++;
tokPtr += len;
}
static void skipLongStr(const ushort *&tokPtr)
{
uint len = getBlockLen(tokPtr);
tokPtr += len;
}
static void skipBlock(const ushort *&tokPtr)
{
uint len = getBlockLen(tokPtr);
tokPtr += len;
}
static void skipToken(ushort tok, const ushort *&tokPtr, int &lineNo)
{
switch (tok) {
case TokLine:
lineNo = *tokPtr++;
break;
case TokAssign:
case TokAppend:
case TokAppendUnique:
case TokRemove:
case TokReplace:
skipHashStr(tokPtr);
skipLongStr(tokPtr);
break;
case TokBranch:
skipBlock(tokPtr);
skipBlock(tokPtr);
break;
case TokForLoop:
skipHashStr(tokPtr);
skipLongStr(tokPtr);
skipBlock(tokPtr);
break;
case TokTestDef:
case TokReplaceDef:
skipHashStr(tokPtr);
skipBlock(tokPtr);
break;
case TokNot:
case TokAnd:
case TokOr:
break;
case TokCondition:
skipStr(tokPtr);
break;
default: Q_ASSERT_X(false, "skipToken", "unexpected item type");
}
}
void ProWriter::addFiles(ProFile *profile, QStringList *lines,
const QDir &proFileDir, const QStringList &filePaths,
const QString &var)
{
// Check if variable item exists as child of root item
for (ProItem *item = profile->items(); item; item = item->next()) {
if (item->kind() == ProItem::VariableKind) {
ProVariable *proVar = static_cast<ProVariable*>(item);
if (var == proVar->variable().toQString()
&& proVar->variableOperator() != ProVariable::RemoveOperator
&& proVar->variableOperator() != ProVariable::ReplaceOperator) {
int lineNo = proVar->lineNumber() - 1;
for (; lineNo < lines->count(); lineNo++) {
const ushort *tokPtr = (const ushort *)profile->items().constData();
int lineNo = 0;
QString tmp;
while (ushort tok = *tokPtr++) {
if (tok == TokAssign || tok == TokAppend || tok == TokAppendUnique) {
if (var == getHashStr(tokPtr, tmp)) {
for (--lineNo; lineNo < lines->count(); lineNo++) {
QString line = lines->at(lineNo);
int idx = line.indexOf(QLatin1Char('#'));
if (idx >= 0)
......@@ -74,6 +152,9 @@ void ProWriter::addFiles(ProFile *profile, QStringList *lines,
lines->insert(lineNo, added);
return;
}
skipLongStr(tokPtr);
} else {
skipToken(tok, tokPtr, lineNo);
}
}
......@@ -84,17 +165,25 @@ void ProWriter::addFiles(ProFile *profile, QStringList *lines,
*lines << added;
}
static void findProVariables(ProItem *item, const QStringList &vars,
QList<ProVariable *> *proVars)
static void findProVariables(const ushort *tokPtr, const QStringList &vars,
QList<int> *proVars)
{
for (; item; item = item->next()) {
if (item->kind() == ProItem::BranchKind) {
findProVariables(static_cast<ProBranch*>(item)->thenItems(), vars, proVars);
findProVariables(static_cast<ProBranch*>(item)->elseItems(), vars, proVars);
} else if (item->kind() == ProItem::VariableKind) {
ProVariable *proVar = static_cast<ProVariable*>(item);
if (vars.contains(proVar->variable().toQString()))
*proVars << proVar;
int lineNo = 0;
QString tmp;
while (ushort tok = *tokPtr++) {
if (tok == TokBranch) {
uint blockLen = getBlockLen(tokPtr);
findProVariables(tokPtr, vars, proVars);
tokPtr += blockLen;
blockLen = getBlockLen(tokPtr);
findProVariables(tokPtr, vars, proVars);
tokPtr += blockLen;
} else if (tok == TokAssign || tok == TokAppend || tok == TokAppendUnique) {
if (vars.contains(getHashStr(tokPtr, tmp)))
*proVars << lineNo;
skipLongStr(tokPtr);
} else {
skipToken(tok, tokPtr, lineNo);
}
}
}
......@@ -105,8 +194,8 @@ QStringList ProWriter::removeFiles(ProFile *profile, QStringList *lines,
{
QStringList notChanged = filePaths;
QList<ProVariable *> proVars;
findProVariables(profile->items(), vars, &proVars);
QList<int> varLines;
findProVariables((const ushort *)profile->items().constData(), vars, &varLines);
// This is a tad stupid - basically, it can remove only entries which
// the above code added.
......@@ -116,109 +205,105 @@ QStringList ProWriter::removeFiles(ProFile *profile, QStringList *lines,
// This code expects proVars to be sorted by the variables' appearance in the file.
int delta = 1;
foreach (ProVariable *proVar, proVars) {
if (proVar->variableOperator() != ProVariable::RemoveOperator
&& proVar->variableOperator() != ProVariable::ReplaceOperator) {
bool first = true;
int lineNo = proVar->lineNumber() - delta;
typedef QPair<int, int> ContPos;
QList<ContPos> contPos;
while (lineNo < lines->count()) {
QString &line = (*lines)[lineNo];
int lineLen = line.length();
bool killed = false;
bool saved = false;
int idx = line.indexOf(QLatin1Char('#'));
if (idx >= 0)
lineLen = idx;
QChar *chars = line.data();
forever {
if (!lineLen) {
if (idx >= 0)
goto nextLine;
goto nextVar;
}
QChar c = chars[lineLen - 1];
if (c != QLatin1Char(' ') && c != QLatin1Char('\t'))
break;
lineLen--;
}
{
int contCol = -1;
if (chars[lineLen - 1] == QLatin1Char('\\'))
contCol = --lineLen;
int colNo = 0;
if (first) {
colNo = line.indexOf(QLatin1Char('=')) + 1;
first = false;
saved = true;
}
while (colNo < lineLen) {
QChar c = chars[colNo];
if (c == QLatin1Char(' ') || c == QLatin1Char('\t')) {
colNo++;
continue;
}
int varCol = colNo;
while (colNo < lineLen) {
QChar c = chars[colNo];
if (c == QLatin1Char(' ') || c == QLatin1Char('\t'))
break;
colNo++;
}
QString fn = line.mid(varCol, colNo - varCol);
if (relativeFilePaths.contains(fn)) {
notChanged.removeOne(QDir::cleanPath(proFileDir.absoluteFilePath(fn)));
if (colNo < lineLen)
colNo++;
else if (varCol)
varCol--;
int len = colNo - varCol;
colNo = varCol;
line.remove(varCol, len);
lineLen -= len;
contCol -= len;
idx -= len;
if (idx >= 0)
line.insert(idx, QLatin1String("# ") + fn + QLatin1Char(' '));
killed = true;
} else {
saved = true;
}
}
if (saved) {
// Entries remained
contPos.clear();
} else if (killed) {
// Entries existed, but were all removed
if (contCol < 0) {
// This is the last line, so clear continuations leading to it
foreach (const ContPos &pos, contPos) {
QString &bline = (*lines)[pos.first];
bline.remove(pos.second, 1);
if (pos.second == bline.length())
while (bline.endsWith(QLatin1Char(' '))
|| bline.endsWith(QLatin1Char('\t')))
bline.chop(1);
}
contPos.clear();
}
if (idx < 0) {
// Not even a comment stayed behind, so zap the line
lines->removeAt(lineNo);
delta++;
continue;
}
}
if (contCol >= 0)
contPos.append(qMakePair(lineNo, contCol));
}
nextLine:
lineNo++;
}
nextVar: ;
}
foreach (int ln, varLines) {
bool first = true;
int lineNo = ln - delta;
typedef QPair<int, int> ContPos;
QList<ContPos> contPos;
while (lineNo < lines->count()) {
QString &line = (*lines)[lineNo];
int lineLen = line.length();
bool killed = false;
bool saved = false;
int idx = line.indexOf(QLatin1Char('#'));
if (idx >= 0)
lineLen = idx;
QChar *chars = line.data();
forever {
if (!lineLen) {
if (idx >= 0)
goto nextLine;