Commit e2f8a988 authored by Marco Bubke's avatar Marco Bubke Committed by Nikolai Kosjar

Introduce codemodelbackend process and library

This is a partial result of wip/clang-oop. More will follow.

This allows us to invoke the completion out of the Qt Creator process
and thus safes us as against libclang crashes.

At this point only the completion use case is supported.

Some notes on the individual components:

 src/libs/codemodelbackendipc
  * library encapsulating the inter process communication handling
  * used by the backend application and in a follow-up change by the
    creator integration

 src/libs/3rdparty/sqlite
  * version 3.8.10.2
  * dependency of codemodelbackendipc, will be used to storage indexing
    data, among others

 src/tools/codemodelbackend
  * the backend application

 tests/unit:
  * unit tests

Change-Id: I91a48e27467581a22fb760a18d8eb926008fea60
Reviewed-by: default avatarAlessandro Portale <alessandro.portale@theqtcompany.com>
Reviewed-by: default avatarNikolai Kosjar <nikolai.kosjar@theqtcompany.com>
Reviewed-by: default avatarMarco Bubke <marco.bubke@theqtcompany.com>
Reviewed-by: default avatarOswald Buddenhagen <oswald.buddenhagen@theqtcompany.com>
parent 51fec1a3
......@@ -117,6 +117,7 @@ tmp/
*.dll
*.exe
/bin/buildoutputparser
/bin/codemodelbackend
/bin/cpaster
/bin/cplusplus-ast2png
/bin/cplusplus-frontend
......
function llvmConfig(qbs)
{
var llvmInstallDirFromEnv = qbs.getEnv("LLVM_INSTALL_DIR")
var llvmConfigVariants = [
"llvm-config", "llvm-config-3.2", "llvm-config-3.3", "llvm-config-3.4",
"llvm-config-3.5", "llvm-config-3.6", "llvm-config-4.0", "llvm-config-4.1"
];
// Prefer llvm-config* from LLVM_INSTALL_DIR
if (llvmInstallDirFromEnv) {
for (var i = 0; i < llvmConfigVariants.length; ++i) {
var variant = llvmInstallDirFromEnv + "/bin/" + llvmConfigVariants[i];
if (File.exists(variant))
return variant;
}
}
// Find llvm-config* in PATH
var pathListString = qbs.getEnv("PATH");
var separator = qbs.hostOS.contains("windows") ? ";" : ":";
var pathList = pathListString.split(separator);
for (var i = 0; i < llvmConfigVariants.length; ++i) {
for (var j = 0; j < pathList.length; ++j) {
var variant = pathList[j] + "/" + llvmConfigVariants[i];
if (File.exists(variant))
return variant;
}
}
return undefined;
}
function includeDir(llvmConfig, processOutputReader)
{
return processOutputReader.readOutput(llvmConfig, ["--includedir"])
}
function libDir(llvmConfig, processOutputReader)
{
return processOutputReader.readOutput(llvmConfig, ["--libdir"])
}
function version(llvmConfig, processOutputReader)
{
return processOutputReader.readOutput(llvmConfig, ["--version"])
.replace(/(\d+\.\d+\.\d+).*/, "$1")
}
function libraries(targetOS)
{
return ["clang"] + (targetOS.contains("windows") ? ["advapi32", "shell32"] : [])
}
#include <math.h>
#include <assert.h>
#include "sqlite3.h"
static void okapi_bm25(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal) {
assert(sizeof(int) == 4);
unsigned int *matchinfo = (unsigned int *)sqlite3_value_blob(apVal[0]);
int searchTextCol = sqlite3_value_int(apVal[1]);
double K1 = ((nVal >= 3) ? sqlite3_value_double(apVal[2]) : 1.2);
double B = ((nVal >= 4) ? sqlite3_value_double(apVal[3]) : 0.75);
int P_OFFSET = 0;
int C_OFFSET = 1;
int X_OFFSET = 2;
int termCount = matchinfo[P_OFFSET];
int colCount = matchinfo[C_OFFSET];
int N_OFFSET = X_OFFSET + 3*termCount*colCount;
int A_OFFSET = N_OFFSET + 1;
int L_OFFSET = (A_OFFSET + colCount);
double totalDocs = matchinfo[N_OFFSET];
double avgLength = matchinfo[A_OFFSET + searchTextCol];
double docLength = matchinfo[L_OFFSET + searchTextCol];
double sum = 0.0;
for (int i = 0; i < termCount; i++) {
int currentX = X_OFFSET + (3 * searchTextCol * (i + 1));
double termFrequency = matchinfo[currentX];
double docsWithTerm = matchinfo[currentX + 2];
double idf = log(
(totalDocs - docsWithTerm + 0.5) /
(docsWithTerm + 0.5)
);
double rightSide = (
(termFrequency * (K1 + 1)) /
(termFrequency + (K1 * (1 - B + (B * (docLength / avgLength)))))
);
sum += (idf * rightSide);
}
sqlite3_result_double(pCtx, sum);
}
//
// Created by Joshua Wilson on 27/05/14.
// Copyright (c) 2014 Joshua Wilson. All rights reserved.
// https://github.com/neozenith/sqlite-okapi-bm25
//
// This is an extension to the work of "Radford 'rads' Smith"
// found at: https://github.com/rads/sqlite-okapi-bm25
// which is covered by the MIT License
// http://opensource.org/licenses/MIT
// the following code shall also be covered by the same MIT License
static void okapi_bm25f(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal) {
assert(sizeof(int) == 4);
unsigned int *matchinfo = (unsigned int *)sqlite3_value_blob(apVal[0]);
//Setting the default values and ignoring argument based inputs so the extra
//arguments can be the column weights instead.
double K1 = 1.2;// ((nVal >= 3) ? sqlite3_value_double(apVal[2]) : 1.2);
double B = 0.75;// ((nVal >= 4) ? sqlite3_value_double(apVal[3]) : 0.75);
//For a good explanation fo the maths and how to choose these variables
//http://stackoverflow.com/a/23161886/622276
//NOTE: the rearranged order of parameters to match the order presented on
//SQLite3 FTS3 documentation 'pcxnals' (http://www.sqlite.org/fts3.html#matchinfo)
int P_OFFSET = 0;
int C_OFFSET = 1;
int X_OFFSET = 2;
int termCount = matchinfo[P_OFFSET];
int colCount = matchinfo[C_OFFSET];
int N_OFFSET = X_OFFSET + 3*termCount*colCount;
int A_OFFSET = N_OFFSET + 1;
int L_OFFSET = (A_OFFSET + colCount);
// int S_OFFSET = (L_OFFSET + colCount); //useful as a pseudo proximity weighting per field/column
double totalDocs = matchinfo[N_OFFSET];
double avgLength = 0.0;
double docLength = 0.0;
for (int col = 0; col < colCount; col++)
{
avgLength += matchinfo[A_OFFSET + col];
docLength += matchinfo[L_OFFSET + col];
}
double epsilon = 1.0 / (totalDocs*avgLength);
double sum = 0.0;
for (int t = 0; t < termCount; t++) {
for (int col = 0 ; col < colCount; col++)
{
int currentX = X_OFFSET + (3 * col * (t + 1));
double termFrequency = matchinfo[currentX];
double docsWithTerm = matchinfo[currentX + 2];
double idf = log(
(totalDocs - docsWithTerm + 0.5) /
(docsWithTerm + 0.5)
);
// "...terms appearing in more than half of the corpus will provide negative contributions to the final document score."
//http://en.wikipedia.org/wiki/Okapi_BM25
idf = (idf < 0) ? epsilon : idf; //common terms could have no effect (\epsilon=0.0) or a very small effect (\epsilon=1/NoOfTokens which asymptotes to 0.0)
double rightSide = (
(termFrequency * (K1 + 1)) /
(termFrequency + (K1 * (1 - B + (B * (docLength / avgLength)))))
);
rightSide += 1.0;
//To comply with BM25+ that solves a lower bounding issue where large documents that match are unfairly scored as
//having similar relevancy as short documents that do not contain as many terms
//Yuanhua Lv and ChengXiang Zhai. 'Lower-bounding term frequency normalization.' In Proceedings of CIKM'2011, pages 7-16.
//http://sifaka.cs.uiuc.edu/~ylv2/pub/cikm11-lowerbound.pdf
double weight = ((nVal > col+1) ? sqlite3_value_double(apVal[col+1]) : 1.0);
// double subsequence = matchinfo[S_OFFSET + col];
sum += (idf * rightSide) * weight; // * subsequence; //useful as a pseudo proximty weighting
}
}
sqlite3_result_double(pCtx, sum);
}
static void okapi_bm25f_kb(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal) {
assert(sizeof(int) == 4);
unsigned int *matchinfo = (unsigned int *)sqlite3_value_blob(apVal[0]);
//Setting the default values and ignoring argument based inputs so the extra
//arguments can be the column weights instead.
if (nVal < 2) sqlite3_result_error(pCtx, "wrong number of arguments to function okapi_bm25_kb(), expected k1 parameter", -1);
if (nVal < 3) sqlite3_result_error(pCtx, "wrong number of arguments to function okapi_bm25_kb(), expected b parameter", -1);
double K1 = sqlite3_value_double(apVal[1]);
double B = sqlite3_value_double(apVal[2]);
//For a good explanation fo the maths and how to choose these variables
//http://stackoverflow.com/a/23161886/622276
//NOTE: the rearranged order of parameters to match the order presented on
//SQLite3 FTS3 documentation 'pcxnals' (http://www.sqlite.org/fts3.html#matchinfo)
int P_OFFSET = 0;
int C_OFFSET = 1;
int X_OFFSET = 2;
int termCount = matchinfo[P_OFFSET];
int colCount = matchinfo[C_OFFSET];
int N_OFFSET = X_OFFSET + 3*termCount*colCount;
int A_OFFSET = N_OFFSET + 1;
int L_OFFSET = (A_OFFSET + colCount);
// int S_OFFSET = (L_OFFSET + colCount); //useful as a pseudo proximity weighting per field/column
double totalDocs = matchinfo[N_OFFSET];
double avgLength = 0.0;
double docLength = 0.0;
for (int col = 0; col < colCount; col++)
{
avgLength += matchinfo[A_OFFSET + col];
docLength += matchinfo[L_OFFSET + col];
}
double epsilon = 1.0 / (totalDocs*avgLength);
double sum = 0.0;
for (int t = 0; t < termCount; t++) {
for (int col = 0 ; col < colCount; col++)
{
int currentX = X_OFFSET + (3 * col * (t + 1));
double termFrequency = matchinfo[currentX];
double docsWithTerm = matchinfo[currentX + 2];
double idf = log(
(totalDocs - docsWithTerm + 0.5) /
(docsWithTerm + 0.5)
);
// "...terms appearing in more than half of the corpus will provide negative contributions to the final document score."
//http://en.wikipedia.org/wiki/Okapi_BM25
idf = (idf < 0) ? epsilon : idf; //common terms could have no effect (\epsilon=0.0) or a very small effect (\epsilon=1/NoOfTokens which asymptotes to 0.0)
double rightSide = (
(termFrequency * (K1 + 1)) /
(termFrequency + (K1 * (1 - B + (B * (docLength / avgLength)))))
);
rightSide += 1.0;
//To comply with BM25+ that solves a lower bounding issue where large documents that match are unfairly scored as
//having similar relevancy as short documents that do not contain as many terms
//Yuanhua Lv and ChengXiang Zhai. 'Lower-bounding term frequency normalization.' In Proceedings of CIKM'2011, pages 7-16.
//http://sifaka.cs.uiuc.edu/~ylv2/pub/cikm11-lowerbound.pdf
double weight = ((nVal > col+3) ? sqlite3_value_double(apVal[col+3]) : 1.0);
// double subsequence = matchinfo[S_OFFSET + col];
sum += (idf * rightSide) * weight; // * subsequence; //useful as a pseudo proximty weighting
}
}
sqlite3_result_double(pCtx, sum);
}
INCLUDEPATH *= $$PWD
VPATH *= $$PWD
HEADERS += okapi_bm25.h \
sqlite3.h \
sqlite3ext.h
SOURCES += sqlite3.c
win32:DEFINES += SQLITE_API=__declspec(dllexport)
unix:DEFINES += SQLITE_API=\"__attribute__((visibility(\\\"default\\\")))\"
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
/*
** 2006 June 7
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This header file defines the SQLite interface for use by
** shared libraries that want to be imported as extensions into
** an SQLite instance. Shared libraries that intend to be loaded
** as extensions by SQLite should #include this file instead of
** sqlite3.h.
*/
#ifndef _SQLITE3EXT_H_
#define _SQLITE3EXT_H_
#include "sqlite3.h"
typedef struct sqlite3_api_routines sqlite3_api_routines;
/*
** The following structure holds pointers to all of the SQLite API
** routines.
**
** WARNING: In order to maintain backwards compatibility, add new
** interfaces to the end of this structure only. If you insert new
** interfaces in the middle of this structure, then older different
** versions of SQLite will not be able to load each other's shared
** libraries!
*/
struct sqlite3_api_routines {
void * (*aggregate_context)(sqlite3_context*,int nBytes);
int (*aggregate_count)(sqlite3_context*);
int (*bind_blob)(sqlite3_stmt*,int,const void*,int n,void(*)(void*));
int (*bind_double)(sqlite3_stmt*,int,double);
int (*bind_int)(sqlite3_stmt*,int,int);
int (*bind_int64)(sqlite3_stmt*,int,sqlite_int64);
int (*bind_null)(sqlite3_stmt*,int);
int (*bind_parameter_count)(sqlite3_stmt*);
int (*bind_parameter_index)(sqlite3_stmt*,const char*zName);
const char * (*bind_parameter_name)(sqlite3_stmt*,int);
int (*bind_text)(sqlite3_stmt*,int,const char*,int n,void(*)(void*));
int (*bind_text16)(sqlite3_stmt*,int,const void*,int,void(*)(void*));
int (*bind_value)(sqlite3_stmt*,int,const sqlite3_value*);
int (*busy_handler)(sqlite3*,int(*)(void*,int),void*);
int (*busy_timeout)(sqlite3*,int ms);
int (*changes)(sqlite3*);
int (*close)(sqlite3*);
int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*,
int eTextRep,const char*));
int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*,
int eTextRep,const void*));
const void * (*column_blob)(sqlite3_stmt*,int iCol);
int (*column_bytes)(sqlite3_stmt*,int iCol);
int (*column_bytes16)(sqlite3_stmt*,int iCol);
int (*column_count)(sqlite3_stmt*pStmt);
const char * (*column_database_name)(sqlite3_stmt*,int);
const void * (*column_database_name16)(sqlite3_stmt*,int);
const char * (*column_decltype)(sqlite3_stmt*,int i);
const void * (*column_decltype16)(sqlite3_stmt*,int);
double (*column_double)(sqlite3_stmt*,int iCol);
int (*column_int)(sqlite3_stmt*,int iCol);
sqlite_int64 (*column_int64)(sqlite3_stmt*,int iCol);
const char * (*column_name)(sqlite3_stmt*,int);
const void * (*column_name16)(sqlite3_stmt*,int);
const char * (*column_origin_name)(sqlite3_stmt*,int);
const void * (*column_origin_name16)(sqlite3_stmt*,int);
const char * (*column_table_name)(sqlite3_stmt*,int);
const void * (*column_table_name16)(sqlite3_stmt*,int);
const unsigned char * (*column_text)(sqlite3_stmt*,int iCol);
const void * (*column_text16)(sqlite3_stmt*,int iCol);
int (*column_type)(sqlite3_stmt*,int iCol);
sqlite3_value* (*column_value)(sqlite3_stmt*,int iCol);
void * (*commit_hook)(sqlite3*,int(*)(void*),void*);
int (*complete)(const char*sql);
int (*complete16)(const void*sql);
int (*create_collation)(sqlite3*,const char*,int,void*,
int(*)(void*,int,const void*,int,const void*));
int (*create_collation16)(sqlite3*,const void*,int,void*,
int(*)(void*,int,const void*,int,const void*));
int (*create_function)(sqlite3*,const char*,int,int,void*,
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
void (*xFinal)(sqlite3_context*));
int (*create_function16)(sqlite3*,const void*,int,int,void*,
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
void (*xFinal)(sqlite3_context*));
int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*);
int (*data_count)(sqlite3_stmt*pStmt);
sqlite3 * (*db_handle)(sqlite3_stmt*);
int (*declare_vtab)(sqlite3*,const char*);
int (*enable_shared_cache)(int);
int (*errcode)(sqlite3*db);
const char * (*errmsg)(sqlite3*);
const void * (*errmsg16)(sqlite3*);
int (*exec)(sqlite3*,const char*,sqlite3_callback,void*,char**);
int (*expired)(sqlite3_stmt*);
int (*finalize)(sqlite3_stmt*pStmt);
void (*free)(void*);
void (*free_table)(char**result);
int (*get_autocommit)(sqlite3*);
void * (*get_auxdata)(sqlite3_context*,int);
int (*get_table)(sqlite3*,const char*,char***,int*,int*,char**);
int (*global_recover)(void);
void (*interruptx)(sqlite3*);
sqlite_int64 (*last_insert_rowid)(sqlite3*);
const char * (*libversion)(void);
int (*libversion_number)(void);
void *(*malloc)(int);
char * (*mprintf)(const char*,...);
int (*open)(const char*,sqlite3**);
int (*open16)(const void*,sqlite3**);
int (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**);
int (*prepare16)(sqlite3*,const void*,int,sqlite3_stmt**,const void**);
void * (*profile)(sqlite3*,void(*)(void*,const char*,sqlite_uint64),void*);
void (*progress_handler)(sqlite3*,int,int(*)(void*),void*);
void *(*realloc)(void*,int);
int (*reset)(sqlite3_stmt*pStmt);
void (*result_blob)(sqlite3_context*,const void*,int,void(*)(void*));
void (*result_double)(sqlite3_context*,double);
void (*result_error)(sqlite3_context*,const char*,int);
void (*result_error16)(sqlite3_context*,const void*,int);
void (*result_int)(sqlite3_context*,int);
void (*result_int64)(sqlite3_context*,sqlite_int64);
void (*result_null)(sqlite3_context*);
void (*result_text)(sqlite3_context*,const char*,int,void(*)(void*));
void (*result_text16)(sqlite3_context*,const void*,int,void(*)(void*));
void (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*));
void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*));
void (*result_value)(sqlite3_context*,sqlite3_value*);
void * (*rollback_hook)(sqlite3*,void(*)(void*),void*);
int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,
const char*,const char*),void*);
void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*));
char * (*snprintf)(int,char*,const char*,...);
int (*step)(sqlite3_stmt*);
int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,
char const**,char const**,int*,int*,int*);
void (*thread_cleanup)(void);
int (*total_changes)(sqlite3*);
void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*);
int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*);
void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*,
sqlite_int64),void*);
void * (*user_data)(sqlite3_context*);
const void * (*value_blob)(sqlite3_value*);
int (*value_bytes)(sqlite3_value*);
int (*value_bytes16)(sqlite3_value*);
double (*value_double)(sqlite3_value*);
int (*value_int)(sqlite3_value*);
sqlite_int64 (*value_int64)(sqlite3_value*);
int (*value_numeric_type)(sqlite3_value*);
const unsigned char * (*value_text)(sqlite3_value*);
const void * (*value_text16)(sqlite3_value*);
const void * (*value_text16be)(sqlite3_value*);
const void * (*value_text16le)(sqlite3_value*);
int (*value_type)(sqlite3_value*);
char *(*vmprintf)(const char*,va_list);
/* Added ??? */
int (*overload_function)(sqlite3*, const char *zFuncName, int nArg);
/* Added by 3.3.13 */
int (*prepare_v2)(sqlite3*,const char*,int,sqlite3_stmt**,const char**);
int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**);
int (*clear_bindings)(sqlite3_stmt*);
/* Added by 3.4.1 */
int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*,
void (*xDestroy)(void *));
/* Added by 3.5.0 */
int (*bind_zeroblob)(sqlite3_stmt*,int,int);
int (*blob_bytes)(sqlite3_blob*);
int (*blob_close)(sqlite3_blob*);
int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64,
int,sqlite3_blob**);
int (*blob_read)(sqlite3_blob*,void*,int,int);
int (*blob_write)(sqlite3_blob*,const void*,int,int);
int (*create_collation_v2)(sqlite3*,const char*,int,void*,
int(*)(void*,int,const void*,int,const void*),
void(*)(void*));
int (*file_control)(sqlite3*,const char*,int,void*);
sqlite3_int64 (*memory_highwater)(int);
sqlite3_int64 (*memory_used)(void);
sqlite3_mutex *(*mutex_alloc)(int);
void (*mutex_enter)(sqlite3_mutex*);
void (*mutex_free)(sqlite3_mutex*);
void (*mutex_leave)(sqlite3_mutex*);
int (*mutex_try)(sqlite3_mutex*);
int (*open_v2)(const char*,sqlite3**,int,const char*);
int (*release_memory)(int);
void (*result_error_nomem)(sqlite3_context*);
</