How to debug error when loading a custom sqlite fts5 extension? - sqlite

Written a custom ranking function for the sqlite3 full text engine (fts5). The extension compiles but throws a "segmentation fault (core dumped) sqlite3" error when attempting to load it with ".load /var/www/rankprice.so" . Unsure about the correctness of my extension code.
I have replaced my original ranking function with code from the docs. The extension loading code is also pretty much copied from the docs.
THere are 3 functions:
static void rankprice( //Actual extension function,
fts5_api *fts5_api_from_db(sqlite3 *db) // Gets pointer to the fts5_api pointer
int sqlite3_extension_init // Loads the extension
The docs have not been updated on https://www.sqlite.org/fts5.html#extending_fts5 . In the function
fts5_api *fts5_api_from_db(sqlite3 *db){
fts5_api *pRet = 0;
sqlite3_stmt *pStmt = 0;
if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0) ){
sqlite3_bind_pointer(pStmt, (void*)&pRet, "fts5_api_ptr", NULL);
sqlite3_step(pStmt);
}
sqlite3_finalize(pStmt);
return pRet;
}
sqlite3_bind_pointer should have 5 arguments , the second argument should be an int, but I am unsure of its correct value.
Here is the complete code:
#include <stdio.h>
#include <assert.h>
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
/*
** Actual extension function, copied from https://www.sqlite.org/fts5.html
*/
static void rankprice(
const Fts5ExtensionApi *pApi,
Fts5Context *pFts,
sqlite3_context *pCtx,
int nVal,
sqlite3_value **apVal
){
int rc;
int nToken;
rc = pApi->xColumnSize(pFts, -1, &nToken);
if( rc==SQLITE_OK ){
sqlite3_result_int(pCtx, nToken);
}else{
sqlite3_result_error_code(pCtx, rc);
}
}
/*
** Return a pointer to the fts5_api pointer for database connection db.
** If an error occurs, return NULL and leave an error in the database
** handle (accessible using sqlite3_errcode()/errmsg()).
*/
fts5_api *fts5_api_from_db(sqlite3 *db){
fts5_api *pRet = 0;
sqlite3_stmt *pStmt = 0;
if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0) ){
sqlite3_bind_pointer(pStmt, 1,(void*)&pRet, "fts5_api_ptr", NULL);
sqlite3_step(pStmt);
}
sqlite3_finalize(pStmt);
return pRet;
}
/* Creates a SQL extension function. Tells sqlite which function to load as extension*/
int sqlite3_extension_init(
sqlite3 *db,
char **pzErrMsg,
const sqlite3_api_routines *pApi
){
fts5_api *fpApi;
fpApi = fts5_api_from_db(db);
SQLITE_EXTENSION_INIT2(pApi);
struct Builtin {
const char *zFunc; /* Function name (nul-terminated) */
void *pUserData; /* User-data pointer */
fts5_extension_function xFunc; /* Callback function */
void (*xDestroy)(void*); /* Destructor function */
} aBuiltin [] = {
{ "rankingMe", 0, rankprice, 0 },
};
int rc = SQLITE_OK;
rc = fpApi->xCreateFunction(fpApi,
aBuiltin[0].zFunc,
aBuiltin[0].pUserData,
aBuiltin[0].xFunc,
aBuiltin[0].xDestroy
);
return rc;
}
Compiled using
gcc -shared -fPIC -g /var/www/rankprice.c -o /var/www/rankprice.so
Exact error message:
SQLite version 3.28.0 2019-04-16 19:49:53
Connected to a transient in-memory database.
sqlite> .load /var/www/rankprice.so
[2] 52632 segmentation fault (core dumped) sqlite3
Links:
https://www.sqlite.org/fts5.html
https://github.com/sqlite/sqlite/blob/ff119f04b42a7ad3a16c306446af7e301d34026a/ext/fts5/fts5_aux.c

Related

SQLite: How embed the memvfs extension to Amalgamation?

I need to load/save SQLite database in memory buffer. For this, I want embed the memvfs extension into sqlite3 code and compile it wholly as sqlite3.dll.
How do it?
Update1:
I want use the memvfs as temp memory buffer. My program load data from net to buffer, connect to this memory buffer and restore data into empty in-memory db. I thoutgh that inclusion of memvfs to sqlite amalgamation would improve perfomance.
Update2:
If you want to use memvfs extension pay attention to bug in readme comment in source. Use "PRAGMA journal_mode=OFF" instead "journal_mode=NONE"
Update3:
Another bug in memvfs.c - use 'max' instead 'maxsz' for maxsz param in URI.
The sqlite developers carefully set a rakes :(
Test program to demonstrate using memvfs:
#include <fcntl.h>
#include <sqlite3.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
int main(void) {
sqlite3 *db;
char *err;
// Open an in-memory database to use as a handle for loading the memvfs extension
if (sqlite3_open(":memory:", &db) != SQLITE_OK) {
fprintf(stderr, "open :memory: %s\n", sqlite3_errmsg(db));
return EXIT_FAILURE;
}
sqlite3_enable_load_extension(db, 1);
if (sqlite3_load_extension(db, "./memvfs", NULL, &err) != SQLITE_OK) {
fprintf(stderr, "load extension: %s\n", err);
return EXIT_FAILURE;
}
// Done with this database
sqlite3_close(db);
// Read the real database into memory
int fd = open("foo.db", O_RDONLY);
if (fd < 0) {
perror("open");
return EXIT_FAILURE;
}
struct stat s;
if (fstat(fd, &s) < 0) {
perror("fstat");
return EXIT_FAILURE;
}
void *memdb = sqlite3_malloc64(s.st_size);
if (read(fd, memdb, s.st_size) != s.st_size) {
perror("read");
return EXIT_FAILURE;
}
close(fd);
// And open that memory with memvfs now that it holds a valid database
char *memuri = sqlite3_mprintf("file:whatever?ptr=0x%p&sz=%lld&freeonclose=1",
memdb, (long long)s.st_size);
printf("Trying to open '%s'\n", memuri);
if (sqlite3_open_v2(memuri, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI,
"memvfs") != SQLITE_OK) {
fprintf(stderr, "open memvfs: %s\n", sqlite3_errmsg(db));
return EXIT_FAILURE;
}
sqlite3_free(memuri);
// Try querying the database to show it works.
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, "SELECT b FROM test", -1, &stmt, NULL) !=
SQLITE_OK) {
fprintf(stderr, "prepare: %s\n", sqlite3_errmsg(db));
sqlite3_close(db);
return EXIT_FAILURE;
}
for (int rc = sqlite3_step(stmt); rc == SQLITE_ROW; rc = sqlite3_step(stmt)) {
printf("%d\n", sqlite3_column_int(stmt, 0));
}
sqlite3_finalize(stmt);
sqlite3_close(db);
return 0;
}
Usage:
# Create a test database to use with memvfs
$ sqlite3 foo.db
sqlite> CREATE TABLE test(b INTEGER);
sqlite> INSERT INTO test VALUES (1), (2);
sqlite> .quit
# Compile the memvfs module and test program
$ gcc -O -fPIC -shared -o memvfs.so memvfs.c
$ gcc -O -Wall -Wextra testmem.c -lsqlite3
# And run it.
$ ./a.out
Trying to open 'file:whatever?ptr=0x56653FE2B940&sz=8192&freeonclose=1'
1
2
Same workflow if you compile it directly into your program instead of using a loadable module; you just have to call sqlite3_memvfs_init() with the right arguments instead of using sqlite3_load_extension().

undirected Graph adjacency list implementation using C

Code :
#include <stdio.h>
#include <stdlib.h>
#define N 5
typedef struct list
{
int data;
struct list * next;//self referenced structure
}slist;
void displayList(slist * start)
{
slist * temp;
if(start==NULL)
{
printf("Empty Linked List");
return;
}
for(temp=start;temp!=NULL;temp=temp->next)
printf("%d ",temp->data);
return;
}
void insertLast(slist * * start,slist * node)
{
slist * temp;
if(start==NULL){
(* start)=node;
return;
}
temp=(* start);
while((temp->next)!= NULL)
temp=temp->next;
temp->next=node;
return;
}
int main()
{
int i,j;
//slist * node;
char Ans;
/*printf("Write the number of vertices\n");
scanf("%d",&N);*/
slist * start[N];
for(i=0;i<N;i++)
start[i]=NULL;
for(i=0;i<N;i++)
{
for(j=i+1;j<N;j++)
{
printf("Is there a connection between V[%d] and V[%d]\n",(i+1),(j+1));
scanf(" %c",&Ans);
if(Ans=='y'||Ans=='Y')
{
slist * node1=(slist *)malloc(sizeof(slist));
node1->data=(j+1); node1->next=NULL;
insertLast(&start[i],node1);enter code here
slist * node2=(slist *)malloc(sizeof(slist));
node2->data=(i+1); node2->next=NULL;
insertLast(&start[j],node2);
}
}
}
for(i=0;i<N;i++)
{
displayList(start[i]);
printf("\n");
}
return 0;
}
The above code is showing segmentation fault at the line where while((temp->next)!=NULL) is written whereas while creation of linked lists, the same insertLast worked just fine. What is the fault in the code?
Your program crashed as you are checking if start is NULL or not. But that does not guarantee that *start is also not NULL. In such situation, temp gets NULL and in while loop temp->next actually trying to access next element of NULL pointer and that is why the crash.
Changing this line -
if(start==NULL)
to
if(*start==NULL)
in insertLast() will fix the crash.
I also recommend to use a debugger like gdb to debug such issues.

Adding a custom sqlite function to a Qt application

I am trying to add a custom sqlite3 regexp function into my Qt application (as recommended by this answer). But as soon as I call the sqlite3_create_function function, I get the message The program has unexpectedly finished. When I debug, it terminates in a segmentation fault in sqlite3_mutex_enter. There is a MWE below, with apologies for the absolute file paths.
The regexp implementation in my code is from this site; it also fails with the msign function here. The various checks of driver()->handle() are straight from the Qt docs.
Incidentally, I used select sqlite_version(); to determine that Qt 5.5 uses sqlite version 3.8.8.2. I found that version by looking through old commits in the Qt GitHub repository.
MWE.pro
QT += core gui
TARGET = MWE
TEMPLATE = app
QT += sql
SOURCES += main.cpp \
D:\Qt\Qt5.5.0\5.5\Src\3rdparty\sqlite\sqlite3.c
HEADERS += D:\Qt\Qt5.5.0\5.5\Src\3rdparty\sqlite\sqlite3.h
main.cpp
#include <QtSql>
#include "D:/Qt/Qt5.5.0/5.5/Src/3rdparty/sqlite/sqlite3.h"
void qtregexp(sqlite3_context* ctx, int argc, sqlite3_value** argv)
{
QRegExp regex;
QString str1((const char*)sqlite3_value_text(argv[0]));
QString str2((const char*)sqlite3_value_text(argv[1]));
regex.setPattern(str1);
regex.setCaseSensitivity(Qt::CaseInsensitive);
bool b = str2.contains(regex);
if (b)
{
sqlite3_result_int(ctx, 1);
}
else
{
sqlite3_result_int(ctx, 0);
}
}
int main(int argc, char *argv[])
{
QSqlDatabase db = QSqlDatabase::addDatabase("QSQLITE");
db.setDatabaseName("my.db");
db.open();
QVariant v = db.driver()->handle();
if (v.isValid() && qstrcmp(v.typeName(), "sqlite3*")==0) {
sqlite3 *db_handle = *static_cast<sqlite3 **>(v.data());
if (db_handle != 0) { // check that it is not NULL
// This shows that the database handle is generally valid:
qDebug() << sqlite3_db_filename(db_handle, "main");
sqlite3_create_function(db_handle, "regexp", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, NULL, &qtregexp, NULL, NULL);
qDebug() << "This won't be reached."
QSqlQuery query;
query.prepare("select regexp('p$','tap');");
query.exec();
query.next();
qDebug() << query.value(0).toString();
}
}
db.close();
}
You need to call sqlite3_initialize() after you get the database handle from Qt, according to this forum post.
...
sqlite3 *db_handle = *static_cast<sqlite3 **>(v.data());
if (db_handle != 0) { // check that it is not NULL
sqlite3_initialize();
...
This solves the issue.

Count the running process in AIX by it's name in C

Operating system: IBM AIX 5.3
Compiler: xlc
Hello Everyone
I have a project that using C to process some files using multi-processes. The number of sub-processes are mainly depends on the the number of incoming files and the number of current running processes. I need a reliable method to count on how many sub-processes are running at background.
By comparing the efficiency, directly reading /proc directory may have better performance than invoking popen() to execute $ ps -ef | grep blah.
I wrote a function to read psinfo in /proc/pid/psinfo and compare the arugments.
Pseudo code is as follow:
int count = 0;
dp = opendir("/proc");
while (readdir_r(...))
{
if (dir is not a process)
return -1;
if (dir's owner is not current user)
return -2;
if (failed to open "/proc/[pid]/psinfo")
return -3;
if (failed to read "/proc/[pid]/psinfo")
return -4;
if (process's name matches the given pname)
count += 1;
}
return count;
The function generally runs perfectly at single call. However, it returns -2 or -3 or even wrong counts when embedded in while loop.
The function failed to read the attribute of /proc/pid randomly. It tells No such file or directory.
There is also a small chance to get wrong counts at the end. There seems to be an extra process with certain pid but disappeared when printing the current processes using ps.
I think there is any change were made when reading from sub-directory rapidly after parent directory were being listed.
Is there any thing I made wrong or is there any method to avoid the race condition?
Come extra information about psinfo in AIX
http://www-01.ibm.com/support/knowledgecenter/ssw_aix_53/com.ibm.aix.files/doc/aixfiles/proc.htm%23files-proc?lang=en[233]
Here is the full source code:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <dirent.h>
#include <sys/procfs.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
int countProcess(char *pname)
{
DIR *dir;
int fd;
int pid;
int uid;
int pcounter = 0;
struct psinfo pinfo;
struct dirent entry;
struct dirent *result;
struct stat fsstat;
char path[256];
char process_path[256];
/* Open the /proc directory */
if ((dir = opendir("/proc")) == NULL)
{
return -1;
}
/* Retrieve the current user id */
uid = getuid();
/* Walk through the /proc dir */
for (readdir_r(dir, &entry, &result); result != NULL; readdir_r(dir, &entry, &result))
{
/* See if this is a process, e.g., the dirname is a number
If not, then start off again
*/
if ((pid = atoi(entry.d_name)) == 0)
{
continue;
}
/* Get the attributes of process dir */
snprintf(process_path, sizeof(process_path), "/proc/%s", entry.d_name);
if (stat(process_path, &fsstat) == -1)
{
closedir(dir);
return -2;
}
/* Verify if the process runs by current user
If not, then start off again
*/
if (fsstat.st_uid != uid)
{
continue;
}
/* Open and read from psinfo file */
snprintf(path, sizeof(path), "/proc/%s/psinfo", entry.d_name);
if ((fd = open(path, O_RDONLY)) < 0)
{
close(fd);
closedir(dir);
return -3;
}
if (read(fd, &pinfo, sizeof(pinfo)) < 0)
{
close(fd);
closedir(dir);
return -4;
}
/* String comparison, if same, increase the counter */
if (!strcmp(pinfo.pr_psargs, pname))
{
pcounter++;
}
close(fd);
}
/* returns counter */
closedir(dir);
return pcounter;
}
Update 13/Jan/2015
Thanks to CoreyStup. The race condition can be bypassed by getprocs() function provided in procinfo.h
Here is the code for the solution
#include <stdio.h>
#include <unistd.h>
#include <procinfo.h>
#include <sys/types.h>
int countProcess(const char *pname)
{
struct procsinfo pinfo;
pid_t pid = 0;
uid_t uid;
char args[256];
int index;
int pcounter = 0;
memset(args, 0, sizeof(args));
uid = getuid();
/* Get procsinfo from internal API */
while (0 < getprocs(&pinfo, (int)sizeof(struct procsinfo), NULL, 0, &pid, 1))
{
/* Skip the process that doesn't belong to current user */
if (pinfo.pi_uid != uid)
{
continue;
}
/* Get process arguments */
if (getargs(&pinfo, sizeof(struct procsinfo), args, sizeof(args)) != 0)
{
return -1;
}
/* getargs returns the args list seperated by 0, we need to use space to replace 0 */
for (index = 0; index < 256 - 1 && !(args[index] == 0 && args[index + 1] == 0); index++)
{
if (args[index] == 0)
{
args[index] = ' ';
}
}
if (!strncmp(args, pname, strlen(pname)))
{
pcounter++;
}
}
return pcounter;
}
Try using getprocs(). I find it works better than shelling out with /proc or ps.
I gave an example here: Need help in getting the process name based on the pid in aix

mpg123 library with Qt

I want add support for playback of mp3 file in my Qt app for embedded linux.
I'm not able to use phonon in Qt. After adding QT += phonon in .pro file it gives me the following error during compilation :
/usr/lib/gcc/i486-linux-gnu/4.4.1/../../../../lib/libphonon.so: undefined reference to `QWidget::x11Event(_XEvent*)'
/usr/lib/gcc/i486-linux-gnu/4.4.1/../../../../lib/libphonon.so: undefined reference to `QDataStream::QDataStream(QByteArray*, int)'
collect2: ld returned 1 exit status
So now i'm thinking of using the mpg123 lib for decoding mp3 files.
I need help integrating the library in Qt. I've never used a pure c++ library in Qt before so i don't have much idea on how to integrate it.
Hey all !! Finally I figured it out !!
int MP3Player::Init(const char *pFileName)
{
mpg123_init();
m_mpgHandle = mpg123_new(0, 0);
if(mpg123_open(m_mpgHandle, pFileName) != MPG123_OK)
{
qFatal("Cannot open %s: %s", pFileName, mpg123_strerror(m_mpgHandle));
return 0;
}
}
int MP3Player::Play()
{
unsigned char *audio;
int mc;
size_t bytes;
qWarning("play_frame");
static unsigned char* arr = 0;
/* The first call will not decode anything but return MPG123_NEW_FORMAT! */
mc = mpg123_decode_frame(m_mpgHandle, &m_framenum, &audio, &bytes);
if(bytes)
{
/* Normal flushing of data, includes buffer decoding. */
/*This function is my already implemented audio class which uses ALSA to output decoded audio to Sound Card*/
if (m_audioPlayer.Play(arr,bytes) < (int)bytes)
{
qFatal("Deep trouble! Cannot flush to my output anymore!");
}
}
/* Special actions and errors. */
if(mc != MPG123_OK)
{
if(mc == MPG123_ERR)
{
qFatal("...in decoding next frame: %s", mpg123_strerror(m_mpgHandle));
return CSoundDecoder::EOFStream;
}
if(mc == MPG123_DONE)
{
return CSoundDecoder::EOFStream;
}
if(mc == MPG123_NO_SPACE)
{
qFatal("I have not enough output space? I didn't plan for this.");
return CSoundDecoder::EOFStream;
}
if(mc == MPG123_NEW_FORMAT)
{
long iFrameRate;
int encoding;
mpg123_getformat(m_mpgHandle, &iFrameRate, &m_iChannels, &encoding);
m_iBytesPerChannel = mpg123_encsize(encoding);
if (m_iBytesPerChannel == 0)
qFatal("bytes per channel is 0 !!");
m_audioPlayer.Init(m_iChannels , iFrameRate , m_iBytesPerChannel);
}
}
}
In order to get mpg123 working with your QT project you try following steps:
1.download and install mpg123: from the folder where you extracted it to (e.g /home/mpg123-1.13.0/) run ./configure and then "sudo make install"
2.if there are no errors put this line to your *.pro file
LIBS += /usr/local/lib/libmpg123.so
3.then code below should run fine for you:
#include "mpg123.h"
#include <QDebug>
void MainWindow::on_pushButton_2_clicked()
{
const char **decoders = mpg123_decoders();
while (*decoders != NULL)
{
qDebug() << *decoders;
decoders++;
}
}
alternatively you can call mpg123 via system call:
system("mpg123 /home/test.mp3");
hope this helps, regards

Resources