I googled a lot and couldn't find any solution. I'm currently writing myself an
app in Codegear 2007 C++. I'm writing this app for my little kittens, its kinda' a dairy
but I call it KittyBook.
So i have two tables(sorry didn't understand how to CodeBlock) :
Kitten Info
Kitten Data.
KittenInfo Stores their names and their ID ( primary key ), their gender and their birth. This one works.
The other one should store a Blob. So after trying so many ways. It won't be stored in the table, not even the other data if I do the Query with normal insert but excluding the blob table.
So, I dunno what I'm doing wrong BUT I love SQLite so far. No turning back then eh?
The function is :
void CDatabase::InsertKittenData(int Kitten_ID, int kittenDay, bool DayOrWeek,
char * kitten_weight, char * Kitten_Comment, string PhotoFile) {
unsigned char * blob;
ifstream::pos_type size;
int size2 = 0;
if (FileExists(PhotoFile.c_str())) {
ifstream file(PhotoFile.c_str(), ios::in | ios::binary | ios::ate);
if (file.is_open()) {
size = file.tellg();
blob = new char[size];
file.seekg(0, ios::beg);
file.read(blob, size);
file.close();
}
}
else {
blob = NULL;
}
sqlite3 *dbp;
sqlite3_stmt *ppStmt;
// NULL = primary key autoinc.
char * Sql = "INSERT INTO KittenData VALUES ( NULL, ? , ? ,? , ? , ? , ?);";
int rc = sqlite3_open("KittyBook.db", &dbp);
if (rc)
return;
if (sqlite3_prepare_v2(dbp, Sql, -1, &ppStmt, NULL) != SQLITE_OK) {
return;
}
if (ppStmt) {
if (sqlite3_bind_int(ppStmt, 1, Kitten_ID) != SQLITE_OK)
return;
if (sqlite3_bind_int(ppStmt, 2, kittenDay) != SQLITE_OK)
return;
if (sqlite3_bind_int(ppStmt, 3, DayOrWeek) != SQLITE_OK)
return;
if (sqlite3_bind_text(ppStmt, 4, // Index of wildcard
kitten_weight, strlen(kitten_weight), // length of text
SQLITE_STATIC) != SQLITE_OK)
return;
if (sqlite3_bind_text(ppStmt, 5, // Index of wildcard
Kitten_Comment, strlen(Kitten_Comment), // length of text
SQLITE_STATIC) != SQLITE_OK)
return;
if (sqlite3_bind_blob(ppStmt, 6, blob, size2, SQLITE_TRANSIENT)
!= SQLITE_OK)
return;
if (sqlite3_step(ppStmt) != SQLITE_DONE)
return;
}
sqlite3_finalize(ppStmt);
sqlite3_exec(dbp, "COMMIT", NULL, NULL, NULL);
sqlite3_close(dbp);
}
You declare and initialize size2 as zero:
int size2 = 0;
then the next use of size2 is when you bind the blob:
sqlite3_bind_blob(ppStmt, 6, blob, size2, SQLITE_TRANSIENT)
From the fine manual:
In those routines that have a fourth argument, its value is the number of bytes in the parameter. To be clear: the value is the number of bytes in the value, not the number of characters.
and the fourth argument in this case is size2 and that's zero. The result is that you're telling SQLite to bind a zero-length blob and then you're wondering why nothing gets stored. Well, nothing gets stored because you're asking SQLite to store zero bytes and it is only doing what it is told.
Perhaps you want to use size instead of size2.
mu is too short, gave me the idea to do the Function step by step and increasing column
by column. Dunno what i did wrong, but its working now.
Cheers mu is too short :)
Related
I have setup the SQLite3 C Interface on a ESP32 (Compiler: Arduino IDE) with a CPU frequency of 240Mhz.
All the functions working properly, except the sqlite3_step() function.
This command is "incredible" slow.
My Function for insert 3 data-pairs in a vector which has a size of 1k (vector<struct DATA>):
struct DATA{
pair<int, byte> id;
pair<int, byte> value;
pair<string, byte> s_timestamp;
};
bool ESPDB::insert_vector(const vector<struct DATA> &data_buffer) {
char *errmsg;
if(sqlite3_open(("/sdcard/" + database_).c_str(), &db) != SQLITE_OK){
return 0;
}
sqlite3_stmt *stmt;
sqlite3_prepare_v2(db, ("INSERT INTO " + table_ + " " + columns_ + " VALUES " +
placeholder_).c_str(), -1, &stmt, 0);
sqlite3_exec(db, "BEGIN TRANSACTION", NULL, NULL, &errmsg);
for (const struct DATA &data_pair : data_buffer){
if(data_pair.id.first !=0 && data_pair.id.second !=0){
sqlite3_bind_int(stmt, data_pair.id.second, data_pair.id.first);
}
if(data_pair.value.first !=0 && data_pair.value.second !=0){
sqlite3_bind_int(stmt, data_pair.value.second, data_pair.value.first);
}
if(data_pair.s_timestamp.first.length() > 0 && data_pair.s_timestamp.second !=0){
sqlite3_bind_text(stmt, data_pair.s_timestamp.second, data_pair.s_timestamp.first.c_str(),
data_pair.s_timestamp.first.length(), SQLITE_STATIC);
}
sqlite3_step(stmt);
sqlite3_reset(stmt);
}
sqlite3_exec(db, "END TRANSACTION", NULL, NULL, &errmsg);
sqlite3_finalize(stmt);
sqlite3_close(db);
return 1;
}
The average result of the performance test was around 3000ms for 1k vector - (a few times, its excecuting it in less than 300ms).
The step command itself taking around 16-25ms for each row insert.
The strange thing is, that the sqlite3_step() command speeds up after around 100 inserts to finally 0ms (<1ms) for each row.
The sequence of the code is following:
-open database
-drop table
-create table if not exists
-single insert function (single DATA struct)
-select query
-insert the vector (size = 1000)
-select query in JSON
in ALL functions which handling the database connection, i open and close the database by following code:
Drop-Table Example:
bool ESPDB::drop_table(const string &table){
if(sqlite3_open(("/sdcard/" + database_).c_str(), &db) != SQLITE_OK){
return 0;
}
sql_stmt_(("DROP TABLE IF EXISTS " + table).c_str());
sqlite3_close(db);
return 1;
}
Update:
opened database only once
ensured 240MHz CPU speed
Here a chart for visualisation:
(the issue is, lateron i have a vector size of 250 due to small available heap size in my code - this makes it really slow - as you can see, the first ~130 taking around 15ms. after this its operating really fast)
chart3:
time of insert a vector with size 100 in a while loop - total excecutions 85~]
I am writing a function to prepare a sql query and execute it against a sqlite db. I am using a query to insert values in to a table, the query looks like this
"insert into files values (?, ?, ?, ?, ?, 0)";
A row is inserted but all values for text fields are empty except the last field which is 0. The first 5 fields are of type TEXT
// If I hard code a value for value.data() then the row is inserted correctly with my hardcoded data, the exact line is below
status = sqlite3_bind_text(ppStmt, index, /* if I hardcode it works*/ value.data(), -1, SQLITE_STATIC);
The full function is shown below.The list is created on stack by the calller, I am not sure why it is not replacing question marks with my string args, hardcoded ones work though.. no error codes are returned
//db is already open before I call this
void MediaCache::prepareAndExecuteQuery(string query, list<string> args)
{
sqlite3_stmt *ppStmt = 0;
const char **pzTail = 0;
int status = 0;
if( sqlite3_prepare_v2(db, query.data(), query.length(), &ppStmt, pzTail) != SQLITE_OK )
{
string error = sqlite3_errmsg(db);
//throw an exception
}
if(ppStmt)
{
list<string>::iterator current = args.begin();
int index = 1;
for(current = args.begin() ; current != args.end(); current++)
{
string value = *current;
status = sqlite3_bind_text(ppStmt, index, value.data(), -1, SQLITE_STATIC);
if(status != SQLITE_OK)
{
//log error;
}
index++;
}
status = sqlite3_step(ppStmt);
status = sqlite3_finalize(ppStmt);
//sqlite3_exec(db, "COMMIT", NULL, NULL, NULL);
}
else
{
//ppStmt is null
//throw an exception
}
}
I suspect the problem is here:
string value = *current;
status = sqlite3_bind_text(ppStmt, index, value.data(), -1, SQLITE_STATIC);
In this case value goes out of scope before the execution of the statement takes place. So the pointers given to sqlite3_bind_text are no longer valid. To "fix" that, you could use SQLITE_TRANSIENT, which would force the library to make its own copy of the data before returning.
Also, if you are not using C++11, I don't believe string::data() is guaranteed to be NULL terminated in which case the -1 parameter could be incorrect. It should maybe be value.length() instead.
I use BLOB to insert an object to a SQLite databse. After the insertion, I can get the data with "SELECT" sentence and the data is correct, although the row of "TASK_HEAD" is "Empty" when browsing the database with "SQLite Database Browser".
However, if I destroy the object which has just been inserted, I can't get the correct data anymore, with the pointer "pHead" points to an address where the content of its "id" member is "铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪铪UG?" when read in VS2008 in debug mode.
Here is an example:
// user-defined data type
typedef std::string TASK_ID;
struct TASK_HEAD
{
TASK_ID id;
std::string userData;
int Size()
{
return (id.size() + userData.size()) * sizeof(TCHAR);
}
};
// when TEST_INSIDE is defined, pHead is invalid; but if undef it, I can get the "head" I just inserted
// and if the blob data is a string (when USING_STRING is defined), I can get the string inserted into the db even though the "test" string has been destroyed
void CDBWriter::WriteTestData()
{
// open db
sqlite3* db = NULL;
int nRet = sqlite3_open(DATABASE_NAME.c_str(), &db);
if (nRet != SQLITE_OK)
{
return;
}
if (db != NULL)
{
// create a table
std::string cmdCreate("CREATE TABLE IF NOT EXISTS testTable (id TEXT NOT NULL, TASK_HEAD BLOB, PRIMARY KEY(id));");
char* errMsg = NULL;
nRet = sqlite3_exec( db , cmdCreate.c_str() , 0 , 0 , &errMsg );
if( errMsg != NULL )
{
sqlite3_free( errMsg );
errMsg = NULL;
return;
}
//#define USING_STRING
#define TEST_INSIDE
#ifndef TEST_INSIDE
TASK_HEAD head;
#endif // TEST_INSIDE
// insert blob data
const TASK_ID newID(NewGUID()); // NewGUID returns string like this: "5811307F-7AA7-4C44-831F-774FC5832627"
string query = "INSERT OR REPLACE INTO testTable (id, TASK_HEAD) VALUES ('";
query += newID;
query += "', ?1);";
sqlite3_stmt* res = NULL;
nRet = sqlite3_prepare_v2(db, query.c_str(), query.length(), &res, 0);
{
#ifdef TEST_INSIDE
TASK_HEAD head;
#endif // TEST_INSIDE
head.id = newID;
#ifdef USING_STRING
std::string test("ewsjoafijdoaijeofsafsd");
nRet = sqlite3_bind_blob (res, 1, test.c_str(), test.size(), SQLITE_TRANSIENT);
#else
int nsizeHead = sizeof(head);
int nSizeHeadSt = sizeof(TASK_HEAD);
int sizeString = sizeof(std::string);
size_t nLen = newID.size();
//nRet = sqlite3_bind_blob (res, 1, &head, sizeof(head), SQLITE_TRANSIENT);
nRet = sqlite3_bind_blob (res, 1, &head, head.Size(), SQLITE_TRANSIENT);
#endif // USING_STRING
if (SQLITE_OK == nRet)
{
nRet = sqlite3_step(res);
}
if (nRet != SQLITE_OK && nRet != SQLITE_DONE)
{
return;
}
}
// get all columns in the database
query = "SELECT * FROM testTable;";
nRet = sqlite3_prepare_v2 (db, query.c_str(), query.length(), &res, 0);
if (SQLITE_OK == nRet)
{
while (SQLITE_ROW == sqlite3_step(res))
{
#ifdef USING_STRING
const char* pHead = (const char*)sqlite3_column_blob(res, 1);
#else
const TASK_HEAD *pHead = (const TASK_HEAD*)sqlite3_column_blob(res, 1);
#endif // USING_STRING
continue;
}
}
sqlite3_finalize(res);
sqlite3_close(db);
}
}
At first, I thought it might be the problem of bytes passed to sqlite3_bind_blob, so I get the bytes of the object with a stupid method, as you can see here (the size() function of TASK_HEAD), but that doesn't help.
Then I tried to use SQLITE_STATIC instead of SQLITE_TRANSIENT, still not working.
What's wrong?
Ps: I know it's a bad solution to insert an object to the db, and I just wanna know why I can't read back my data inserted into the db.
The content of userData is likely to be stored on the heap. Even if it's stored inside the std::string (for SSO) it still may use a pointer to itself internally, and so it won't work when you bitwise copy it to another place in memory (what you're doing is equivalent to a memcpy).
However, it doesn't matter why exactly it doesn't work, since it's just undefined behavior. Just don't "insert an object to the db" like this. Either serialize it using some serialization library and then insert it, or use two columns in the table, one for id and one for userData.
I think the problem is at:
nRet = sqlite3_bind_blob (res, 1, &head, head.Size(), SQLITE_TRANSIENT);
You cannot get the address of the TASK_HEAD structure and pass it to sqlite like this. To build a blob you need flat data, nothing with pointers and dynamic buffers like std::string objects.
You need to serialize the TASK_HEAD structure in a buffer before the binding operation.
For instance:
struct TASK_HEAD
{
TASK_ID id;
std::string userData;
std::string Data()
{
return id+userData;
}
int Size()
{
return (id.size() + userData.size()) * sizeof(TCHAR);
}
};
and:
nRet = sqlite3_bind_blob (res, 1, head.Data().c_str(), head.Size(), SQLITE_TRANSIENT);
Please note adding the fields to serialize as shown above is very poor (since this format cannot be unserialized). To deal with blobs, you need to find a good serialization library or format (protocol buffer, message pack, JSON, etc ...) or roll your own.
There is a second issue in your code at:
const TASK_HEAD *pHead = (const TASK_HEAD*)sqlite3_column_blob(res, 1);
This will not work, for a similar reason.
I have the following code for SQLite:
std::vector<std::vector<std::string> > InternalDatabaseManager::query(std::string query)
{
sqlite3_stmt *statement;
std::vector<std::vector<std::string> > results;
if(sqlite3_prepare_v2(internalDbManager, query.c_str(), -1, &statement, 0) == SQLITE_OK)
{
int cols = sqlite3_column_count(statement);
int result = 0;
while(true)
{
result = sqlite3_step(statement);
std::vector<std::string> values;
if(result == SQLITE_ROW)
{
for(int col = 0; col < cols; col++)
{
std::string s;
char *ptr = (char*)sqlite3_column_text(statement, col);
if(ptr) s = ptr;
values.push_back(s);
}
results.push_back(values);
} else
{
break;
}
}
sqlite3_finalize(statement);
}
std::string error = sqlite3_errmsg(internalDbManager);
if(error != "not an error") std::cout << query << " " << error << std::endl;
return results;
}
When I try to pass a query string like:
INSERT INTO CpuUsage (NODE_ID, TIME_ID, CORE_ID, USER, NICE, SYSMODE, IDLE, IOWAIT, IRQ, SOFTIRQ, STEAL, GUEST) VALUES (1, 1, -1, 1014711, 117915, 175551, 5908257, 112996, 2613, 4359, 0, 0); INSERT INTO CpuUsage (NODE_ID, TIME_ID, CORE_ID, USER, NICE, SYSMODE, IDLE, IOWAIT, IRQ, SOFTIRQ, STEAL, GUEST) VALUES (1, 1, 0, 1014711, 117915, 175551, 5908257, 112996, 2613, 4359, 0, 0); INSERT INTO CpuUsage (NODE_ID, TIME_ID, CORE_ID, USER, NICE, SYSMODE, IDLE, IOWAIT, IRQ, SOFTIRQ, STEAL, GUEST) VALUES (1, 1, 1, 1014711, 117915, 175551, 5908257, 112996, 2613, 4359, 0, 0);
It results just inserting the first insert. Using some other tool lite SQLiteStudio it performs ok.
Any ideas to help me, please?
Thanks,
Pedro
EDIT
My query is a std::string.
const char** pzTail;
const char* q = query.c_str();
int result = -1;
do {
result = sqlite3_prepare_v2(internalDbManager, q, -1, &statement, pzTail);
q = *pzTail;
}
while(result == SQLITE_OK);
This gives me Description: cannot convert ‘const char*’ to ‘const char**’ for argument ‘5’ to ‘int sqlite3_prepare_v2(sqlite3*, const char*, int, sqlite3_stmt*, const char*)’
SQLite's prepare_v2 will only create a statement from the first insert in your string. You can think of it as a "pop front" mechanism.
int sqlite3_prepare_v2(
sqlite3 *db, /* Database handle */
const char *zSql, /* SQL statement, UTF-8 encoded */
int nByte, /* Maximum length of zSql in bytes. */
sqlite3_stmt **ppStmt, /* OUT: Statement handle */
const char **pzTail /* OUT: Pointer to unused portion of zSql */
);
From http://www.sqlite.org/c3ref/prepare.html
If pzTail is not NULL then *pzTail is made to point to the first byte
past the end of the first SQL statement in zSql. These routines only
compile the first statement in zSql, so *pzTail is left pointing to
what remains uncompiled.
The pzTail parameter will point to the rest of the inserts, so you can loop through them all until they have all been prepared.
The other option is to only do one insert at a time, which makes the rest of your handling code a little bit simpler usually.
Typically I have seen people do this sort of thing under the impression that they will be evaluated under the same transaction. This is not the case, though. The second one may fail and the first and third will still succeed.
I'm trying to execute a query in PostgreSQL using the following code. It's written in C/C++ and I keep getting the following error when declaring a cursor:
DECLARE CURSOR failed: ERROR: could not determine data type of parameter $1
Searching here and on google, I can't find a solution. Can anyone find where I have made and error and why this is happening?
void searchdb( PGconn *conn, char* name, char* offset )
{
// Will hold the number of field in table
int nFields;
// Start a transaction block
PGresult *res = PQexec(conn, "BEGIN");
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
printf("BEGIN command failed: %s", PQerrorMessage(conn));
PQclear(res);
exit_nicely(conn);
}
// Clear result
PQclear(res);
printf("BEGIN command - OK\n");
//set the values to use
const char *values[3] = {(char*)name, (char*)RESULTS_LIMIT, (char*)offset};
//calculate the lengths of each of the values
int lengths[3] = {strlen((char*)name), sizeof(RESULTS_LIMIT), sizeof(offset)};
//state which parameters are binary
int binary[3] = {0, 0, 1};
res = PQexecParams(conn, "DECLARE emprec CURSOR for SELECT name, id, 'Events' as source FROM events_basic WHERE name LIKE '$1::varchar%' UNION ALL "
" SELECT name, fsq_id, 'Venues' as source FROM venues_cache WHERE name LIKE '$1::varchar%' UNION ALL "
" SELECT name, geo_id, 'Cities' as source FROM static_cities WHERE name LIKE '$1::varchar%' OR FIND_IN_SET('$1::varchar%', alternate_names) != 0 LIMIT $2::int4 OFFSET $3::int4",
3, //number of parameters
NULL, //ignore the Oid field
values, //values to substitute $1 and $2
lengths, //the lengths, in bytes, of each of the parameter values
binary, //whether the values are binary or not
0); //we want the result in text format
// Fetch rows from table
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
printf("DECLARE CURSOR failed: %s", PQerrorMessage(conn));
PQclear(res);
exit_nicely(conn);
}
// Clear result
PQclear(res);
res = PQexec(conn, "FETCH ALL in emprec");
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
printf("FETCH ALL failed");
PQclear(res);
exit_nicely(conn);
}
// Get the field name
nFields = PQnfields(res);
// Prepare the header with table field name
printf("\nFetch record:");
printf("\n********************************************************************\n");
for (int i = 0; i < nFields; i++)
printf("%-30s", PQfname(res, i));
printf("\n********************************************************************\n");
// Next, print out the record for each row
for (int i = 0; i < PQntuples(res); i++)
{
for (int j = 0; j < nFields; j++)
printf("%-30s", PQgetvalue(res, i, j));
printf("\n");
}
PQclear(res);
// Close the emprec
res = PQexec(conn, "CLOSE emprec");
PQclear(res);
// End the transaction
res = PQexec(conn, "END");
// Clear result
PQclear(res);
}
Try replacing
WHERE name LIKE '$1::varchar%'
with
WHERE name LIKE ($1::varchar || '%')
Treat all other occurances of '$1::varchar%' similarly.
More information on string concatenation in the manual.