I'm testing Sqlite C Api for multiprocess concurrency and I'm hitting a bug that I don't understand.
The test is simple: 1 database, 1 table of 1000 rows with 2 columns. Column 1 are uuids, column 2 is a status set to 0. I'm launching then 2 workers that concurrently select 2 uuids at status 0 and update them to 1 until no uuids are left to be updated.
Here the code in the main loop of the worker:
std::vector<std::string> uuids;
res = sqlite3_exec(database,
"BEGIN IMMEDIATE TRANSACTION;", nullptr, nullptr, nullptr);
if (res != SQLITE_OK)
{
Log("Worker: Failed to begin transaction, wait for other worker to finish");
std::this_thread::sleep_for(std::chrono::milliseconds(5));
continue;
}
Log("Worker: Begin Transaction");
// SELECT UUIDS
std::string selectString = "SELECT " + m_uuidStr + " FROM " + m_tableStr + " WHERE " + m_statusStr + "=" + std::to_string(0) + " LIMIT " + std::to_string(2);
// Log("Worker: SelectString = " + selectString);
auto selectStatement = selectString.c_str();
res = sqlite3_exec(database,
selectStatement,
fill_uuids,
&uuids, nullptr);
if (res != SQLITE_OK)
{
Log("Worker: Failed to select uuids");
return 3;
}
if (uuids.empty())
{
sqlite3_exec(database, "ROLLBACK;", nullptr, nullptr, nullptr);
Log("Worker: nothing more to update");
break;
}
// UPDATE UUIDS
std::stringstream ss;
ss << '\"' << uuids[0] << '\"';
for (int i = 1; i < uuids.size(); ++i)
ss << ", \"" << uuids[i] << '\"';
Log("Worker: UUIDs selected = " + ss.str());
std::string updateString = "UPDATE " + m_tableStr
+ " SET " + m_statusStr + "=" + std::to_string(1)
+ " WHERE (" + m_uuidStr + " in (" + ss.str() + ")) AND (" + m_statusStr + "=" + std::to_string(0) + ");";
auto updateStatement = updateString.c_str();
// BIG CRASH HERE WITH IMMEDIATE TRANSACTION ///
res = sqlite3_exec(database,
updateStatement,
nullptr, nullptr, nullptr);
if (res != SQLITE_OK)
{
Log("Worker: Failed to update uuids, err = " + std::string(err));
return 4;
}
// COMMIT CHANGES
res = sqlite3_exec(database,
"COMMIT;", nullptr, nullptr, nullptr);
if (res != SQLITE_OK)
{
Log("Worker: Failed to commit, err = " + std::string(err));
return 5;
}
Log("Worker: Transaction ended");
When using immediate transactions, one of the worker crashes with a negative error code (-1073741819 but I doubt it helps) when executing the UPDATE statement. When using exclusive transactions, I don't have any crash. The version with immediate transactions works with a unique worker. The database is open using the flags SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, Sqlite is compiled with option SQLITE_THREADSAFE=1.
There are no other process accessing the database while the workers are running. In that respect I don't understand why immediate transactions causes a crash and exclusive transactions do not, as the behavior should be similar (since no thread/process is creating reading statement during the worker execution).
I'm very new to Sqlite so maybe there's an obvious mistake lying in my code. Any help is appreciated.
Related
my problem is the following:
I have a UI created in Visual Studio with C++/CLI.
Now I have a button event that creates a process which runs a C++ Console Application with Parameters.
This works fine, the console application gets executed with parameters with no errors and im also seeing the console window with the right output.
But I still get an exception:
StandardOut has not been redirected or the process hasn't started yet.
I think this has something to do with the Streamreader and some DataReceivedEvents but I have no clue.
This is my code:
try {
String^ parameters = tbLoadXML->Text + " " + tbNumberAnts->Text + " " + tbIteration->Text + " " + tbReduction->Text + " " + tbPheromoneDeposit->Text + " " + tbPheromoneReduction->Text + " " + tbAlpha->Text + " " + tbBeta->Text + " " + reduce.ToString() + " " + algorithm.ToString() + " " + probabilityalgorithm.ToString() + " " + numbercities.ToString();
Process^ process = gcnew Process();
process->StartInfo->UseShellExecute = false;
process->StartInfo->CreateNoWindow = true;
process->StartInfo->RedirectStandardOutput = true;
process->Start("TSPACO.exe", parameters);
StreamReader^ reader = process->StandardOutput;
String^ output = reader->ReadToEnd();
textBox1->Text += output;
process->WaitForExit();
process->Close();
}
catch (Exception^ ex) {
MessageBox::Show(ex->Message);
}
I also tried a while-loop:
while (!process->StandardOutput->EndOfStream)
{
textBox1->Text += process->StandardOutput->ReadLine();
}
But nothing really works - I have no idea why.
So I figured out a solution for my problem - I have no clue why this works but I think the trick is to create a ProcessStartInfo-Object with all the needed properties and then just use the default method of Process::Start without any parameters:
ProcessStartInfo^ startInfo = gcnew ProcessStartInfo();
startInfo->FileName = "TSPACO.exe";
startInfo->Arguments = parameters;
startInfo->UseShellExecute = false;
startInfo->CreateNoWindow = true;
startInfo->RedirectStandardOutput = true;
Process^ process = gcnew Process();
process->StartInfo = startInfo;
process->Start();
StreamReader^ reader = process->StandardOutput;
String^ output = reader->ReadToEnd();
textBox1->Text += output;
process->WaitForExit();
process->Close();
At runtime in loop by twice, crashed by Bad_Access in while calling PrepareStatement by requesting query.
So I checked the all ResultSet memory and release it all but there is no idea
any idea solve this problem?
Description about member value;
PrepareStatement* pstmt;
ResultSet* res;
and the full code
bool laskdjlaskdj12::RaspBerry::grup_pw_chk(const Json::Value j){
//check the id and group is valid
try{
string group = j["Group"].asString();
const char* id = j["id"].asCString();
string grp;
string pub;
BIO* tmp; //BIO structor to change the RSA* structor
if(group == "NULL"){
group = "";
}
//request the Client is existence
pstmt = con->prepareStatement("SELECT * FROM `raspberry_cli` WHERE `Cli_id` = (?) AND `Cli_Group` = (?)");
pstmt->setString(1, id);
pstmt->setString(2, group);
res = pstmt->executeQuery();
//if query reply is NULL
if(res->next() == false){
std::cout<<"[INFO] : There is no query about raspberry_cli"<<std::endl;
return false;
}
//if query Reply
grp = res->getString("Cli_Group");
pub = res->getString("Cli_Pub");
//if There is no group in Raspberry_pi
if(ras.grp.compare(grp) == false){
//sql의 버퍼 flush
this->SQL_Flush();
return false;
}
// if the group is equal
else{
//save the client information
acc_cli.S_id = id;
acc_cli.S_Group = grp;
//save the client public_key
tmp = BIO_new_mem_buf(pub.c_str(), -1);
acc_cli.Pub_key = PEM_read_bio_RSA_PUBKEY(tmp, NULL, 0, NULL);
//if public key is not wright in section
if(acc_cli.Pub_key == NULL){
return false;
}
delete[] tmp;
//flush SQL_flush = delete ResultSet
this->SQL_Flush();
return true;
}
}catch(sql::SQLException& e){
std::cout << "# ERR: SQLException in " << __FILE__;
std::cout << "(" << __FUNCTION__ << ") on line "<< __LINE__ << std::endl;
std::cout << "# ERR: " << e.what();
std::cout << " (MySQL error code: " << e.getErrorCode();
std::cout << ", SQLState: " << e.getSQLState() << " )" << std::endl;
return false;
}
The Detal StackTrace is this
* thread #1: tid = 0x2778b, 0x00000001000e29c3 RaspBerry`list_add(root=0x0000000100903e38, element=0x00000001009043e8) + 19 at list.c:33, queue = 'com.apple.main-thread', stop reason = EXC_BAD_ACCESS (code=EXC_I386_GPFLT)
frame #0: 0x00000001000e29c3 RaspBerry`list_add(root=0x0000000100903e38, element=0x00000001009043e8) + 19 at list.c:33 [opt]
frame #1: 0x00000001000de27f RaspBerry`mysql_stmt_init(mysql=0x0000000102802220) + 143 at libmysql.c:1568 [opt]
frame #2: 0x0000000100168f41 RaspBerry`sql::mysql::NativeAPI::MySQL_NativeConnectionWrapper::stmt_init(this=0x0000000100a00580) + 33 at mysql_native_connection_wrapper.cpp:421 [opt]
frame #3: 0x000000010011c415 RaspBerry`sql::mysql::MySQL_Connection::prepareStatement(this=0x0000000100a00550, sql=0x00007fff5fbfe138) + 53 at mysql_connection.cpp:1137 [opt]
* frame #4: 0x0000000100073071 RaspBerry`laskdjlaskdj12::RaspBerry::grup_pw_chk(this=0x00007fff5fbfeba0, j=const Json::Value # 0x00007fff5fbfea30) + 881 at RaspBerry.cpp:438
frame #5: 0x000000010007c222 RaspBerry`main(argc=1, argv=0x00007fff5fbff7f8) + 3474 at main.cpp:81
frame #6: 0x00007fffced8b255 libdyld.dylib`start + 1
frame #7: 0x00007fffced8b255 libdyld.dylib`start + 1
Find the answer
pstmt = con->prepareStatement("SELECT * FROM `raspberry_cli` WHERE `Cli_id` = (?) AND `Cli_Group` = (?)");
pstmt->setString(1, id);
pstmt->setString(2, group);
res = pstmt->executeQuery();
[delete pstmt;] //===================> This part is missing
in this section i didn't delete pstmt(sql::preparestatement*)
so the sql have send that malloc error.
So Be careful about allocating memory.
I have a simple c++ program running on MSVC using an ODBC connection to execute a stored procedure.
When running it on the main thread it works fine, but when I attempt to multithread the same class I am getting this error:
(Process ID XX) was deadlocked on lock resource....
As mentioned above I can run it fine on the main thread and also on two threads with no issues.
This is how I spawn, use and join the threads:
historical apple("AAPL");
historical google("GOOG");
historical yahoo("YHOO");
historical samsung("005930.KS");
historical ibm("IBM");
auto t1 = thread([&] {apple.go(); });
auto t2 = thread([&] {google.go(); });
auto t3 = thread([&] {yahoo.go(); });
auto t4 = thread([&] {samsung.go(); });
auto t5 = thread([&] {ibm.go(); });
//writer(his.getHistorical()); // this writes to the file test.csv
t1.join();
t2.join();
t3.join();
t4.join();
t5.join();
I know it looks like shit, but I'm basically trying to see how hard I can push the software before making it look nice and more formatted.
This is how I call the stored procedure:
#include "stdafx.h"
#include "database_con.h"
////////////////////////////////////////////////////////////////////////
// Show errors from the SQLHANDLE
void database_con::show_error(unsigned int handletype, const SQLHANDLE& handle)
{
SQLWCHAR sqlstate[1024];
SQLWCHAR message[1024];
if (SQL_SUCCESS == SQLGetDiagRec(handletype, handle, 1, sqlstate, NULL, message, 1024, NULL))
wcout << "Message: " << message << "\nSQLSTATE: " << sqlstate << endl;
}
////////////////////////////////////////////////////////////////////////
// Builds the stored procedure query.
std::wstring database_con::buildQuery(vector<std::wstring> input, string symbol)
{
std::wstringstream builder;
builder << L"EXEC sp_addHistorical " << "#Symbol='" << L"" << StringToWString(symbol) << "'," <<
"#Date='" << (wstring)L"" << input.at(0) << "'," <<
"#Open=" << (wstring)L"" << input.at(1) << "," <<
"#Close=" << (wstring)L"" << input.at(2) << "," <<
"#MaxPrice=" << (wstring)L"" << input.at(3) << "," <<
"#MinPrice=" << (wstring)L"" << input.at(4) << "," <<
"#Volume=" << (wstring)L"" << input.at(5) << ";";
return builder.str();
}
////////////////////////////////////////////////////////////////////////
// Adds a substring of the string before the delimiter to a vector<wstring> that is returned.
std::vector<wstring> database_con::parseData(wstring line, char delim) {
size_t pos = 0;
std::vector<std::wstring> vOut;
while ((pos = line.find(delim)) != std::string::npos) {
vOut.push_back(line.substr(0, pos));
line.erase(0, pos + 1);
}
vOut.push_back(line.substr(0, pos));
return vOut;
}
////////////////////////////////////////////////////////////////////////
// Converts a std::string to a std::wstring
std::wstring database_con::StringToWString(const std::string& s)
{
std::wstring temp(s.length(), L' ');
std::copy(s.begin(), s.end(), temp.begin());
return temp;
}
void database_con::historicalLooper(string historical) {
}
////////////////////////////////////////////////////////////////////////
// Constructs a database connector object with the historical data and its symbol
database_con::database_con(std::string historical, string symbol){
/*
Set up the handlers
*/
/* Allocate an environment handle */
SQLAllocHandle(SQL_HANDLE_ENV, SQL_NULL_HANDLE, &env);
/* We want ODBC 3 support */
SQLSetEnvAttr(env, SQL_ATTR_ODBC_VERSION, (void *)SQL_OV_ODBC3, 0);
/* Allocate a connection handle */
SQLAllocHandle(SQL_HANDLE_DBC, env, &dbc);
/* Connect to the DSN */
SQLDriverConnectW(dbc, NULL, L"DRIVER={SQL Server};SERVER=ERA-PC-STUART\\JBK_DB;DATABASE=master;UID=geo;PWD=kalle123;", SQL_NTS, NULL, 0, NULL, SQL_DRIVER_COMPLETE);
/* Check for success */
if (SQL_SUCCESS != SQLAllocHandle(SQL_HANDLE_STMT, dbc, &stmt))
{
show_error(SQL_HANDLE_DBC, dbc);
std::cout << "Failed to connect";
}
_mSymbol = symbol;
std::wstringstream stream(StringToWString(historical));
std::wstring line;
int row = 0;
while (std::getline(stream, line)) {
if (row > 0) {
vector<wstring> vHistorical = parseData(L"" + line, ',');
std::wstring SQL = buildQuery(vHistorical, _mSymbol);
if (SQL_SUCCESS != SQLExecDirectW(stmt, const_cast<SQLWCHAR*>(SQL.c_str()), SQL_NTS)) {
std::cout << "Execute error " << std::endl;
show_error(SQL_HANDLE_STMT, stmt);
std::wcout << L"Unsuccessful Query: " << SQL << std::endl;
}
// Close Cursor before next iteration starts:
SQLRETURN closeCursRet = SQLFreeStmt(stmt, SQL_CLOSE);
if (!SQL_SUCCEEDED(closeCursRet))
{
show_error(SQL_HANDLE_STMT, stmt);
// maybe add some handling for the case that closing failed.
}
}
row++;
}
std::cout << "Query " << _mSymbol << " ready" << std::endl;
}
database_con::~database_con() {
}
And finally this is the stored procedure:
GO
CREATE PROCEDURE sp_addHistorical
#Symbol nchar(10),#Date datetime,
#Open decimal(12,2),#Close decimal(12,2),#MinPrice decimal(12,2),
#MaxPrice decimal(12,2),#Volume int
AS
SET TRANSACTION ISOLATION LEVEL SERIALIZABLE
BEGIN TRANSACTION
MERGE HistoricalStock WITH (UPDLOCK) AS myTarget
USING (SELECT #Symbol AS Symbol,
#Date AS Date, #Open AS [Open], #Close AS [Close],
#MinPrice AS MinPrice, #MaxPrice AS MaxPrice,#Volume AS Volume) AS mySource
ON mySource.Symbol = myTarget.Symbol AND mySource.Date = myTarget.Date
WHEN MATCHED
THEN UPDATE
SET [Open] = mySource.[Open], [Close] = mySource.[Close],
MinPrice = mySource.MinPrice, MaxPrice = mySource.MaxPrice, Volume = mySource.Volume
WHEN NOT MATCHED
THEN
INSERT(Symbol,Date,[Open],[Close],MinPrice,MaxPrice,Volume)
VALUES(#Symbol,#Date,#Open,#Close,#MinPrice,#MaxPrice,#Volume);
COMMIT
GO
Any sort of help or advice on either how it's all structured or pointers to good examples regarding the issue I'm facing would be greatly appreciated.
I'm somewhat new to SQL server (more importantly to stored procedures) and can't really figure out what is wrong, even though I understand what a deadlock is.
Thanks!
Using sqlite3_exec on an embedded system. I require the write to be done asap, so if the power is lost, the data is saved. However I've found the machine must be on for ~10 seconds before the data is fully written! Is there a setting to try and speed it up?
Using journal mode DELETE ("PRAGMA journal_mode=DELETE;), and Synchronous mode FULL ("PRAGMA synchronous=FULL;").
I've tried sending "BEGIN;" and "COMMIT;" before and after the UPDATE, respectively. No change.
std::ostringstream oss;
std::ostringstream oss_err;
oss << "UPDATE " << sTableName << " SET " << pon_rstc_db[LookupParamIndex(param)].column_name << "='" << sValue << "';";
if (!isDatabaseOpen())
OpenDatabase(sDatabaseName);
if (isDatabaseOpen())
{
char* db_err = 0;
int rc;
rc = sqlite3_exec(myDatabase, oss.str().c_str(), NULL, 0, &db_err);
if (rc == SQLITE_OK)
{
bResponse = true;
}
else
{
oss_err << db_err;
oss_err << "Update failed for " << LookUpColumnName(param) << ". Data not saved.";
DisplayError(oss_err.str());
bResponse = false;
}
sqlite3_free(db_err);
db_err = 0;
}
else
{
DisplayError("executing statement while database closed");
}
I'm writing a small utility which is supposed to launch several commands in parallel using system() and wait for their results for logging purposes. However, even though I'm calling system() on different threads, by looking at my Activity Monitor I only see one instance of each command at a time. It looks like system is internally synchronized on a mutex, and only one execution is allowed at each time, however this looks like a huge limitation, can someone confirm this behavior? Do you have any ideas on how to solve it?
Update by looking at the threads execution flow, it looks like they're effectively synchronizing on a mutex. Is there an alternative system() which doesn't do that?
I should mention I'm using C++11 (w/ clang and libc++) on Mac OS 10.7.5.
Update code is:
void Batch::run()
{
done.clear();
generator->resetGeneration();
while(generator->hasMoreParameters())
{
// Lock for accessing active
unique_lock<mutex> lock(q_mutex, adopt_lock);
// If we've less experiments than threads
if (active.size() < threads)
{
Configuration conf = generator->generateParameters();
Experiment e(executable, conf);
thread t(&Experiment::run, e, reference_wrapper<Batch>(*this));
thread::id id = t.get_id();
active.insert(id);
t.detach();
}
// Condition variable
q_control.wait(lock, [this] { return active.size() < threads; } );
}
}
void Batch::experimentFinished(std::thread::id pos)
{
unique_lock<mutex> lock(q_mutex, adopt_lock);
active.erase(pos);
lock.unlock();
q_control.notify_all();
}
void Experiment::run(Batch& caller)
{
// Generate run command
stringstream run_command;
run_command << executable + " ";
ParameterExpression::printCommandLine(run_command, config);
if (system(run_command.str().c_str()))
stats["success"] = "true";
else
stats["success"] = "false";
caller.experimentFinished(this_thread::get_id());
}
Just to be clear: the spawning and handling of threads works fine and does what it needs to do, but it looks like you can have just one instance of system() running at a time.
Thanks
POSIX has this to say about system(3):
Using the system() function in more than one thread in a process or when the SIGCHLD signal is being manipulated by more than one thread in a process may produce unexpected results.
Due to the way that SIGCHLD must be blocked during the execution, running system calls concurrently doesn't really work. If you want multiple threads to run external tasks, you'll need to write a bit more code (handling fork/exec/wait yourself).
To whoever comes later, popen did the trick, as it doesn't internally keep a mutex. The code to make it work is
FILE* proc;
char buff[1024];
// Keep track of the success or insuccess of execution
if (!(proc = popen(run_command.str().c_str(), "r")))
stats["success"] = "false";
else
stats["success"] = "true";
// Exhaust output
while(fgets(buff, sizeof(buff), proc) != nullptr);
pclose(proc);
In case this helps, I wrote some fork/exec/wait code in C++ a while back. It captures output into a std::string.
As #Mat points out, fork, exec, and wait aren't really designed to be uses in a multi-threaded process.
So this is more useful if multi-process can be a substitute for multi-threaded in your application.
bool Utility::execAndRedirect(std::string command, std::vector<std::string> args, std::string& output, int& status)
{
int error;
int pipefd[2];
int localStatus;
if (pipe(pipefd) == -1)
{
error = errno;
cerr << "Executing command '" << command << "' failed: " << strerror(error) << endl;
return false;
}
pid_t pid = fork();
if (pid == 0)
{
char** argsC;
argsC = new char*[args.size() + 2];
argsC[0] = new char[command.size() + 1];
strncpy(argsC[0], command.c_str(), command.size());
argsC[0][command.size()] = '\0';
for (size_t count = 0; count < args.size(); count++)
{
argsC[count + 1] = new char[args[count].size() + 1];
strncpy(argsC[count + 1], args[count].c_str(), args[count].size());
argsC[count + 1][args[count].size()] = '\0';
}
argsC[args.size() + 1] = NULL;
close(pipefd[0]);
if (dup2(pipefd[1], STDOUT_FILENO) == -1)
{
error = errno;
cerr << "Executing command '" << command << "' failed: " << strerror(error) << endl;
exit(1);
}
if (dup2(pipefd[1], STDERR_FILENO) == -1)
{
error = errno;
cerr << "Executing command '" << command << "' failed: " << strerror(error) << endl;
exit(1);
}
close(pipefd[1]);
if (execvp(command.c_str(), argsC) == -1)
{
error = errno;
cerr << "Executing command '" << command << "' failed: " << strerror(error) << endl;
exit(1);
}
}
else if (pid > 0)
{
size_t BUFFER_SIZE = 1024;
char buffer[BUFFER_SIZE + 1];
close(pipefd[1]);
ostringstream oss;
ssize_t num_b;
while ((num_b = read(pipefd[0], buffer, BUFFER_SIZE)) != 0)
{
buffer[num_b] = '\0';
oss << buffer;
}
output = oss.str();
waitpid(pid, &localStatus, 0);
close(pipefd[0]);
}
else
{
error = errno;
cerr << "Executing command '" << command << "' failed: " << strerror(error) << endl;
return false;
}
if(WIFEXITED(localStatus))
{
status = WEXITSTATUS(localStatus);
//DateTime current = DateTime::now(); //this is a custom class
if(status == 0)
{
return true;
}
else
{
return false;
}
}
else
{
error = errno;
cerr << "Executing command '" << command << "' failed: child didn't terminate normally" << endl;
return false;
}
}