LLVM Instrumenting C++ lock and unlock functions - llvm

Intent: Given some source code, my intent is to instrument entry/exit of critical sections and associated lock/s.
Approach: I used a function pass which basically iterate over all direct function calls, and check name against my {mutex}.lock() and {mutex}.unlock() mangled names.
Issue: I don't know how to get mutex instance on which lock() is called and then pass it to my hook function(ex. enter_cs(mtx)) as argument.
I looked at .ll code and saw this
call void #_ZNSt3__15mutex4lockEv(%"class.std::__1::mutex"* nonnull dereferenceable(64) #mtx)
I don't know how can I get access to #mtx.
I am new to LLVM so please help. Also if there is better approach for overall problem, please explain!
sample code:
mutex mtx;
void enter_cs(mutex mtx) {
/** some code **/
return;
}
void exit_cs(mutex mtx) {
/** some code **/
return;
}
int main() {
mtx.lock();
/** LLVM inserted call: enter_cs(mtx) **/
// critical section code //
mtx.unlock();
/** LLVM inserted call: exit_cs(mtx) **/
return 0;
}
LLVM Pass:
static StringRef mutexLock = "_ZNSt3__15mutex4lockEv";
bool runOnFunction(Function &F) override {
for(BasicBlock &bb : F) {
for(Instruction &i : bb) {
if(!isa<CallInst>(i)) continue; // should be a call instruction
CallInst *ci = cast<CallInst>(&i);
Function* f = ci->getCalledFunction();
if(!f) continue; // should be a direct call
if(f->getName()==mutexLock) {
errs() << "locked" << ", From: " << F.getName() << "\n";
/** enter_cs({mutex object on which lock is called}) **/
}
}
}
return false;
}

Related

Failure to create a monitor with mutex and condition_variable in C++11

I am trying to create a monitor class to get sum of data[]. In the class, I use a monitor a do the reading/writer working. However, the condition_variable, unique_lock and mutex confuse me. When the data size is not over 56, my code works correctly but with a bigger data size the code fails, and the condition_variable.wait(cond_lock) cannot work when debugging in lldb.
The error: type std::__1::system_error: unique_lock::unlock: not locked:
Operation not permitted
does not help me to understand the problem.
#include <iostream>
#include <thread>
#include <mutex>
#include <condition_variable>
const int datasize = 58;//*****datasize cannot be bigger than 56*****//
int *data = new int[datasize];
void data_init(){
for (int i = 0; i < datasize; ++i) {
data[i] = random() % datasize + 1;
}
int sum = 0;
for (int i = 0; i < datasize; ++i) {
sum += data[i];
}
std::cout << "true answer: " << sum << std::endl;
}
class monitor{
public:
std::mutex the_mutex;//the mutex to lock the function
std::unique_lock<std::mutex> cond_mutex;//trying to use this for condition_variable
std::condition_variable read_to_go, write_to_go;
int active_reader, active_writer, waiting_reader, waiting_writer;
bool write_flag;
void getTask(int Rank, int& task_one, int& task_two, int& second_rank);//**reader**//
void putResult(int Rank, int the_answer, int next_Rank);//**writer**//
explicit monitor(){
write_flag = true;
active_reader = active_writer = waiting_reader = waiting_writer = 0;
}
private:
inline void startRead();
inline void endRead();
inline void startWrite();
inline void endWrite();
};
monitor imonitor;
inline void monitor::startRead() {
the_mutex.lock();//lock the function code
cond_mutex.lock();//updated 1st
while((active_writer + active_reader) > 0){//if there are working reader and writer
waiting_reader++;//add one
read_to_go.wait(cond_mutex);//wait the thread
/*****when debugging with lldb, error appears here*****/
waiting_reader--;//one less reader waiting when notified
}
active_reader++;//one more working reader
the_mutex.unlock();
}
inline void monitor::endRead() {
the_mutex.lock();
active_reader--;//one less reader working
if(active_reader == 0 && waiting_writer > 0){//if there is not any reader working and there are some writer waiting
write_to_go.notify_one();//notify one writer
}//else get out directly
the_mutex.unlock();
}
inline void monitor::startWrite() {
the_mutex.lock();
cond_mutex.lock();//updated 1st
while((active_writer + active_reader) > 0){//if any reader or writer is working
waiting_writer++;//one more writer waiting
write_to_go.wait(cond_mutex);//block this thread
waiting_writer--;//when notfied, the number of waiting writer become less
}
active_writer++;//one more active writer
the_mutex.unlock();//updated 1st
}
inline void monitor::endWrite() {//write is over
the_mutex.lock();
active_writer--;//one less writer working
if(waiting_writer > 0){//if any writer waiting
write_to_go.notify_one();//notify one of them
}
else if(waiting_reader > 0){//if any reader waiting
read_to_go.notify_all();//notify all of them
}
the_mutex.unlock();
}
void monitor::getTask(int Rank, int &task_one, int &task_two, int &second_rank) {
startRead();
task_one = data[Rank];
while(Rank < (datasize - 1) && data[++Rank] == 0);
task_two = data[Rank];
second_rank = Rank;
//std::cout << "the second Rank is " << Rank << std::endl;
endRead();
}
void monitor::putResult(int Rank, int the_answer, int next_Rank) {
startWrite();
data[Rank] = the_answer;
data[next_Rank] = 0;
endWrite();
}
void reducer(int Rank){
//std::cout << "a reducer begins" << Rank << std::endl;
do {
int myTask1, myTask2, secondRank;
imonitor.getTask(Rank, myTask1, myTask2, secondRank);
if(myTask2 == 0) return;
//std::cout << "the second value Rank: " << secondRank << std::endl;
int answer = myTask1 + myTask2;
imonitor.putResult(Rank, answer, secondRank);
}while (true);
}
int main() {
std::cout << "Hello, World!" << std::endl;
data_init();
std::thread Reduce1(reducer, 0);
std::thread Reduce2(reducer, datasize/2);
/*std::thread Reduce3(reducer, 4);
std::thread Reduce4(reducer, 6);
std::thread Reduce5(reducer, 8);
std::thread Reduce6(reducer, 10);
std::thread Reduce7(reducer, 12);
std::thread Reduce8(reducer, 14);*/
Reduce1.join(); //std::cout << "A reducer in" <<std::endl;
Reduce2.join();
/*Reduce3.join();
Reduce4.join();
Reduce5.join();
Reduce6.join();
Reduce7.join();
Reduce8.join();*/
std::cout << data[0] << std::endl;
return 0;
}
My goal used to use 8 threads, but now the code can work with only one thread. Some cout for debugging are left in the code. Thank you for any help!
Updated 1st: I add cond_mutex.lock() in startWrite() and startRead() after the_mutex.lock(). The error in the last line of startWrite about cond_mutex.unlock() is fixed, which is replaced by the_mutex.unlock(). However, the problem is not fixed.
OK, guys. Thanking for your comments, I am inspired to work this problem out.
At the very beginning, I used std::unique_lock<std::mutex> cond_mutex; in the declaration of the monitor class, meaning the default initializer of unique_lock is called.
class monitor{ public: std::mutex the_mutex, assist_lock;/***NOTE HERE***/ std::unique_lock<std::mutex> cond_mutex;/***NOTE HERE***/ std::condition_variable read_to_go, write_to_go; int active_reader, active_writer, waiting_reader, waiting_writer; ...... };
Let us check the header __mutex_base, where the mutex and unique_lock are defined.(begin at line 104)
`template
class _LIBCPP_TEMPLATE_VIS unique_lock
{
public:
typedef _Mutex mutex_type;
private:
mutex_type* _m;
bool _owns;
public:
_LIBCPP_INLINE_VISIBILITY
unique_lock() _NOEXCEPT : _m(nullptr), _owns(false) {}/first and default/
_LIBCPP_INLINE_VISIBILITY
explicit unique_lock(mutex_type& __m)
: _m(_VSTD::addressof(__m)), _owns(true) {_m->lock();}/second and own an object/
......
};`
Obviously, the first initializer is used instead of second one. __m_ is nullptr.
When the <condition_variable>.wait(cond_mutex); call the lock(), we will find the error information: "unique_lock::lock: references null mutex"(in the header __mutex_base, line 207).
When <unique_lock>.unlock() called, because the lock() does not work the bool __owns_ is false(line 116&211), we will get error information: "unique_lock::unlock: not locked"(line 257).
To fix the problem, we cannot use an all-covered unique_lock cond_mutex, but to initialize cond_mutex each time we want to use startRead() and endStart() with an object the_mutex. In this way, the <unique_lock> cond_mutex(the_mutex) is initialized by the second initializer function, at the same time the_mutex is locked and <unique_lock> cond_mutex owns the_mutex. At the end of
startRead() and endStart(), cond_mutex.unlock() unlocks itself and the_mutex, then the cond_mutex dies and releases the_mutex.
class monitor{
public:
std::mutex the_mutex;
std::condition_variable read_to_go, write_to_go;
int active_reader, active_writer, waiting_reader, waiting_writer;
bool write_flag;
void getTask(int Rank, int& task_one, int& task_two, int& second_rank);//reader
void putResult(int Rank, int the_answer, int next_Rank);//writer
explicit monitor(){
write_flag = true;
active_reader = active_writer = waiting_reader = waiting_writer = 0;
}
private:
inline void startRead();
inline void endRead();
inline void startWrite();
inline void endWrite();
};
inline void monitor::startRead() {
std::unique_lock<std::mutex> cond_mutex(the_mutex);
while((active_writer + active_reader) > 0){
waiting_reader++;
read_to_go.wait(cond_mutex);
waiting_reader--;
}
active_reader++;
cond_mutex.unlock();
}
inline void monitor::startWrite() {
std::unique_lock<std::mutex> cond_mutex(the_mutex);
while((active_writer + active_reader) > 0){
waiting_writer++;
write_to_go.wait(cond_mutex);
waiting_writer--;
}
active_writer++;
cond_mutex.unlock();
}
The solution is here, just to replace my a part of original code with the code above. After all, thank everyone who gives comments.
You seem to be writing this:
class write_wins_mutex {
std::condition_variable cv_read;
std::condition_variable cv_write;
std::mutex m;
int readers=0;
int writers=0;
int waiting_writers=0;
int waiting_readers=0;
std::unique_lock<std::mutex> internal_lock(){ return std::unique_lock<std::mutex>( m );
public:
void shared_lock() {
auto l = internal_lock();
++waiting_readers;
cv_read.wait(l, [&]{ return !readers && !writers && !waiting_writers; } );
--waiting_readers;
++readers;
}
void lock() {
auto l = internal_lock();
++waiting_writers;
cv_write.wait(l, [&]{ return !readers && !writers; } );
--waiting_writers;
++writers;
}
private:
void notify(){
if (waiting_writers) {
if(!readers) cv_write.notity_one();
}
else if (waiting_readers) cv_read.notify_all();
}
public:
void unlock_shared(){
auto l = internal_lock();
--readers;
notify();
}
void unlock(){
auto l = internal_lock();
--writers;
notify();
}
};
probably has typos; wrote this on phone without compiling it.
But this is a mutex compatible with shared_lock and unique_lock (no try lock (for) but that just means those methods don't work).
In this model, readers are shared, but if any writer shows up it gets priority (even to the level of being able to starve readers)
If you want something less biased, just use shared mutex directly.

c++ how can I write single-process named_mutex?

I need a class which will allow me to lock/unlock specific names (or simply indexes), and I don't want it to be multi-processing, so I can run multiple instances of my application. Also I want to avoid use of system-specific APIs, just std or boost. (For simplicity sake, we can say: max number of names/indexes used at the same time is 100)
Unfortunately I has no usage example for you, I just interested is it possible to make.
I tried to find anything like that, but all I found is boost::interprocess::named_mutex and some WinApi methods, like CreateMutexW.
I also tried to write my own code (below), but it definitely not perfect and has at least one potential bug.
So, does anyone has any suggestion, code ideas, or already existing classes?
Thanks in advance
class IndexMutex
{
public:
void Lock(uint32_t id);
void Unlock(uint32_t id);
private:
struct IndexLock
{
static constexpr uint32_t unlocked = ~0u;
void Lock(uint32_t id) {
index_ = id;
mutex_.lock();
}
void Unlock() {
mutex_.unlock();
index_ = unlocked;
}
bool IsLocked() const {
return index_ != unlocked;
}
std::atomic<uint32_t> index_ = unlocked;
std::mutex mutex_{};
};
std::array<IndexLock, 100> mutexes_{};
std::mutex masterMutex_{};
};
void IndexMutex::Lock(uint32_t id)
{
if (id == IndexLock::unlocked) {
return;
}
const std::lock_guard<std::mutex> __guard{ masterMutex_ };
uint32_t possibleId = IndexLock::unlocked;
for (uint32_t i = 0; i < mutexes_.size(); ++i) {
if (mutexes_[i].index_ == id) {
masterMutex_.unlock();
// POTENTIAL BUG: TIME GAP
mutexes_[i].Lock(id);
return;
}
// Searching for unlocked mutex in the same time.
if (possibleId == IndexLock::unlocked && !mutexes_[i].IsLocked()) {
possibleId = i;
}
}
if (possibleId == IndexLock::unlocked) {
throw std::runtime_error{ "No locks were found." };
}
// We are sure here, that mutex can't be locked
// because we were protected by the muster mutex all that time.
mutexes_[possibleId].Lock(id);
}
void IndexMutex::Unlock(uint32_t id)
{
if (id == IndexLock::unlocked) {
return;
}
const std::lock_guard<std::mutex> __guard{ masterMutex_ };
for (auto& lock : mutexes_) {
if (lock.index_ == id) {
lock.Unlock();
return;
}
}
throw std::runtime_error{ "No mutexes there found by specified index." };
}
You want a reference counted mutex map, protected by a master mutex. An implementation in terms of
std::map<int, std::pair<int, std::mutex>>
would do the job.
The lock operation works like this (untested pseudocode):
master.lock()
std::pair<int, std::mutex>& m = mymap[index]; //inserts a new one if needed
m.first++;
master.unlock();
m.second.lock();
The unlock operation:
master.lock();
std::pair<int, std::mutex>& m = mymap[index];
m.second.unlock();
m.first--;
if (m.first==0) mymap.remove(index);
master.unlock();
No deadlocks! It is possible to first unlock the master and then lock the found mutex. Even if another thread intervenes and unlocks the mutex, the reference count won't drop to zero and the mutex will not be removed.

Thread safe operator<<

I have a quite complicated problem with my logger class. Its is a singleton pattern logging class. A thread is created only for taking out items from queue and logging them. Generally everything works fine and the error occurs very occasionally as segmentation fault. It was happening even more often before I decided to put a mutex on whole method chain. With that mutex i dont understand why is the segmentation fault occuring. The class got quite complicated because of the operator<< usage. Problem with it is that the operator template is run as many times, as many items are passed using <<. Because of that other threads can cut in between those templete calls.
The general usage looks like this:
1. instance method is called (creating or pointing to the instance pointer (singleton). mutex is locked at this moment.
2. any called methods are called, for example operator<< template.
3. finishing method is called, placing log in the queue and unlocking mutex.
I have edited the code and tried to take the fata gathering out of the singleton class to the proxy class.
main.c:
#include <iostream>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include "CLogger.h"
#include "CTestClass.h"
using namespace std;
int main()
{
CLogger::instance().log.startLog("/home/lukasz/Pulpit/test.txt", true);
CLogger::instance().log.setLogLevel(CLogger::ElogLevel::eDebug);
CLogger::instance().printf("Print test");
CLogger::instance().printf("Print test");
CLogger::instance().printf("Print test");
CLogger::instance() << "Stream test" ;
CLogger::instance() << "Stream test" ;
CLogger::instance() << "Stream test" ;
//CTestClass test1(1);
//CTestClass test2(2);
//CTestClass test3(3);
sleep(3);
CLogger::instance().log.stopLog();
return 0;
}
CLogger.h:
#ifndef CLOGGER_H_
#define CLOGGER_H_
#include <iostream>
#include <deque>
#include <string>
#include <mutex>
#include <condition_variable>
#include <pthread.h>
#include <ostream>
#include <fstream>
#include <sstream>
#include <ctime>
#include <iomanip>
#include <sys/time.h>
#include <stdarg.h>
#include <assert.h>
#include "CTeebuf.h"
using namespace std;
class CLoggerProxy;
/*!
* \brief Singleton class used for logging
*/
class CLogger
{
public:
/*!
* \brief Describes the log level of called \ref CLogger object.
*/
enum class ElogLevel { eNone = 0, eError, eWarning, eInfo, eDebug };
/*!
* Structure describing a single log item:
*/
struct logline_t
{
string logString; /*!< String line to be saved to a file (and printed to cout). */
ElogLevel logLevel; /*!< The \ref ElogLevel of this line. */
timeval currentTime; /*!< time stamp of current log line */
};
static CLogger* internalInstance(ElogLevel lLevel = ElogLevel::eDebug);
static CLoggerProxy instance(ElogLevel lLevel = ElogLevel::eDebug);
bool startLog(string fileName, bool verbose);
void setLogLevel(ElogLevel ll);
void stopLog();
void finaliseLine(logline_t* log);
protected:
virtual void threadLoop();
private:
CLogger() {}; // Private so that it can not be called
CLogger(CLogger const&) {}; // copy constructor is private
CLogger& operator= (CLogger const&) {}; // assignment operator is private
/*!< Global static pointer used to ensure a single instance of the class */
static CLogger* mp_instance;
bool m_logStarted;
ElogLevel m_userDefinedLogLevel;
ofstream m_logFileStream;
bool m_verbose;
bool m_finishLog;
timeval m_initialTime;
static void * threadHelper(void* handler)
{
((CLogger*)handler)->threadLoop();
return NULL;
}
deque<logline_t*> m_data;
mutex m_mutex2;
condition_variable m_cv;
pthread_t m_thread;
logline_t pop_front();
void push_back(logline_t* s);
};
/*!
* RAII class used for its destructor, to add a log item to the queue
*/
class CLoggerProxy
{
public:
CLogger &log;
CLoggerProxy(CLogger &logger) : log(logger)
{
mp_logLine = new CLogger::logline_t;
gettimeofday(&mp_logLine->currentTime, NULL);
}
~CLoggerProxy() { log.finaliseLine(mp_logLine); }
void printf(const char* text, ...);
/*!
* Takes the data from the stream and adds it to the current string.
* #param t stream item
* #return \ref object address
*/
template <typename T>
CLoggerProxy& operator<< (const T &t)
{
ostringstream stream;
stream << t;
mp_logLine->logString = (stream.str() + " ");
return *this;
}
private:
CLogger::logline_t* mp_logLine;
};
#endif /* CLOGGER_H_ */
CLogger.cpp:
#include "CLogger.h"
using namespace std;
CLogger* CLogger::mp_instance = NULL;
/*!
* This function is called to create an instance of the class.
* Calling the constructor publicly is not allowed. The constructor
* is private and is only called by this Instance function.
* #param lLevel Log level for current object
*/
CLogger* CLogger::internalInstance(ElogLevel lLevel)
{
// Only allow one instance of class to be generated.
if (!mp_instance)
{
mp_instance = new CLogger;
assert(mp_instance);
}
return mp_instance;
}
/*!
* This method is called in order to use the methods
* within the objects.
* #param lLevel Log level for current object
*/
CLoggerProxy CLogger::instance(ElogLevel lLevel)
{
return CLoggerProxy(*internalInstance(lLevel));
}
/*!
* \brief Starts the logging system.
*
* This method creates and opens the log file,
* then opens it and creates the threadloop for messages deque.
* #param fileName desired log file path,
* #param verbose when set true, logging will also be printed to standard output.
*/
bool CLogger::startLog(string fileName, bool verbose)
{
if(remove(fileName.c_str()) != 0)
perror( "Error deleting file" );
m_logFileStream.open(fileName.c_str(), ios::out | ios::app);
if (!m_logFileStream.is_open())
{
cout << "Could not open log file " << fileName << endl;
return false;
}
m_finishLog = false;
m_verbose = verbose;
m_logStarted = true;
gettimeofday(&m_initialTime, NULL);
return (pthread_create(&(m_thread), NULL, threadHelper, this) == 0);
}
/*!
* \brief puts a \ref logline_t object at the end of the queue
* #param s object to be added to queue
*/
void CLogger::push_back(logline_t* s)
{
unique_lock<mutex> ul(m_mutex2);
m_data.emplace_back(move(s));
m_cv.notify_all();
}
/*!
* \brief takes a \ref logline_t object from the beggining of the queue
* #return first \ref logline_t object
*/
CLogger::logline_t CLogger::pop_front()
{
unique_lock<mutex> ul(m_mutex2);
m_cv.wait(ul, [this]() { return !m_data.empty(); });
logline_t retVal = move(*m_data.front());
assert(m_data.front());
delete m_data.front();
m_data.front() = NULL;
m_data.pop_front();
return retVal;
}
/*!
* \brief Sets the log level for the whole \ref CLogger object.
* If \ref m_logLine is equal or higher than set level, log
* is going to be printed.
* #param lLevel desired user define log level.
*/
void CLogger::setLogLevel(ElogLevel lLevel)
{
m_userDefinedLogLevel = lLevel;
}
/*!
* \brief Stops the logging system.
* Last final logline is being added and then the logging thread
* is being closed.
*/
void CLogger::stopLog()
{
m_finishLog = true;
//instance(ElogLevel::eNone).log << "CLogger Stop";
//pthread_join(m_thread, NULL);
}
/*!
* This function should be run in the \ref CLoggerProxy destructor.
* is pushes the gathered stream to the queue.
*/
void CLogger::finaliseLine(logline_t* log)
{
if (log->logString.size() > 0)
push_back(log);
else
delete log;
}
/*!
* \brief Adds text log to the string in the printf c way.
* Works faster than operator<< and its more atomic.
* #param text pointer to a character string.
* #param ... argptr parameters
*/
void CLoggerProxy::printf(const char* text, ...)
{
va_list argptr;
va_start(argptr, text);
char* output = NULL;
vasprintf(&output, text, argptr);
mp_logLine->logString = output;
va_end(argptr);
}
/*!
* The loop running in a separate thread. It take items of the
* log deque object (if there are any) and saves them to a file.
*/
void CLogger::threadLoop()
{
logline_t logline;
const string logLevelsStrings[] = {"eNone", "eError", "eWarning", "eInfo", "eDebug" };
COteestream tee;
tee.add(m_logFileStream);
if (m_verbose)
tee.add(cout);
struct sched_param param;
param.__sched_priority = 0;
if(!sched_setscheduler(0, SCHED_IDLE, &param))
instance().printf("Clogger scheduler policy set to %d", sched_getscheduler(0));
int secs = 0;
int h = 0;
int m = 0;
int s = 0;
do
{
logline = pop_front(); // waits here for new lines
secs = logline.currentTime.tv_sec - m_initialTime.tv_sec;
h = secs / 3600;
m = ( secs % 3600 ) / 60;
s = ( secs % 3600 ) % 60;
tee << "["
<< setw(2) << setfill('0') << h
<< ":"
<< setw(2) << setfill('0') << m
<< ":"
<< setw(2) << setfill('0') << s
<< "."
<< setw(6) << setfill('0') << logline.currentTime.tv_usec
<< "]"
<< "["
<< setw(2) << setfill('0') << m_data.size()
<< "]"
<< "["
<< logLevelsStrings[(int)logline.logLevel]
<< "] "
<< logline.logString << "\n" << flush;
}
//while(!(m_finishLog && m_data.empty()));
while(1);
m_logFileStream.close();
}
There are several problems with your code.
Singleton
// Only allow one instance of class to be generated.
if (!mp_instance)
{
mp_instance = new CLogger;
assert(mp_instance);
}
This is a classic problem. It may be called by different threads at the same time, and it is not thread safe. You may end up with several instances of your singleton.
The queue (m_data)
Clients of your logger put their messages into this queue (apparently secured by m_mutext).
m_data.emplace_back(move(s));
m_cv.notify_all();
Your logger thread removes the messages in its own thread (secured by m_mutex2).
unique_lock<mutex> ul(m_mutex2);
m_cv.wait(ul, [this]() { return !m_data.empty(); });
logline_t retVal = move(*m_data.front());
assert(m_data.front());
delete m_data.front();
m_data.front() = NULL;
m_data.pop_front();
return retVal;
The problem here is, you use 2 different mutexes to synchronize access to the same object. This cannot work.
In addition you access m_data in your thread without any locking at all:
<< setw(2) << setfill('0') << m_data.size()
or
while(!(m_finishLog && m_data.empty()));
The log message (mp_logLine)
You try to lock too much data. The pointer to your log message is meant to be used by a single thread at a time. But you store it in the main logger class which is accessed by all threads. You already have a proxy for your logger which is private to the thread using it. Store your message there until it's finished, and then add it to the queue.
Generally said, minimize the amount of data to be locked. If you rework your code and the only object needing locking is your queue, you are on the right way.
This doesn't look threadsafe at all.
mp_logLine->logString += (stream.str() + " ");
This looks like it's shared between all threads that log to an instance. It's not clear from your code that += is thread safe.
Another point is when you push back an item into your queue you don't lock the mutex. If two threads are doing it at the same time they can mess up the deque. There's no guarantee that you can do push_back and pop_front in parallel, so put a mutex around it.
When you get an internal instance you lock mp_instance->m_mutex but it doesn't look like you're unlocking it.
You use bool m_logStarted from parallel threads and that can also introduce race conditions and inconsistencies.
Any of the above can cause your segmentation fault.
Getting multi-threading right is really hard. Debugging it is even harder. Try to offload the multi-threading component(s) to a library you know already works and add things in a single threaded context. In this case it would mean to use a separate class instance for every call to the log and then push the item to a producer-consumer queue that's implemented by some library and guaranteed thread-safe. There are many other ways to do it as well.

Modifying data in threads

I have a class with the following structure:
class Nginx_sender
{
private:
std::vector<std::string> mMessagesBuffer;
boost::mutex mMutex;
void SendMessage(const std::string &msg)
{
mMutex.lock();
mMessagesBuffer.push_back(msg);
mMutex.unlock();
std::cout << "Vector size: " << mMessagesBuffer.size() << std::endl;
}
void NewThreadFunction()
{
while(true) {
mMutex.lock();
if (mMessagesBuffer.size() >= 1) std::cout << ">=1\n";
mMutex.unlock();
boost::this_thread::sleep(boost::posix_time::milliseconds(200));
}
}
};
int main()
{
Nginx_sender *NginxSenderHandle;
boost::thread sender(boost::bind(&Nginx_sender::NewThreadFunction, &NginxSenderHandle));
// ...
}
NewThreadFunction is running in new thread and it checks the size of mMessagesBuffer. Now I call anywhere in main function: NginxSenderHandle->SendMessage("Test");
This shows up: Vector size: 1 first time, 2 second time etc.
But! In NewThreadFunction it's always == 0. Why could it be?
You are most probably creating another copy of Nginx_sender when you bind it. Do you really need to reference NginxSenderHandle before passing it to bind() (it's already a pointer)? http://www.boost.org/doc/libs/1_49_0/libs/bind/bind.html#with_member_pointers
I bet compiler is caching some of mMessagesBuffer internals in thread-local cache. Try adding 'volatile' keyword to mMessagesBuffer to disable such optimizations.

How to debug Cygwin failure?

I am porting from Centos to Cygwin and find that my application is exiting with no error message and exit status zero mid execution during the constructor for Botan::InitializationVector.
If I try to attach with gdb proactively in main() where it is waiting on a spin variable, I don't get a normal stack trace:
(gdb) where
#0 0x7c90120f in ntdll!DbgUiConnectToDbg ()
from /cygdrive/c/WINDOWS/system32/ntdll.dll
#1 0x7c952119 in ntdll!KiIntSystemCall ()
from /cygdrive/c/WINDOWS/system32/ntdll.dll
#2 0x00000005 in ?? ()
#3 0x00000000 in ?? ()
So with no gdb, it is hard to figure out what is going wrong.
Why would I get no error message on Cygwin yet the application would exit mid execution?
I deduce it is inside the constructor due to clog only showing for line before and not after constructor:
clog << " About to create iv for Botan.\n";
Botan::InitializationVector iv(_rng, size);
clog << " About to copy iv for Botan.\n";
Botan is open source: http://botan.randombit.net/ Here are some code snippets from src/sym_algo/symkey.{h,cpp}:
typedef OctetString InitializationVector;
class BOTAN_DLL OctetString
{
public:
u32bit length() const { return bits.size(); }
SecureVector<byte> bits_of() const { return bits; }
const byte* begin() const { return bits.begin(); }
const byte* end() const { return bits.end(); }
std::string as_string() const;
OctetString& operator^=(const OctetString&);
void set_odd_parity();
void change(const std::string&);
void change(const byte[], u32bit);
void change(const MemoryRegion<byte>& in) { bits = in; }
OctetString(class RandomNumberGenerator&, u32bit len);
OctetString(const std::string& str = "") { change(str); }
OctetString(const byte in[], u32bit len) { change(in, len); }
OctetString(const MemoryRegion<byte>& in) { change(in); }
private:
SecureVector<byte> bits;
};
OctetString::OctetString(RandomNumberGenerator& rng,
u32bit length)
{
bits.create(length);
rng.randomize(bits, length);
}
I moved the failing code into main() and it works fine. I also put a try catch ... around the code and no exceptions are being thrown. Something goes wrong between main() and the point of failure later in the application. I can do a divide and conquer to narrow down the exact point where it no longer works. One of the Botan developers gave me this stripped down code to use instead that also fails:
Botan::AutoSeeded_RNG _rng;
unsigned int size = 1; // or 16, or 1452 all fail.
Botan::SecureVector<Botan::byte> iv_val(size);
cerr << "We get to here." << endl;
_rng.randomize(&iv_val[0], size);
cerr << "But not here." << endl;
Now that I have the debugger working I see segv:
(gdb) s
Botan::AutoSeeded_RNG::randomize (this=0x1270380, out=0x5841420 "", len=1)
at ../../src/Botan-1.8.11/build/include/botan/auto_rng.h:23
(gdb) s
Program received signal SIGSEGV, Segmentation fault.
0x005d79ee in Botan::AutoSeeded_RNG::randomize (this=0x1270380,
out=0x5841420 "", len=1)
at ../../src/Botan-1.8.11/build/include/botan/auto_rng.h:23
(gdb) p rng
$7 = (class Botan::RandomNumberGenerator *) 0x5841324
(gdb) p *this
$8 = {<Botan::RandomNumberGenerator> = {
_vptr$RandomNumberGenerator = 0x11efc14}, rng = 0x5841324}
(gdb) p *rng
$9 = {_vptr$RandomNumberGenerator = 0x656e6f4e}
Here is auto_rng.h code:
class BOTAN_DLL AutoSeeded_RNG : public RandomNumberGenerator
{
public:
void randomize(byte out[], u32bit len)
{ rng->randomize(out, len); } // SEGV on this line.
bool is_seeded() const
{ return rng->is_seeded(); }
void clear() throw() { rng->clear(); }
std::string name() const
{ return "AutoSeeded(" + rng->name() + ")"; }
void reseed(u32bit poll_bits = 256) { rng->reseed(poll_bits); }
void add_entropy_source(EntropySource* es)
{ rng->add_entropy_source(es); }
void add_entropy(const byte in[], u32bit len)
{ rng->add_entropy(in, len); }
AutoSeeded_RNG(u32bit poll_bits = 256);
~AutoSeeded_RNG() { delete rng; }
private:
RandomNumberGenerator* rng;
};
Cygwin apps are multithreaded (e.g., one thread is the signal listener thread). Use info threads in gdb to find the thread that really faulted.
You could attach gdb proactively and put a breakpoint right before the constructor call that is failing, then single step through.
Based on the new code, you're violating the Rule of Three and this may be causing your problem.
By defining a class with raw pointers and not providing a correct copy constructor (or making it inaccessible) you open yourself up to double-free.
Add the copy constructor. Or open an issue on the project's bug tracker. You are using the latest version, right?