Thread Safe Pipe Termination - c++

Question

(Note before starting: Although my question is general, my code needs to compile with legacy Visual Studio 2008 MFC application and has to use MFC or win32 synchronization, please avoid answers using ie boost or c++ 11)
I am trying to implement a Thread Safe Pipe (A Queue with a single reader and a single writer), I did the following:
template<class T>
class CMultiThreadPipe {
private:
HANDLE hSemaphore, hTerminateEvent1, hTerminateEvent2;
CRITICAL_SECTION listMutex;
CList<T*, T*> list;
public:
CMultiThreadPipe() {
InitializeCriticalSection(&listMutex);
hSemaphore = CreateSemaphore(NULL, 0, LONG_MAX, NULL);
hTerminateEvent1 = ::CreateEvent(NULL, TRUE, FALSE, NULL);
hTerminateEvent2 = ::CreateEvent(NULL, TRUE, FALSE, NULL);
}
// pdata must be allocated with new. The dequeueing thread will delete it
void Enqueue(T* pdata) {
EnterCriticalSection(&listMutex);
list.AddHead(pdata);
LeaveCriticalSection(&listMutex);
ReleaseSemaphore(hSemaphore, 1, NULL);
}
// if Dequeue returns null it means the pipe was destroyed and no further queue method calls are legal
// Dequeue caller is responsible to delete the returned instance
T* Dequeue()
{
HANDLE handles[] = { hTerminateEvent1, hSemaphore };
DWORD waitRes = WaitForMultipleObjects(2, handles, FALSE, INFINITE);
if (waitRes==WAIT_OBJECT_0) {
SetEvent(hTerminateEvent2);
return NULL; // terminated
}
EnterCriticalSection(&listMutex);
T* elem = list.RemoveTail();
LeaveCriticalSection(&listMutex);
return elem; // handler must delete item
}
void Destroy() {
SetEvent(hTerminateEvent1);
WaitForSingleObject(hTerminateEvent2, INFINITE);
EnterCriticalSection(&listMutex);
POSITION pos = list.GetHeadPosition();
for (int i = 0; i < list.GetCount(); i++) delete list.GetNext(pos);
LeaveCriticalSection(&listMutex);
DeleteCriticalSection(&listMutex);
CloseHandle(hSemaphore);
}
~CMultiThreadPipe() {
Destroy();
}
};
The code is used like this:
class QueueData {
public:
QueueData(int i) : m_data(i) {};
int m_data;
};
UINT DequeueThreadProc(LPVOID dummy);
CMultiThreadedPipe<QueueData>* pPipe = NULL;
void main() {
pPipe = new CMultiThreadedPipe<QueueData>();
start new thread running DequeueThreadProc
int counter=0;
for (int counter=0; counter<10; counter++)
{
pPipe->Enqueue(new QueueData(counter));
Sleep(300);
}
delete pPipe;
}
UINT DequeueThreadProc(LPVOID ignore)
{
QueueData* queueData;
while ((queueData = pPipe->Dequeue()) != NULL) {
delete queueData;
Sleep(1000);
};
return 0;
}
The issue I have is with termination, in the above implementation, when the pipe is destroyed (always by the enqueing thread) it is waiting for the dequeing thread to know that it terminated before deleting the queue. It has to do that to prevent a situation where the dequeing thread tries to dequeue after the pipe is destroyed.
If the dequeing thread does not keep calling dequeue the first thread will hang in the destructor, also if the dequeing thread waits a long time between calls to dequeue the destructor of the first thread will get stuck there accordingly.
I read various posts about it none mentions safe destruction. Any help appreciated !

Answer 1

for safe destruction object, which accessed from multiple threads you need use reference counting on it. before pass object pointer to new thread - you increment reference on object. when thread no more using object, or if create thread fail, you decrement reference count. when last reference on object released - you can safe call destructor for object. and you not need here wait for any threads.
also for implement such queue - in windows exist special object - named I/O Completion Ports in user space (in kernel space in know as KQUEUE). with this object - implementation will be more efficient and simply - you not need manage self list (CList in your code), synchronize access to it - all this will be done in kernel space for you (PostQueuedCompletionStatus -> KeInsertQueue, GetQueuedCompletionStatus -> KeRemoveQueue). you need create only iocp, (kqueue) object.
class CMultiThreadPipe {
public:
class __declspec(novtable) QueueData {
public:
virtual void ProcessItem() = 0;
virtual ~QueueData()
{
DbgPrint("%x: %s<%p>\n", GetCurrentThreadId(), __FUNCTION__, this);
}
QueueData()
{
DbgPrint("%x: %s<%p>\n", GetCurrentThreadId(), __FUNCTION__, this);
}
};
private:
HANDLE _hIOCP;
LONG _dwRef;
ULONG _nThreads;
void DequeueThreadProc()
{
ULONG NumberOfBytesTransferred;
QueueData* pData;
OVERLAPPED* pOverlapped;
while (GetQueuedCompletionStatus(_hIOCP,
&NumberOfBytesTransferred,
(ULONG_PTR*)&pData,
&pOverlapped, INFINITE))
{
if (pData)
{
pData->ProcessItem();
}
else
{
break;
}
}
Release();
}
__declspec(noreturn) static DWORD CALLBACK _DequeueThreadProc(PVOID pThis)
{
reinterpret_cast<CMultiThreadPipe*>(pThis)->DequeueThreadProc();
FreeLibraryAndExitThread((HMODULE)&__ImageBase, 0);
}
~CMultiThreadPipe()
{
if (_hIOCP)
{
CloseHandle(_hIOCP);
}
}
public:
CMultiThreadPipe() : _dwRef(1), _hIOCP(0)
{
}
void AddRef()
{
InterlockedIncrement(&_dwRef);
}
void Release()
{
if (!InterlockedDecrement(&_dwRef))
{
delete this;
}
}
ULONG Create(DWORD NumberOfDequeueThreads)
{
if (_hIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, NumberOfDequeueThreads))
{
ULONG n = 0;
do
{
HMODULE hModule;
if (GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, (PCWSTR)_DequeueThreadProc, &hModule))
{
AddRef();
if (HANDLE hThread = CreateThread(0, 0, _DequeueThreadProc, this, 0, 0))
{
CloseHandle(hThread);
n++;
}
else
{
Release();
FreeLibrary(hModule);
}
}
} while (--NumberOfDequeueThreads);
_nThreads = n;
return n ? NOERROR : ERROR_GEN_FAILURE;
}
return GetLastError();
}
ULONG Enqueue(QueueData* pData)
{
return PostQueuedCompletionStatus(_hIOCP, 0, (ULONG_PTR)pData, 0) ? NOERROR : GetLastError();
}
void Destroy()
{
if (ULONG n = _nThreads)
{
do
{
PostQueuedCompletionStatus(_hIOCP, 0, 0, 0);
} while (--n);
}
}
};
and usage:
class QueueData : public CMultiThreadPipe::QueueData
{
int m_data;
virtual void ProcessItem()
{
DbgPrint("%x: %s<%p>(%u)\n", GetCurrentThreadId(), __FUNCTION__, this, m_data);
delete this;
}
public:
QueueData(int i) : m_data(i) {};
};
void testQueue()
{
if (CMultiThreadPipe* pPipe = new CMultiThreadPipe)
{
if (pPipe->Create(8) == NOERROR)
{
int n = 64;
do
{
if (QueueData* pData = new QueueData(n))
{
if (pPipe->Enqueue(pData))
{
delete pData;
}
}
} while (--n);
pPipe->Destroy();
}
pPipe->Release();
}
}
note with such CMultiThreadPipe implementations - you not need wait when working threads exit. even if your code inside dll and you unload dll - you not need wait. every thread have own reference for object and module. and release it on exit