IOCP Threads and GetQueuedCompletionStatus - c++

Question

Been trying to wrap my head around windows IOCP and have run into a few issues as many other people seem to be doing so.
So far what I'm trying to do is run a TCP IOCP server, which creates 3 accept sockets and has a thread for each waiting on a GetQueuedCompletionStatus. Below is the main thread loop for said threads.
The problem I've run into currently - I believe - is as follow.
Spawn 3 "worker" threads
Create 3 accept sockets in a loop and call AcceptEx() for each
Client 1 connects with no issue on Thread 1
GetQueuedCompletionStatus releases Thread 1 to continue on
Thread 1 now waits for data on WSARecv().
!!PROBLEM!! Client one sends data, Thread 1's WSARecv() seems to now send a completion packet
which is picked up BY ANOTHER THREAD OUT OF THE 3
Code hangs because of obvious reason (Thread 1 is waiting on the GQCS after its WSARecv(), but never receives it because it went to Thread 2 and Thread 2 is told to continue even though it hasn't actually received a client)
I say I believe this to be the issue because I've run it through debugger and outputted thread id's to console and have seen that right after starting a single client I get 2 outputs of "Accepted client on thread ..." with 2 different thread ids.
Also, when I was stepping through the code in the debugger, as soon as I went passed the WSARecv() in Thread 1, the code hit a breakpoint in Thread 2 at the GQCS line.
I'm clearly misunderstanding something about the use/function of GetQueuedCompletionStatus or something related to it. Is my fundamental methodology here just wrong or is this a easy fix I'm just not getting?
How can I get it such that, the WSARecv() sends its completion packet/signal thing to the write thread?
// Initialize Winsock...
// Create a handle for the completion port
hCompPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, (u_long)0, 0);
if (hCompPort == NULL) {
wprintf(L"CreateIoCompletionPort failed with error: %u\n",
GetLastError());
WSACleanup();
return 1;
}
//Create worker threads
for (int i = 0; i < 3; i++)
{
threads.push_back(std::thread(WorkerThreadFun));
}
// Create a listening socket
ListenSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
// Associate the listening socket with the completion port
u_long ck = 111;
CreateIoCompletionPort((HANDLE)ListenSocket, hCompPort, (u_long)0, 0);
//----------------------------------------
// Bind the listening socket to the local IP address
// and port 27015
if (bind(ListenSocket, (SOCKADDR*)&service, sizeof(service)) == SOCKET_ERROR) {
wprintf(L"bind failed with error: %u\n", WSAGetLastError());
closesocket(ListenSocket);
WSACleanup();
return 1;
}
//----------------------------------------
// Start listening on the listening socket
iResult = listen(ListenSocket, 100);
printf("Listening on address: %s:%d\n", ip, port);
iResult = WSAIoctl(ListenSocket, SIO_GET_EXTENSION_FUNCTION_POINTER,
&GuidAcceptEx, sizeof(GuidAcceptEx),
&lpfnAcceptEx, sizeof(lpfnAcceptEx),
&dwBytes, NULL, NULL);
if (iResult == SOCKET_ERROR) {
wprintf(L"WSAIoctl failed with error: %u\n", WSAGetLastError());
closesocket(ListenSocket);
WSACleanup();
return 1;
}
//pre-emptively create sockets for connections
for (int i = 0; i < 3; i++)
{
// Create an accepting socket
AcceptSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
}
//create Overlapped struct for AcceptEx and following that GetQueCompStatus
LPWSAOVERLAPPEDPLUS pOl = new WSAOVERLAPPEDPLUS();
//Initialise the "extended" overlapped struct with appropriate data
memset(pOl, 0, sizeof(WSAOVERLAPPEDPLUS));
pOl->operation = OP_ACCEPTEX;
pOl->client = AcceptSocket;
pOl->listenSocket = ListenSocket;
DWORD expected = sizeof(struct sockaddr_in) + 16;
int buflen = (sizeof(SOCKADDR_IN) + 16) * 2;
char* pBuf = new char[buflen];
memset(pBuf, 0, buflen);
// Empty our overlapped structure and accept connections.
//memset(&olOverlap, 0, sizeof(olOverlap));
bRetVal = lpfnAcceptEx(ListenSocket, AcceptSocket, pBuf,
0, //0 to avoid waiting to read data from a send()
expected, expected,
&pOl->dwBytes,
&pOl->ProviderOverlapped);
if (bRetVal == FALSE)
{
int err = WSAGetLastError();
if (err != ERROR_IO_PENDING)
{
wprintf(L"AcceptEx failed with error: %u\n", WSAGetLastError());
closesocket(AcceptSocket);
closesocket(ListenSocket);
WSACleanup();
return 1;
}
}
}//for
WorkerThreadFunc()
while (TRUE)
{
bOk = GetQueuedCompletionStatus(hCompPort, &bytes_transferred, (PULONG_PTR)&completion_key, &pOverlapped, INFINITE);
std::thread::id this_id = std::this_thread::get_id();
std::cout << "Accepted client on thread" << this_id << "Going to sleep for 5 seconds" << std::endl;
//std::this_thread::sleep_for(std::chrono::seconds(5));
if (bOk) {
// Process a successfully completed I/O request
if (completion_key == 0) {
// Safe way to extract the customized structure from pointer
// is to use 'CONTAINING_RECORD'. Read more on 'CONTAINING_RECORD'.
WSAOVERLAPPEDPLUS* pOl = CONTAINING_RECORD(pOverlapped, WSAOVERLAPPEDPLUS, ProviderOverlapped);
if (pOl->operation == OP_ACCEPTEX)
{
if (hCompPort2 == NULL)
{
hCompPort2 = CreateIoCompletionPort((HANDLE)pOl->client, hCompPort, (u_long)&this_id, 0);
// hCompPort2 should be hCompPort if this succeeds
if (hCompPort2 == NULL)
{
wprintf(L"CreateIoCompletionPort associate failed with error: %u\n",
GetLastError());
closesocket(pOl->client);
closesocket(pOl->listenSocket);
WSACleanup();
break;
}//if: CreateIOCP error
}//if: accept socket already IOCP
// Before doing any WSASend/WSARecv, inherit the
// listen socket properties by calling 'setsockopt()'
setsockopt(pOl->client, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT,
(char*)&pOl->listenSocket, sizeof(pOl->listenSocket));
DWORD& recvbytes = pOl->dwBytes;
char buf[MAX_BUF];
WSABUF wsabuf = { MAX_BUF, buf };
DWORD flags = 0;
if (WSARecv(pOl->client, &wsabuf, 1, &pOl->dwBytes, &flags, &pOl->ProviderOverlapped, NULL) == SOCKET_ERROR)
{
int err = WSAGetLastError();
if (err != WSA_IO_PENDING)
{
printf("WSARecv failed with error: %d\n", WSAGetLastError());
//return 1;
}
}
bool qresult = GetQueuedCompletionStatus(hCompPort, &pOl->dwBytes,
(PULONG_PTR)&completion_key, &pOverlapped, INFINITE);
if (!qresult)
{
DWORD err = GetLastError();
printf("* error %d getting completion port status!!!\n", err);
}
int iResult = WSAGetOverlappedResult(pOl->client, &pOl->ProviderOverlapped, &pOl->dwBytes, FALSE, &flags);
if (iResult == FALSE) {
wprintf(L"WSARecv operation failed with error: %d\n", WSAGetLastError());
//return 1;
break;
}
wsabuf.buf[recvbytes] = '\0';
std::cout << "Bytes received: " << recvbytes << std::endl;
std::wcout << "Bytes create: " << wsabuf.buf << std::endl;
}
delete pOl;
}
}
else {
// Handle error ...
}
}
}