Ok this is related to a previous post, but it's a different error so I made a new question. Previous Post: IO Completion port returning NULL Completion Key
So getting a receive message on the IO completion port i set up, It triggers GetQueuedCompletionStatus and returns with the Completion Key and the Overlapped data. Both appear to be good and I can see data popluated in their structures. However the Buffer which was passed to WSARecv was not populated with the incoming message. (the BytesTransfered indicates that there were bytes received, but no data in the WSABUF).
Here is the code as it currently stands, looking for help as to why the Buffer isn't being populated.
networkhandlerthread.ccp
#include "NetworkHandlerThread.h"
// Worker thread, processes IOCP messages.
DWORD ServerWorkerThread(LPVOID lpParam)
{
HANDLE CompletionPort = (HANDLE)lpParam;
DWORD BytesTransferred = 0;
OVERLAPPED* lpOverlapped = NULL;
LPCONNECTED_SOCKET_DATA ConnectedSocketData = NULL;
LPPER_IO_OPERATION_DATA PerIoData = NULL;
DWORD Flags = 0;
WSABUF* DataBuf;
DWORD RecvBytes = 0;
Type1MessageParser Type1MsgParser;
Type2MessageParser Type2MsgParser;
int DestinationAddress = 0;
bool IsType1 = false;
while (TRUE)//run forever
{
//Check for new message
if (GetQueuedCompletionStatus(CompletionPort, &BytesTransferred, (PULONG_PTR)&ConnectedSocketData, (LPOVERLAPPED*)&PerIoData, INFINITE) == 0)
{
DWORD Err = GetLastError();
if (Err != WAIT_TIMEOUT)
{
printf("GetQueuedCompletionStatus() failed with error %d\n", Err);
if (closesocket(ConnectedSocketData->Socket) == SOCKET_ERROR)
{
printf("closesocket() failed with error %d\n", WSAGetLastError());
return 0;
}
GlobalFree(ConnectedSocketData);
}
continue;
}
//We have a message, determine if it's something we receaved or something we should send.
if (PerIoData->OperationType == OPERATION_TYPE_RECV)
{
///tbd process recv
ConnectedSocketData; //this is comming in good and has data
PerIoData->Buffer; // this is empty (pointer is good, but no data)
}
else if (PerIoData->OperationType == OPERATION_TYPE_SEND)
{
///tbd process send
}
}
};
//Thread for handling Listener sockets and Accepting connections
DWORD ListenThread(LPVOID lpParam)
{
LPLISTEN_SOCKET_DATA pSocketData = (LPLISTEN_SOCKET_DATA)(lpParam);
WSANETWORKEVENTS NetworkEvents;
DWORD dwRet;
SOCKADDR_IN NewSockAddr;
SOCKET NewSocket;
int nLen;
while (true) //run forever
{
//Wait for event
dwRet = WSAWaitForMultipleEvents(1,
&(pSocketData->hAcceptEvent),
false,
100,
false);
//Nothing happened, back to top
if (dwRet == WSA_WAIT_TIMEOUT)
continue;
//We got a event, find out which one.
int nRet = WSAEnumNetworkEvents(pSocketData->Socket,
pSocketData->hAcceptEvent,
&NetworkEvents);
if (nRet == SOCKET_ERROR)
{
wprintf(L"WSAEnumNetworkEvents error %ld\n", WSAGetLastError());
break;
}
//We got a Accept event
if (NetworkEvents.lNetworkEvents & FD_ACCEPT)
{
//Check for errors
if (NetworkEvents.iErrorCode[FD_ACCEPT_BIT] == 0)
{
// Accept new connection
nLen = sizeof(SOCKADDR_IN);
NewSocket = WSAAccept(pSocketData->Socket,
(LPSOCKADDR)&NewSockAddr,
&nLen, NULL, NULL);
if (NewSocket == SOCKET_ERROR)
{
wprintf(L"accept() error %ld\n", WSAGetLastError());
break;
}
wprintf(L"Accepted Connection %ld", NewSockAddr.sin_addr.S_un.S_addr);
//Set new connection as TCP connection, No Delay
//const char chOpt = 1;
//int nErr = setsockopt(NewSocket, IPPROTO_TCP, TCP_NODELAY, &chOpt, sizeof(char));
//if (nErr == -1)
//{
// wprintf(L"setsockopt() error %ld\n", WSAGetLastError());
// break;
//}
LPCONNECTED_SOCKET_DATA ConnectedSocketData = new CONNECTED_SOCKET_DATA;
ZeroMemory(ConnectedSocketData, sizeof(CONNECTED_SOCKET_DATA));
ConnectedSocketData->Socket = NewSocket;
ConnectedSocketData->Port = pSocketData->Port;
ConnectedSocketData->IOCP = pSocketData->IOCP;
ConnectedSocketData->CfgHandle = pSocketData->CfgHandle;
ConnectedSocketData->ForwardMessager = pSocketData->ForwardMessager;
//Add the new socket to the completion port, message from the socker will be queued up for proccessing by worker threads.
if (CreateIoCompletionPort((HANDLE)NewSocket, pSocketData->IOCP, (DWORD_PTR)ConnectedSocketData, 0) == NULL)
{
wprintf(L"CreateIOCompletionPort error %ld\n", WSAGetLastError());
delete ConnectedSocketData;
ConnectedSocketData = NULL;
closesocket(NewSocket);
break;
}
//Set the PerIOData, will be used at completion time
LPPER_IO_OPERATION_DATA PerIoData;
PerIoData = (LPPER_IO_OPERATION_DATA)GlobalAlloc(GPTR, sizeof(PER_IO_OPERATION_DATA));
ZeroMemory(&(PerIoData->overlapped), sizeof(OVERLAPPED));
PerIoData->BufferLen = 0;
PerIoData->OperationType = OPERATION_TYPE_RECV;
DWORD RecvBytes = 0;
DWORD Flags = 0;
PerIoData->Buffer.buf = PerIoData->cBuffer;
PerIoData->Buffer.len = DATA_BUFSIZE;
//Kick off the first Recv request for the Socket, will be handled by the completion Queue.
if (WSARecv(NewSocket, &(PerIoData->Buffer), 1, &RecvBytes, &Flags, &(PerIoData->overlapped), NULL) == SOCKET_ERROR)
{
wprintf(L"WSARecv error %ld\n", WSAGetLastError());
return 0;
}
}
else
{
wprintf(L"Unknown network event error %ld\n", WSAGetLastError());
break;
}
}
}
}
NetworkHandlerThread::NetworkHandlerThread()
{
m_CompletionPort = 0;
m_hListenThread = 0;
}
NetworkHandlerThread::~NetworkHandlerThread()
{
}
void NetworkHandlerThread::StartNetworkHandler()
{
int iResult = 0;
SYSTEM_INFO SystemInfo;
unsigned int i = 0;
//Start WSA
iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
if (iResult != NO_ERROR) {
wprintf(L"WSAStartup() failed with error: %d\n", iResult);
return;
}
//Start Completion Port
m_CompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
if (m_CompletionPort != NULL)
{
wprintf(L"Completion Port Created\n");
}
//Get # of system processors
GetSystemInfo(&SystemInfo);
//create Worker Threads for each processor.
for (i = 0; i < SystemInfo.dwNumberOfProcessors * THREADS_PER_PROCESSOR; i++)
{
HANDLE ThreadHandle;
// Create a server worker thread, and pass the
// completion port to the thread.
ThreadHandle = CreateThread(NULL, 0, ServerWorkerThread, m_CompletionPort, 0, NULL);
// Close the thread handle
if (ThreadHandle != NULL)
{
CloseHandle(ThreadHandle);
}
}
}
void NetworkHandlerThread::AddListenThread(int Port,
ConfigHandler* pConfigHandle,
void* ForwardHandle)
{
SOCKADDR_IN InternetAddr;
int iResult = 0;
LPLISTEN_SOCKET_DATA pListenSocketData = new LISTEN_SOCKET_DATA;
if (pListenSocketData == NULL)
{
return;
}
//Create the listener Socket
pListenSocketData->Socket = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
if (pListenSocketData->Socket == INVALID_SOCKET)
{
wprintf(L"socket function failed with error: %ld\n", WSAGetLastError());
WSACleanup();
return;
}
// Create a Event to handle Socket Accepts
pListenSocketData->hAcceptEvent = WSACreateEvent();
if (pListenSocketData->hAcceptEvent == WSA_INVALID_EVENT)
{
wprintf(L"WSACreateEvent() error %ld\n", WSAGetLastError());
closesocket(pListenSocketData->Socket);
return;
}
// Set the Event to Trigger on FD_ACCEPT (this occurs on socket connection attempts)
int nRet = WSAEventSelect(pListenSocketData->Socket,
pListenSocketData->hAcceptEvent,
FD_ACCEPT);
if (nRet == SOCKET_ERROR)
{
wprintf(L"WSAAsyncSelect() error %ld\n", WSAGetLastError());
closesocket(pListenSocketData->Socket);
return;
}
//Assign the Port Number
InternetAddr.sin_family = AF_INET;
InternetAddr.sin_addr.s_addr = htonl(INADDR_ANY);
InternetAddr.sin_port = htons(Port);
pListenSocketData->Port = Port;
pListenSocketData->IOCP = m_CompletionPort;
pListenSocketData->CfgHandle = pConfigHandle;
pListenSocketData->ForwardMessager = ForwardHandle;
//Bind the Socket to the Port
iResult = ::bind((pListenSocketData->Socket), (sockaddr*)&InternetAddr, sizeof(InternetAddr));
if (iResult == SOCKET_ERROR) {
wprintf(L"bind function failed with error %d\n", WSAGetLastError());
iResult = closesocket(pListenSocketData->Socket);
if (iResult == SOCKET_ERROR)
wprintf(L"closesocket function failed with error %d\n", WSAGetLastError());
WSACleanup();
return;
}
//Listen for incoming connection requests.
if (listen(pListenSocketData->Socket, SOMAXCONN) == SOCKET_ERROR)
{
wprintf(L"listen function failed with error: %d\n", WSAGetLastError());
closesocket(pListenSocketData->Socket);
WSACleanup();
return;
}
wprintf(L"Listening on %ld", Port);
m_hListenThread = (HANDLE)CreateThread(NULL, // Security
0, // Stack size - use default
ListenThread, // Thread fn entry point
(void*)pListenSocketData, //Listen Socket Data
0, // Init flag
NULL); // Thread address
}
NetworkHandlerThread.h
#pragma once
#include <WinSock2.h>
#include <ws2tcpip.h>
#include <stdio.h>
#include "ForwardMessageHandler.h"
#include "ConfigHandler.h"
#include "Type1MessageParser.h"
#include "Type2Message-Parser.h"
#include "ThreadUtilities.h"
#define DATA_BUFSIZE 8192
#define THREADS_PER_PROCESSOR 2
class NetworkHandlerThread
{
public:
WSADATA wsaData;
HANDLE m_CompletionPort;
HANDLE m_hListenThread;
public:
NetworkHandlerThread();
~NetworkHandlerThread();
void StartNetworkHandler();
void AddListenThread(int Port,
ConfigHandler* pConfigHandle,
void* ForwardHandle);
};
ThreadUtilities.h
#pragma once
#include <mutex>
#include "ConfigHandler.h"
using namespace std;
#define DATA_BUFSIZE 8192
#define THREADS_PER_PROCESSOR 2
typedef struct _THREAD_MESSAGE
{
mutex cmd_mtx;
string command;
} THREAD_MESSAGE, * LPTHREAD_MESSAGE;
typedef struct _LISTEN_SOCKET_DATA
{
SOCKET Socket;
int Port;
HANDLE hAcceptEvent;
HANDLE IOCP;
VOID* ForwardMessager;
ConfigHandler* CfgHandle;
// Other information useful to be associated with the handle
} LISTEN_SOCKET_DATA, * LPLISTEN_SOCKET_DATA;
typedef struct _CONNECTED_SOCKET_DATA
{
SOCKET Socket;
int Port;
HANDLE IOCP;
VOID* ForwardMessager;
ConfigHandler* CfgHandle;
} CONNECTED_SOCKET_DATA, * LPCONNECTED_SOCKET_DATA;
#define OPERATION_TYPE_UNKNOWN 0
#define OPERATION_TYPE_SEND 1
#define OPERATION_TYPE_RECV 2
typedef struct PER_IO_OPERATION_DATA
{
OVERLAPPED overlapped;
WSABUF Buffer;
char cBuffer[DATA_BUFSIZE];
int BufferLen;
int OperationType;
string PacketName;
};
#define LPPER_IO_OPERATION_DATA PER_IO_OPERATION_DATA
So, dumb mistake after looking at code for to long(caught it right away after a good nights sleep), Recv getting hex data in, first character is 0x00. When looking at it in debugger shows up as empty text string, when examined further, all the bytes are in the buffer.
Above is a good example of a working IOCP socket so I'm going to leave this here for people reference.
I'm writing a tcp proxy and while it seem to work it leaves a memory leak behind. I manipulated the code to forward the incoming packet to itself to create 10000 sockets and close them to see where the leak is. However I can't figure it out. I've used deleaker and it doesn't shows any leak(besides a small one that I don't care.)
But then I untick the two boxes and this comes out.
Any help would be appreciated!
Code:
#include <winsock2.h>
#include <stdio.h>
#include <windows.h>
#include <ws2tcpip.h>
#include <tchar.h>
#include <process.h> /* _beginthread() */
// Need to link with Ws2_32.lib
#pragma comment(lib, "Ws2_32.lib")
#define PORT "1234" /* Port to listen on */
#define BUF_SIZE 4096 /* Buffer for transfers */
typedef struct {
char *host;
char *port;
SOCKET sock;
}
HandleStruct;
unsigned int S2C(SOCKET from, SOCKET to)
{
char buf[BUF_SIZE];
unsigned int disconnected = 0;
size_t bytes_read, bytes_written;
bytes_read = recv(from, buf, BUF_SIZE, 0);
if (bytes_read == 0) {
disconnected = 1;
}
else {
bytes_written = send(to, buf, bytes_read, 0);
if (bytes_written == -1) {
disconnected = 1;
}
}
return disconnected;
}
unsigned int C2S(SOCKET from, SOCKET to)
{
char buf[BUF_SIZE];
unsigned int disconnected = 0;
size_t bytes_read, bytes_written;
bytes_read = recv(from, buf, BUF_SIZE, 0);
if (bytes_read == 0) {
disconnected = 1;
}
else {
bytes_written = send(to, buf, bytes_read, 0);
if (bytes_written == -1) {
disconnected = 1;
}
}
return disconnected;
}
void handle(void *param)
{
HandleStruct *args = (HandleStruct*) param;
SOCKET client = args->sock;
const char *host = args->host;
const char *port = args->port;
SOCKET server = -1;
unsigned int disconnected = 0;
fd_set set;
unsigned int max_sock;
struct addrinfo *res = NULL;
struct addrinfo *ptr = NULL;
struct addrinfo hints;
/* Get the address info */
ZeroMemory( &hints, sizeof(hints) );
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
if (getaddrinfo(host, port, &hints, &res) != 0) {
perror("getaddrinfo");
closesocket(client);
return;
}
/* Create the socket */
server = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
if (server == INVALID_SOCKET) {
perror("socket");
closesocket(client);
return;
}
/* Connect to the host */
if (connect(server, res->ai_addr, res->ai_addrlen) == -1) {
perror("connect");
closesocket(client);
return;
}
if (client > server) {
max_sock = client;
}
else {
max_sock = server;
}
/* Main transfer loop */
while (!disconnected) {
FD_ZERO(&set);
FD_SET(client, &set);
FD_SET(server, &set);
if (select(max_sock + 1, &set, NULL, NULL, NULL) == SOCKET_ERROR) {
perror("select");
break;
}
if (FD_ISSET(client, &set)) {
disconnected = C2S(client, server);
}
if (FD_ISSET(server, &set)) {
disconnected = S2C(server, client);
}
}
closesocket(server);
closesocket(client);
fprintf(stderr, "Sockets Closed: %d/%d", server, client);
_endthread();
return;
}
int _tmain(int argc)
{
WORD wVersion = MAKEWORD(2, 2);
WSADATA wsaData;
int iResult;
SOCKET sock;
struct addrinfo hints, *res;
int reuseaddr = 1; /* True */
/* Initialise Winsock */
if (iResult = (WSAStartup(wVersion, &wsaData)) != 0) {
fprintf(stderr, "WSAStartup failed: %dn", iResult);
return 1;
}
char * host = "127.0.0.1";
char * port = "1234";
/* Get the address info */
ZeroMemory(&hints, sizeof hints);
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if (getaddrinfo(NULL, PORT, &hints, &res) != 0) {
perror("getaddrinfo");
WSACleanup();
return 1;
}
/* Create the socket */
sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
if (sock == INVALID_SOCKET) {
perror("socket");
WSACleanup();
return 1;
}
/* Enable the socket to reuse the address */
if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char*)&reuseaddr,
sizeof(int)) == SOCKET_ERROR) {
perror("setsockopt");
WSACleanup();
return 1;
}
/* Bind to the address */
if (bind(sock, res->ai_addr, res->ai_addrlen) == SOCKET_ERROR) {
perror("bind");
WSACleanup();
return 1;
}
/* Listen */
if (listen(sock, 6500) == SOCKET_ERROR) {
perror("listen");
WSACleanup();
return 1;
}
freeaddrinfo(res);
int i = 0;
HandleStruct *arg;
arg = (HandleStruct *)malloc(sizeof( HandleStruct));
/* Main loop */
while(1) {
int size = sizeof(struct sockaddr);
struct sockaddr_in their_addr;
SOCKET newsock;
ZeroMemory(&their_addr, sizeof (struct sockaddr));
newsock = accept(sock, (struct sockaddr*)&their_addr, &size);
if (newsock == INVALID_SOCKET) {
perror("acceptn");
}
else {
arg->sock = newsock;
arg->host = host;
arg->port = port;
if (i < 10000) {
_beginthread(handle, 0, (void*) arg);
i++;
}
}
}
closesocket(sock);
WSACleanup();
return 0;
}
I'm not familiar with reading the program in the screenshots you posted; however, you should probably be concerned about this line:
arg = (HandleStruct *)malloc(sizeof( HandleStruct));
Here you are allocating memory for a HandleStruct via malloc() which doesn't appear to be cleaned up anywhere with a subsequent call to free(). You pass arg into handle() but still don't deallocate the memory.
It doesn't appear to be handle()'s responsibility to clean arg up, so you should probably have a call to free() after the while loop, or you could allocate the HandleStruct at the beginning of each loop and deallocate it at the end.
Or you could save yourself the hassle and use std::unique_ptr, and optionally change your threads to std::thread, which self-documents who owns the memory etc:
void handle(std::unique_ptr<HandleStruct> args)
{
// Manipulate args
...
}
int main()
{
std::unique_ptr<HandleStruct> pHandle = std::make_unique<HandleStruct>();
for (;;)
{
...
pHandle->sock = newsock;
pHandle->host = host;
pHandle->port = port;
// Create thread:
std::thread t(&handle, pHandle);
// Wait for thread to finish so pHandle doesn't change while we are using it on another thread
// t.join();
}
}
Every socket uses some memory in the operating system.
Here the description in Linux : accept
ENOBUFS, ENOMEM
Not enough free memory. This often means that the memory
allocation is limited by the socket buffer limits, not by the
system memory.
The OS might not clean them up.
You are also trying to create 10000 threads, these might also take some memory, if the creation doesn't fail long before with errno set to EAGAIN.
I'm playing around with Overlapped IO and suddenly found out that it looks like I'm the only one who can't encourage Completion callback to work (All claims was about: it works and I don't like it).
The idea of my application is: a client (telnet localhost 27015) connects to the server and server starts pushing huge amount of data to the the client. And I've never had CompletionCallback called.
Here is the code:
#include <winsock2.h>
#include <ws2tcpip.h>
#include <atomic>
#pragma comment(lib, "ws2_32.lib")
#define DATA_BUFSIZE 16384
class CSync
{
private:
CRITICAL_SECTION m_cs;
public:
CSync() { ZeroMemory(&m_cs, sizeof(m_cs)); InitializeCriticalSection(&m_cs); }
~CSync() { DeleteCriticalSection(&m_cs); ZeroMemory(&m_cs, sizeof(m_cs)); }
inline void Lock() { EnterCriticalSection(&m_cs); }
inline void Unlock() { LeaveCriticalSection(&m_cs); }
inline BOOL WINAPI TryLock() { return TryEnterCriticalSection(&m_cs); }
};
class ScopedLock
{
public:
ScopedLock(CSync& lock) : m_lock(lock) { m_lock.Lock(); }
~ScopedLock() { m_lock.Unlock(); }
private:
CSync m_lock;
};
class SendServer
{
private:
SOCKET socket;
// std::atomic<bool> busy;
char buffer[2][DATA_BUFSIZE];
CSync syncer;
WSABUF wsabuf;
OVERLAPPED overlapped;
// HANDLE socketEvent;
std::atomic_flag busy;
char toBuffer; // 0 or 1
DWORD sent;
public:
SendServer(SOCKET& sock);
virtual ~SendServer();
bool Write(char* buff);
};
static void __stdcall Produce(SendServer *server);
void CALLBACK CompletionCallback(DWORD dwError, DWORD cbTransferred, LPWSAOVERLAPPED lpOverlapped, DWORD dwFlags);
static bool run = 1;
int main(int argc, char* argv[])
{
WSADATA wsd;
struct addrinfo *result = NULL;
struct addrinfo hints;
SOCKET ListenSocket = INVALID_SOCKET;
SOCKET AcceptSocket = INVALID_SOCKET;
int err = 0;
int rc;
// Load Winsock
rc = WSAStartup((2, 2), &wsd);
if (rc != 0) {
printf("Unable to load Winsock: %d\n", rc);
return 1;
}
// Make sure the hints struct is zeroed out
SecureZeroMemory((PVOID)& hints, sizeof(struct addrinfo));
// Initialize the hints to obtain the
// wildcard bind address for IPv4
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
hints.ai_flags = AI_PASSIVE;
rc = getaddrinfo(NULL, "27015", &hints, &result);
if (rc != 0) {
printf("getaddrinfo failed with error: %d\n", rc);
return 1;
}
ListenSocket = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
//socket(result->ai_family, result->ai_socktype, result->ai_protocol);
if (ListenSocket == INVALID_SOCKET) {
printf("socket failed with error: %d\n", WSAGetLastError());
freeaddrinfo(result);
return 1;
}
rc = bind(ListenSocket, result->ai_addr, (int)result->ai_addrlen);
if (rc == SOCKET_ERROR) {
printf("bind failed with error: %d\n", WSAGetLastError());
freeaddrinfo(result);
closesocket(ListenSocket);
return 1;
}
rc = listen(ListenSocket, 1);
if (rc == SOCKET_ERROR) {
printf("listen failed with error: %d\n", WSAGetLastError());
freeaddrinfo(result);
closesocket(ListenSocket);
return 1;
}
// Accept an incoming connection request
AcceptSocket = accept(ListenSocket, NULL, NULL);
if (AcceptSocket == INVALID_SOCKET) {
printf("accept failed with error: %d\n", WSAGetLastError());
freeaddrinfo(result);
closesocket(ListenSocket);
return 1;
}
printf("Client Accepted...\n");
SendServer server(AcceptSocket);
HANDLE h[1];
h[0] = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)&Produce, &server, 0, NULL);
getchar();
run = 0;
WaitForMultipleObjects(1, h, TRUE, INFINITE);
return 0;
}
void __stdcall Produce(SendServer *server)
{
char buf[] = "------------------------------------------------------------------------------------------";
char s = 0;
while (run) {
buf[0] = '0' + s++;
if (s > 9)
s = 0;
server->Write(buf);
Sleep(10);
}
}
void CALLBACK CompletionCallback(DWORD dwError, DWORD cbTransferred, LPWSAOVERLAPPED lpOverlapped, DWORD dwFlags)
{
((SendServer*)(lpOverlapped->hEvent))->Write(NULL);
}
SendServer::SendServer(SOCKET& sock) : toBuffer(0)
{
socket = sock;
ZeroMemory(buffer, DATA_BUFSIZE << 1);
busy.clear();
}
SendServer::~SendServer()
{
shutdown(socket, 2);
closesocket(socket);
}
bool SendServer::Write(char* buff)
{
ScopedLock lock(syncer);
int size = strlen(buffer[toBuffer]), toAdd = 0;
if (buff == NULL) {
busy.clear();
SecureZeroMemory(buffer[!toBuffer], DATA_BUFSIZE);
}
else {
toAdd = strlen(buff);
if (size + toAdd < DATA_BUFSIZE) {
memcpy_s(buffer[toBuffer] + size, toAdd, buff, toAdd);
size += toAdd;
buffer[toBuffer][size] = 0;
return TRUE;
}
else {
printf("\nCan't add anymore!\n");
}
}
if (size > 0 && !busy.test_and_set()) {
wsabuf.buf = (char*)buffer[toBuffer];
wsabuf.len = size;
SecureZeroMemory(&overlapped, sizeof OVERLAPPED);
overlapped.hEvent = this;
toBuffer = !toBuffer;
size = WSASend(socket, &wsabuf, 1, &sent, 0, &overlapped, CompletionCallback);
if (size == 0) {
//return Write(NULL);
}
if (WSA_IO_PENDING != WSAGetLastError()) {
return FALSE;
}
}
return TRUE;
}
Thank you.
Completion callbacks are invoked during alertable wait. You have no alertable wait, so the completion callbacks get queued up but never get a chance to run.
Change WaitForMultipleObjects to a loop with WaitForMultipleObjectsEx and Sleep to SleepEx, and pass TRUE as the bAlertable parameter.
This is explained right in the WSASend documentation
The completion routine follows the same rules as stipulated for Windows file I/O completion routines. The completion routine will not be invoked until the thread is in an alertable wait state such as can occur when the function WSAWaitForMultipleEvents with the fAlertable parameter set to TRUE is invoked.
I’ve recently started learning IOCP on Windows and been reading the following article:
http://www.codeproject.com/Tips/95363/Another-TCP-echo-server-using-IOCP
You can download the sample for the article from:
http://dl.dropbox.com/u/281215/documentation/iocp-1.00.html
The sample contains two simple applications – iocp_echo_server and TcpEchoClient.
I understand that IOCP is usually used on the server side of the client/server model but I’d like to create a client using IOCP.
I’ve so far tried modifying the client sample above so that whenever the server sends a response to the client, it gets picked up automatically, however it doesn’t work.
I’ve left iocp_echo_server.c as is. My modified version of TcpEchoClient.c looks like:
//TcpEchoClient.c - a minimalistic echo client
// -----------------------------------------------------------------------------
// C language includes
#include <stdio.h>
#include <winsock2.h>
#include "mswsock.h" // for AcceptEx
#include <stdlib.h> // exit
#include <string.h>
// Windows includes
#include <windows.h>
#pragma warning(disable: 4996) // sprintf
// -----------------------------------------------------------------------------
// configuration
enum
{
BUFLEN = 1000,
SERVICE_PORT = 4000,
SERVER_ADDRESS = INADDR_LOOPBACK
};
enum // socket operations
{
OP_NONE,
OP_ACCEPT,
OP_READ,
OP_WRITE
};
typedef struct _SocketState // socket state & control
{
char operation;
SOCKET socket;
DWORD length;
char buf[1024];
} SocketState;
// variables
static HANDLE cpl_port;
static SOCKET sock;
static SocketState sock_state;
static WSAOVERLAPPED sock_ovl;
static LPFN_ACCEPTEX pfAcceptEx;
static GUID GuidAcceptEx = WSAID_ACCEPTEX;
static int msgNumber;
static char msgBuf[BUFLEN];
static struct sockaddr_in sin;
// prototypes
static void createConnection(void);
static void createSocket(void);
static void init(void);
static void initWinsock(void);
static void prepareEndpoint(void);
static void recvBuffer(void);
static void run(void);
static void sendBuffer(void);
static SOCKET create_accepting_socket(void);
static void create_io_completion_port(void);
static BOOL get_completion_status(DWORD*, SocketState**,WSAOVERLAPPED**);
// -----------------------------------------------------------------------------
void main(void)
{
init();
run();
}
// -----------------------------------------------------------------------------
static void createConnection(void)
{
printf("* connecting\n");
if (WSAConnect(sock, (LPSOCKADDR)&sin, sizeof(sin), NULL, NULL, NULL, NULL) == SOCKET_ERROR)
{
int err = WSAGetLastError();
printf("* error %d in connect\n", err);
exit(1);
}
printf("* connected\n");
}
// -----------------------------------------------------------------------------
static void createSocket(void)
{
sock = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
if (sock == INVALID_SOCKET)
{
int err = WSAGetLastError();
printf("* error %d creating socket\n", err);
exit(1);
}
// for use by AcceptEx
sock_state.socket = 0; // to be updated later
sock_state.operation = OP_ACCEPT;
if (CreateIoCompletionPort((HANDLE)sock, cpl_port, (ULONG_PTR)&sock_state, 0) != cpl_port)
{
int err = WSAGetLastError();
printf("* error %d in listener\n", err);
exit(1);
}
}
// -----------------------------------------------------------------------------
static void init(void)
{
initWinsock();
create_io_completion_port();
createSocket();
prepareEndpoint();
createConnection();
}
// -----------------------------------------------------------------------------
static void initWinsock(void)
{
WSADATA wsaData;
if (WSAStartup(0x202, &wsaData) == SOCKET_ERROR)
{
int err = WSAGetLastError();
printf("* error %d in WSAStartup\n", err);
exit(1);
}
}
// -----------------------------------------------------------------------------
static void prepareEndpoint(void)
{
sin.sin_family = AF_INET;
sin.sin_addr.s_addr = htonl(SERVER_ADDRESS);
sin.sin_port = htons(SERVICE_PORT);
// bind_listening_socket()
{
//if (bind(sock, (SOCKADDR*)&sin, sizeof(sin)) == SOCKET_ERROR)
//{
// printf("* error in bind!\n");
// exit(1);
//}
}
// start_listening()
{
//if (listen(sock, 100) == SOCKET_ERROR)
//{
// printf("* error in listen!\n");
// exit(1);
//}
//printf("* started listening for connection requests...\n");
}
// load_accept_ex()
{
//DWORD dwBytes;
// black magic for me!!!
// You do not need to call in your code WSAIoctl. You can directly use AcceptEx and adds Mswsock.lib.
//WSAIoctl(sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &GuidAcceptEx, sizeof(GuidAcceptEx), &pfAcceptEx, sizeof(pfAcceptEx), &dwBytes, NULL, NULL);
}
// start_accepting()
{
//SOCKET acceptor = create_accepting_socket();
//DWORD expected = sizeof(struct sockaddr_in) + 16;
//printf("* started accepting connections...\n");
// uses listener's completion key and overlapped structure
//sock_state.socket = acceptor;
//memset(&sock_ovl, 0, sizeof(WSAOVERLAPPED));
// starts asynchronous accept
//if (!pfAcceptEx(sock, acceptor, sock_state.buf, 0 /* no recv */, expected, expected, NULL, &sock_ovl))
//{
// int err = WSAGetLastError();
// if (err != ERROR_IO_PENDING)
// {
// printf("* error %d in AcceptEx\n", err);
// exit(1);
// }
//}
}
}
// -----------------------------------------------------------------------------
static void recvBuffer(void)
{
char* buf = msgBuf;
int pendingLen = BUFLEN;
printf("* receiving reply\n");
while (pendingLen > 0)
{
int partialLen = recv(sock, buf, pendingLen, 0);
if (partialLen > 0)
{
pendingLen -= partialLen;
buf += partialLen;
continue;
}
// ------
if (partialLen == 0)
{
printf("* connection closed by the server\n");
}
else // partialLen < 0
{
int err = WSAGetLastError();
printf("* error %d in recv\n", err);
}
exit(1);
}
}
// -----------------------------------------------------------------------------
static void run(void)
{
DWORD length;
BOOL resultOk;
WSAOVERLAPPED* ovl_res;
SocketState* socketState;
for (;;)
{
sendBuffer();
resultOk = get_completion_status(&length, &socketState, &ovl_res);
recvBuffer();
}
}
// -----------------------------------------------------------------------------
static void sendBuffer(void)
{
char* buf = msgBuf;
int pendingLen = BUFLEN;
printf("* sending message\n");
sprintf(msgBuf, "%05 *****", msgNumber++);
while (pendingLen > 0)
{
int partialLen = send(sock, buf, pendingLen, 0);
if (partialLen > 0)
{
pendingLen -= partialLen;
buf += partialLen;
continue;
}
// -----------
if (partialLen == 0)
{
printf("* connection closed by the server\n");
}
else // partialLen < 0
{
int err = WSAGetLastError();
printf("* error %d in send\n", err);
}
exit(1);
}
}
// -----------------------------------------------------------------------------
static SOCKET create_accepting_socket(void)
{
SOCKET acceptor = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (acceptor == INVALID_SOCKET)
{
printf("* error creating accept socket!\n");
exit(1);
}
return acceptor;
}
// -----------------------------------------------------------------------------
static void create_io_completion_port(void)
{
cpl_port = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
if (!cpl_port)
{
int err = WSAGetLastError();
printf("* error %d in line %d CreateIoCompletionPort\n", err, __LINE__);
exit(1);
}
}
// -----------------------------------------------------------------------------
static BOOL get_completion_status(DWORD* length, SocketState** socketState, WSAOVERLAPPED** ovl_res)
{
BOOL resultOk;
*ovl_res = NULL;
*socketState = NULL;
resultOk = GetQueuedCompletionStatus(cpl_port, length, (PULONG_PTR)socketState, ovl_res, INFINITE);
if (!resultOk)
{
DWORD err = GetLastError();
printf("* error %d getting completion port status!!!\n", err);
}
if (!*socketState || !*ovl_res)
{
printf("* don't know what to do, aborting!!!\n");
exit(1);
}
return resultOk;
}
// -----------------------------------------------------------------------------
// the end
When the server send a response by calling:
WSASend(socketState->socket, &wsabuf, 1, NULL, 0, ovl, NULL)
I’d expect it to be picked up by the client on this line:
resultOk = get_completion_status(&length, &socketState, &ovl_res);
But it doesn’t…
Would anybody be able to tell me what I’m doing wrong?
Edit:
I’ve taken the following points:
On the client side, you use WSAConnect() to create an outbound connection.
Call WSARecv() and WSASend() to start reading/writing operations when needed
you have to use WSASend/WSARecv if you want to use I/O completion ports.
and attempted to create a simple IOCP based client:
#include <iostream>
#include <winsock2.h>
#pragma comment(lib,"ws2_32.lib")
static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter);
int main(void)
{
WSADATA WsaDat;
if (WSAStartup(MAKEWORD(2, 2), &WsaDat) != NO_ERROR)
return 0;
// Step 1 - Create an I/O completion port.
HANDLE hCompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
if (!hCompletionPort)
return 0;
// Step 2 - Find how many processors.
SYSTEM_INFO systemInfo;
GetSystemInfo(&systemInfo);
const int nNumberOfProcessors = systemInfo.dwNumberOfProcessors;
// Step 3 - Create worker threads.
for (int i = 0; i < nNumberOfProcessors; i++)
{
HANDLE hThread = CreateThread(NULL, 0, ClientWorkerThread, hCompletionPort, 0, NULL);
CloseHandle(hThread);
}
// Step 4 - Create a socket.
SOCKET Socket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
if (Socket == INVALID_SOCKET)
return 0;
struct hostent *host;
if ((host = gethostbyname("localhost")) == NULL)
return 0;
SOCKADDR_IN SockAddr;
SockAddr.sin_family = AF_INET;
SockAddr.sin_addr.s_addr = *((unsigned long*)host->h_addr);
SockAddr.sin_port = htons(8888);
// Step 5 - Associate the socket with the I/O completion port.
CreateIoCompletionPort((HANDLE)Socket, hCompletionPort, (ULONG_PTR)0, 0);
if (WSAConnect(Socket, (SOCKADDR*)(&SockAddr), sizeof(SockAddr), NULL, NULL, NULL, NULL) == SOCKET_ERROR)
return 0;
char buffer[1000];
memset(buffer, 0, 999);
WSABUF wsaBuf = {strlen(buffer), buffer};
DWORD dwSendBytes = 0;
DWORD dwReceivedBytes = 0;
DWORD dwFlags = 0;
WSAOVERLAPPED wsaOverlapped;
SecureZeroMemory((PVOID)&wsaOverlapped, sizeof(wsaOverlapped));
wsaOverlapped.hEvent = WSACreateEvent();
for(;;)
{
WSARecv(Socket, &wsaBuf, 1, &dwReceivedBytes, &dwFlags, &wsaOverlapped, NULL);
std::cout << wsaBuf.buf;
//WSASend(Socket, &wsaBuf, 1, &dwSendBytes, 0, &wsaOverlapped, NULL);
int nError = WSAGetLastError();
if(nError != WSAEWOULDBLOCK&&nError != 0)
{
std::cout << "Winsock error code: " << nError << "\r\n";
std::cout << "Server disconnected!\r\n";
shutdown(Socket, SD_SEND);
closesocket(Socket);
break;
}
Sleep(1000);
}
WSACleanup();
system("PAUSE");
return 0;
}
static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter)
{
HANDLE hCompletionPort = (HANDLE)lpParameter;
DWORD dwBytesTransferred = 0;
while (TRUE)
{
BOOL bRet = GetQueuedCompletionStatus(hCompletionPort, &dwBytesTransferred, (LPDWORD)0, (LPOVERLAPPED*)0, INFINITE);
}
return 0;
}
I know there are several things I’m doing wrong but I don’t know what they are.
Could somebody take a look at my code and give me some hints please?
Many thanks
Edit 2:
Sorry this post is getting too long.
I've had another go trying to implement an IOCP based client after reading Remy's comments below but I'm still not sure if I'm on the right track.
I'd really appreciate it if somebody could take a look at my new code (compiles fine under VS2010 & error checking omitted) below and give me some feedback.
NonBlockingClient:
#include <iostream>
#include <winsock2.h>
#pragma comment(lib, "ws2_32.lib")
static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter);
typedef struct _PER_HANDLE_DATA
{
SOCKET Socket;
} PER_HANDLE_DATA, * LPPER_HANDLE_DATA;
typedef struct
{
WSAOVERLAPPED wsaOverlapped;
WSABUF wsaBuf;
int OperationType;
} PER_IO_DATA, * LPPER_IO_DATA;
int main(void)
{
WSADATA WsaDat;
WSAStartup(MAKEWORD(2, 2), &WsaDat);
// Step 1 - Create an I/O completion port.
HANDLE hCompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
// Step 2 - Find how many processors.
SYSTEM_INFO systemInfo;
GetSystemInfo(&systemInfo);
// Step 3 - Create worker threads.
for (int i = 0; i < (int)systemInfo.dwNumberOfProcessors; i++)
{
HANDLE hThread = CreateThread(NULL, 0, ClientWorkerThread, hCompletionPort, 0, NULL);
CloseHandle(hThread);
}
// Step 4 - Create a socket.
SOCKET Socket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
PER_HANDLE_DATA *pPerHandleData = new PER_HANDLE_DATA;
pPerHandleData->Socket = Socket;
struct hostent *host;
host = gethostbyname("localhost");
SOCKADDR_IN SockAddr;
SockAddr.sin_family = AF_INET;
SockAddr.sin_addr.s_addr = *((unsigned long*)host->h_addr);
SockAddr.sin_port = htons(8888);
// Step 5 - Associate the socket with the I/O completion port.
CreateIoCompletionPort((HANDLE)Socket, hCompletionPort, (DWORD)pPerHandleData, 0);
WSAConnect(Socket, (SOCKADDR*)(&SockAddr), sizeof(SockAddr), NULL, NULL, NULL, NULL);
static char buffer[1000];
memset(buffer, 0, 999);
PER_IO_DATA *pPerIoData = new PER_IO_DATA;
pPerIoData->wsaBuf.buf = buffer;
pPerIoData->wsaBuf.len = sizeof(buffer);
DWORD dwSendBytes = 0;
DWORD dwReceivedBytes = 0;
DWORD dwFlags = 0;
SecureZeroMemory((PVOID)&pPerIoData->wsaOverlapped, sizeof(pPerIoData->wsaOverlapped));
pPerIoData->wsaOverlapped.hEvent = WSACreateEvent();
WSARecv(Socket, &pPerIoData->wsaBuf, 1, &dwReceivedBytes, &dwFlags, &pPerIoData->wsaOverlapped, NULL);
std::cout << pPerIoData->wsaBuf.buf;
for (;;)
{
int nError = WSAGetLastError();
if (nError != WSAEWOULDBLOCK&&nError != 0)
{
std::cout << "Winsock error code: " << nError << "\r\n";
std::cout << "Server disconnected!\r\n";
shutdown(Socket, SD_SEND);
closesocket(Socket);
break;
}
Sleep(1000);
}
delete pPerHandleData;
delete pPerIoData;
WSACleanup();
return 0;
}
static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter)
{
HANDLE hCompletionPort = (HANDLE)lpParameter;
DWORD bytesCopied = 0;
OVERLAPPED *overlapped = 0;
LPPER_HANDLE_DATA PerHandleData;
LPPER_IO_DATA PerIoData;
DWORD SendBytes, RecvBytes;
DWORD Flags;
BOOL bRet;
while (TRUE)
{
bRet = GetQueuedCompletionStatus(hCompletionPort, &bytesCopied, (LPDWORD)&PerHandleData, (LPOVERLAPPED*)&PerIoData, INFINITE);
if (bytesCopied == 0)
{
break;
}
else
{
Flags = 0;
ZeroMemory(&(PerIoData->wsaOverlapped), sizeof(WSAOVERLAPPED));
PerIoData->wsaBuf.len = 1000;
WSARecv(PerHandleData->Socket, &(PerIoData->wsaBuf), 1, &RecvBytes, &Flags, &(PerIoData->wsaOverlapped), NULL);
}
}
return 0;
}
NonBlockingServer:
#include <iostream>
#include <winsock2.h>
#pragma comment(lib,"ws2_32.lib")
int main()
{
WSADATA WsaDat;
WSAStartup(MAKEWORD(2,2), &WsaDat);
SOCKET listenSocket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
SOCKADDR_IN server;
server.sin_family = AF_INET;
server.sin_addr.s_addr = INADDR_ANY;
server.sin_port = htons(8888);
bind(listenSocket, (SOCKADDR*)(&server), sizeof(server));
listen(listenSocket, 1);
SOCKET acceptSocket = SOCKET_ERROR;
sockaddr_in saClient;
int nClientSize = sizeof(saClient);
while (acceptSocket == SOCKET_ERROR)
{
std::cout << "Waiting for incoming connections...\r\n";
acceptSocket = WSAAccept(listenSocket, (SOCKADDR*)&saClient, &nClientSize, NULL, NULL);
}
std::cout << "Client connected!\r\n\r\n";
char *szMessage = "Welcome to the server!\r\n";
WSAOVERLAPPED SendOverlapped;
DWORD SendBytes;
WSABUF DataBuf;
DataBuf.len = 1000;
DataBuf.buf = szMessage;
SecureZeroMemory((PVOID)&SendOverlapped, sizeof(WSAOVERLAPPED));
SendOverlapped.hEvent = WSACreateEvent();
for (;;)
{
WSASend(acceptSocket, &DataBuf, 1, &SendBytes, 0, &SendOverlapped, NULL);
int nError = WSAGetLastError();
if (nError != WSAEWOULDBLOCK && nError != 0)
{
std::cout << "Winsock error code: " << nError << "\r\n";
std::cout << "Client disconnected!\r\n";
shutdown(acceptSocket, SD_SEND);
closesocket(acceptSocket);
break;
}
Sleep(1000);
}
WSACleanup();
return 0;
}
Thanks again!
Try something like this:
Client:
#include <iostream>
#include <string>
#include <winsock2.h>
#pragma comment(lib, "ws2_32.lib")
typedef struct
{
WSAOVERLAPPED Overlapped;
SOCKET Socket;
WSABUF wsaBuf;
char Buffer[1024];
DWORD Flags;
} PER_IO_DATA, * LPPER_IO_DATA;
static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter)
{
HANDLE hCompletionPort = (HANDLE)lpParameter;
DWORD NumBytesRecv = 0;
ULONG CompletionKey;
LPPER_IO_DATA PerIoData;
while (GetQueuedCompletionStatus(hCompletionPort, &NumBytesRecv, &CompletionKey, (LPOVERLAPPED*)&PerIoData, INFINITE))
{
if (!PerIoData)
continue;
if (NumBytesRecv == 0)
{
std::cout << "Server disconnected!\r\n\r\n";
}
else
{
// use PerIoData->Buffer as needed...
std::cout << std::string(PerIoData->Buffer, NumBytesRecv);
PerIoData->wsaBuf.len = sizeof(PerIoData->Buffer);
PerIoData->Flags = 0;
if (WSARecv(PerIoData->Socket, &(PerIoData->wsaBuf), 1, &NumBytesRecv, &(PerIoData->Flags), &(PerIoData->Overlapped), NULL) == 0)
continue;
if (WSAGetLastError() == WSA_IO_PENDING)
continue;
}
closesocket(PerIoData->Socket);
delete PerIoData;
}
return 0;
}
int main(void)
{
WSADATA WsaDat;
if (WSAStartup(MAKEWORD(2, 2), &WsaDat) != 0)
return 0;
HANDLE hCompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
if (!hCompletionPort)
return 0;
SYSTEM_INFO systemInfo;
GetSystemInfo(&systemInfo);
for (DWORD i = 0; i < systemInfo.dwNumberOfProcessors; ++i)
{
HANDLE hThread = CreateThread(NULL, 0, ClientWorkerThread, hCompletionPort, 0, NULL);
CloseHandle(hThread);
}
SOCKET Socket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
if (Socket == INVALID_SOCKET)
return 0;
SOCKADDR_IN SockAddr;
SockAddr.sin_family = AF_INET;
SockAddr.sin_addr.s_addr = inet_addr("127.0.0.1");
SockAddr.sin_port = htons(8888);
CreateIoCompletionPort((HANDLE)Socket, hCompletionPort, 0, 0);
if (WSAConnect(Socket, (SOCKADDR*)(&SockAddr), sizeof(SockAddr), NULL, NULL, NULL, NULL) == SOCKET_ERROR)
return 0;
PER_IO_DATA *pPerIoData = new PER_IO_DATA;
ZeroMemory(pPerIoData, sizeof(PER_IO_DATA));
pPerIoData->Socket = Socket;
pPerIoData->Overlapped.hEvent = WSACreateEvent();
pPerIoData->wsaBuf.buf = pPerIoData->Buffer;
pPerIoData->wsaBuf.len = sizeof(pPerIoData->Buffer);
DWORD dwNumRecv;
if (WSARecv(Socket, &(pPerIoData->wsaBuf), 1, &dwNumRecv, &(pPerIoData->Flags), &(pPerIoData->Overlapped), NULL) == SOCKET_ERROR)
{
if (WSAGetLastError() != WSA_IO_PENDING)
{
delete pPerIoData;
return 0;
}
}
while (TRUE)
Sleep(1000);
shutdown(Socket, SD_BOTH);
closesocket(Socket);
WSACleanup();
return 0;
}
Server:
#include <iostream>
#include <winsock2.h>
#pragma comment(lib,"ws2_32.lib")
typedef struct
{
WSAOVERLAPPED Overlapped;
SOCKET Socket;
WSABUF wsaBuf;
char Buffer[1024];
DWORD BytesSent;
DWORD BytesToSend;
} PER_IO_DATA, * LPPER_IO_DATA;
static DWORD WINAPI ServerWorkerThread(LPVOID lpParameter)
{
HANDLE hCompletionPort = (HANDLE)lpParameter;
DWORD NumBytesSent = 0;
ULONG CompletionKey;
LPPER_IO_DATA PerIoData;
while (GetQueuedCompletionStatus(hCompletionPort, &NumBytesSent, &CompletionKey, (LPOVERLAPPED*)&PerIoData, INFINITE))
{
if (!PerIoData)
continue;
if (NumBytesSent == 0)
{
std::cout << "Client disconnected!\r\n\r\n";
}
else
{
PerIoData->BytesSent += NumBytesSent;
if (PerIoData->BytesSent < PerIoData->BytesToSend)
{
PerIoData->wsaBuf.buf = &(PerIoData->Buffer[PerIoData->BytesSent]);
PerIoData->wsaBuf.len = (PerIoData->BytesToSend - PerIoData->BytesSent);
}
else
{
PerIoData->wsaBuf.buf = PerIoData->Buffer;
PerIoData->wsaBuf.len = strlen(PerIoData->Buffer);
PerIoData->BytesSent = 0;
PerIoData->BytesToSend = PerIoData->wsaBuf.len;
}
if (WSASend(PerIoData->Socket, &(PerIoData->wsaBuf), 1, &NumBytesSent, 0, &(PerIoData->Overlapped), NULL) == 0)
continue;
if (WSAGetLastError() == WSA_IO_PENDING)
continue;
}
closesocket(PerIoData->Socket);
delete PerIoData;
}
return 0;
}
int main()
{
WSADATA WsaDat;
if (WSAStartup(MAKEWORD(2,2), &WsaDat) != 0)
return 0;
HANDLE hCompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
if (!hCompletionPort)
return 0;
SYSTEM_INFO systemInfo;
GetSystemInfo(&systemInfo);
for (DWORD i = 0; i < systemInfo.dwNumberOfProcessors; ++i)
{
HANDLE hThread = CreateThread(NULL, 0, ServerWorkerThread, hCompletionPort, 0, NULL);
CloseHandle(hThread);
}
SOCKET listenSocket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
if (listenSocket == INVALID_SOCKET)
return 0;
SOCKADDR_IN server;
ZeroMemory(&server, sizeof(server));
server.sin_family = AF_INET;
server.sin_addr.s_addr = INADDR_ANY;
server.sin_port = htons(8888);
if (bind(listenSocket, (SOCKADDR*)(&server), sizeof(server)) != 0)
return 0;
if (listen(listenSocket, 1) != 0)
return 0;
std::cout << "Waiting for incoming connection...\r\n";
SOCKET acceptSocket;
do
{
sockaddr_in saClient;
int nClientSize = sizeof(saClient);
acceptSocket = WSAAccept(listenSocket, (SOCKADDR*)&saClient, &nClientSize, NULL, NULL);
}
while (acceptSocket == INVALID_SOCKET);
std::cout << "Client connected!\r\n\r\n";
CreateIoCompletionPort((HANDLE)acceptSocket, hCompletionPort, 0, 0);
LPPER_IO_DATA pPerIoData = new PER_IO_DATA;
ZeroMemory(pPerIoData, sizeof(PER_IO_DATA));
strcpy(pPerIoData->Buffer, "Welcome to the server!\r\n");
pPerIoData->Overlapped.hEvent = WSACreateEvent();
pPerIoData->Socket = acceptSocket;
pPerIoData->wsaBuf.buf = pPerIoData->Buffer;
pPerIoData->wsaBuf.len = strlen(pPerIoData->Buffer);
pPerIoData->BytesToSend = pPerIoData->wsaBuf.len;
DWORD dwNumSent;
if (WSASend(acceptSocket, &(pPerIoData->wsaBuf), 1, &dwNumSent, 0, &(pPerIoData->Overlapped), NULL) == SOCKET_ERROR)
{
if (WSAGetLastError() != WSA_IO_PENDING)
{
delete pPerIoData;
return 0;
}
}
while (TRUE)
Sleep(1000);
shutdown(acceptSocket, SD_BOTH);
closesocket(acceptSocket);
WSACleanup();
return 0;
}
Have you had a look at the example in the MSDN documentation for WSARecv?
Essentially, you have to start the asynchronous WSARecv operation first and then get notified via the completion port of its completion.
Or to put it another way: Windows I/O completion ports are using a proactor model (in contrast to the reactor model of Linux/FreeBSD/NetBSD).