Well i make a IOCP for handling client connections with the following details:
- Threads = (CPU cores * 2)
- Assigning an completion port to each socket
- Accessing the socket context by Client Index or overlapped struct (either way is the same)
So i am trying to debug the incoming packets, its works like a charm, except for a little but nasty detail... I set a break point on WorkersThread function (where i recv the packet) i am watching the buffer with the packet i recv, when suddenly the buffer gets overwritten with a new packet that i got from client.
Why is that? according to what i read, IOCP should wait till i process the packet, send a response to client before recv any other packet. So i set a flag on my socket context called "Processing" and still got the overwritten buffer with an incoming packet. So it doesn't let me debug at all and its driving me crazy
Is ollydbg (debugger) fault that let the other threads running while i set a break point? Or is some error in my IOCP implementation?
Here is how my WorkerThread is coded:
DWORD WINAPI WorkerThread(void* argument)
{
int BytesTransfer;
int BytesRecv;
int ClientID;
int result;
OVERLAPPED* overlapped = 0;
ClientInfo* clientinfo = 0;
WSABUF wsabuf;
int flags;
//Exit only when shutdown signal is recv
while (WaitForSingleObject(IOCPBase::internaldata->sockcontext.ShutDownSignal, NULL) != WAIT_OBJECT_0)
{
flags = 0; BytesTransfer = 0; BytesRecv = 0; ClientID = 0;
//Get from queued list
if (GetQueuedCompletionStatus(IOCPBase::internaldata->sockcontext.CompletionPort, (LPDWORD)&BytesTransfer, (PULONG_PTR)&ClientID, &overlapped, INFINITE) == TRUE)
{
if (overlapped == 0)
{
//Fatal error
break;
}
clientinfo = (ClientInfo*)overlapped;
if (BytesTransfer != 0)
{
//Assign the buffer pointer and buffer len to WSABUF local
clientinfo->RecvContext.RecvBytes = BytesTransfer;
wsabuf.buf = (char*)clientinfo->RecvContext.Buffer;
wsabuf.len = clientinfo->RecvContext.Len;
//Switch for OperationCode
//switch (IOCPBase::internaldata->ClientContext[ClientID].OperationCode)
switch (clientinfo->OperationCode)
{
case FD_READ:
// Check if we have send all data to the client from a previous send
if (clientinfo->SendContext.SendBytes < clientinfo->SendContext.TotalBytes)
{
clientinfo->OperationCode = FD_READ; //We set FD_READ caused on the next send, there could still be bytes left to send
wsabuf.buf += clientinfo->SendContext.SendBytes; //The buffer position is + sended bytes
wsabuf.len = clientinfo->SendContext.TotalBytes - clientinfo->SendContext.SendBytes; //the buffer len is total - sended bytes
//Send the remain bytes
result = WSASend(clientinfo->sock, &wsabuf, 1, (LPDWORD)&BytesRecv, flags, &clientinfo->overlapped, NULL);
if (result == SOCKET_ERROR && (WSAGetLastError() != WSA_IO_PENDING))
{
CloseClient(ClientID);
}
clientinfo->SendContext.SendBytes += BytesRecv;
}
else
{
if (clientinfo->Processing == 0)
{
clientinfo->OperationCode = FD_WRITE; //If no more bytes left to send now we can set the operation code to write (in fact is read)
memset(clientinfo->RecvContext.Buffer, NULL, MAX_DATA_BUFFER_SIZE); //Clean the buffer for recv new data
//Recv data from our client
clientinfo->RecvContext.RecvBytes = WSARecv(clientinfo->sock, &wsabuf, 1, (LPDWORD)&BytesRecv, (LPDWORD)&flags, &clientinfo->overlapped, NULL);
if (clientinfo->RecvContext.RecvBytes == SOCKET_ERROR && WSAGetLastError() != WSA_IO_PENDING)
{
CloseClient(ClientID);
break;
}
}
}
break;
case FD_WRITE:
//Send data to the RecvProtocol
clientinfo->Processing = 1;
IOCPBase::internaldata->callback.RecvProtocol(clientinfo->RecvContext.Buffer, clientinfo->RecvContext.Len, ClientID);
clientinfo->Processing = 0;
default:
break;
}
}
}
}
return false;
}
The problem appears when looking at clientinfo->RecvContext.Buffer. I am watching the packet, past a few seconds and boom the buffer is overwritten with a new packet.
Thanks !
Never mind i fix the debug problem by copy the packet to the stack frame of the function i use to analyze the packet, this way i have no overwritten problem.
Related
I'm writing IOCP server for video streaming from desktop client to browser.
Both sides uses WebSocket protocol to unify server's achitecture (and because there is no other way for browsers to perform a full-duplex exchange).
The working thread starts like this:
unsigned int __stdcall WorkerThread(void * param){
int ThreadId = (int)param;
OVERLAPPED *overlapped = nullptr;
IO_Context *ctx = nullptr;
Client *client = nullptr;
DWORD transfered = 0;
BOOL QCS = 0;
while(WAIT_OBJECT_0 != WaitForSingleObject(EventShutdown, 0)){
QCS = GetQueuedCompletionStatus(hIOCP, &transfered, (PULONG_PTR)&client, &overlapped, INFINITE);
if(!client){
if( Debug ) printf("No client\n");
break;
}
ctx = (IO_Context *)overlapped;
if(!QCS || (QCS && !transfered)){
printf("Error %d\n", WSAGetLastError());
DeleteClient(client);
continue;
}
switch(auto opcode = client->ProcessCurrentEvent(ctx, transfered)){
// Client owed to receive some data
case OPCODE_RECV_DEBT:{
if((SOCKET_ERROR == client->Recv()) && (WSA_IO_PENDING != WSAGetLastError())) DeleteClient(client);
break;
}
// Client received all data or the beginning of new message
case OPCODE_RECV_DONE:{
std::string message;
client->GetInput(message);
// Analizing the first byte of WebSocket frame
switch( opcode = message[0] & 0xFF ){
// HTTP_HANDSHAKE is 'G' - from GET HTTP...
case HTTP_HANDSHAKE:{
message = websocket::handshake(message);
while(!client->SetSend(message)) Sleep(1); // Set outgoing data
if((SOCKET_ERROR == client->Send()) && (WSA_IO_PENDING != WSAGetLastError())) DeleteClient(client);
break;
}
// Browser sent a closing frame (0x88) - performing clean WebSocket closure
case FIN_CLOSE:{
websocket::frame frame;
frame.parse(message);
frame.masked = false;
if( frame.pl_len == 0 ){
unsigned short reason = 1000;
frame.payload.resize(sizeof(reason));
frame.payload[0] = (reason >> 8) & 0xFF;
frame.payload[1] = reason & 0xFF;
}
frame.pack(message);
while(!client->SetSend(message)) Sleep(1);
if((SOCKET_ERROR == client->Send()) && (WSA_IO_PENDING != WSAGetLastError())) DeleteClient(client);
shutdown(client->Socket(), SD_SEND);
break;
}
IO context struct:
struct IO_Context{
OVERLAPPED overlapped;
WSABUF data;
char buffer[IO_BUFFER_LENGTH];
unsigned char opcode;
unsigned long long debt;
std::string message;
IO_Context(){
debt = 0;
opcode = 0;
data.buf = buffer;
data.len = IO_BUFFER_LENGTH;
overlapped.Offset = overlapped.OffsetHigh = 0;
overlapped.Internal = overlapped.InternalHigh = 0;
overlapped.Pointer = nullptr;
overlapped.hEvent = nullptr;
}
~IO_Context(){ while(!HasOverlappedIoCompleted(&overlapped)) Sleep(1); }
};
Client Send function:
int Client::Send(){
int var_buf = O.message.size();
// "O" is IO_Context for Output
O.data.len = (var_buf>IO_BUFFER_LENGTH)?IO_BUFFER_LENGTH:var_buf;
var_buf = O.data.len;
while(var_buf > 0) O.data.buf[var_buf] = O.message[--var_buf];
O.message.erase(0, O.data.len);
return WSASend(connection, &O.data, 1, nullptr, 0, &O.overlapped, nullptr);
}
When the desktop client disconnects (it uses just closesocket() to do it, no shutdown()) the GetQueuedCompletionStatus returns TRUE and sets transfered to 0 - in this case WSAGetLastError() returns 64 (The specified network name is no longer available), and it has sense - client disconnected (line with if(!QCS || (QCS && !transfered))). But when the browser disconnects, the error codes confuse me... It can be 0, 997 (pending operation), 87 (invalid parameter)... and no codes related to end of connection.
Why do IOCP select this events? How can it select a pending operation? Why the error is 0 when 0 bytes transferred? Also it leads to endless trying to delete an object associated with the overlapped structure, because the destructor calls ~IO_Context(){ while(!HasOverlappedIoCompleted(&overlapped)) Sleep(1); } for secure deleting. In DeleteClient call the socket is closing with closesocket(), but, as you can see, I'm posting a shutdown(client->Socket(), SD_SEND); call before it (in FIN_CLOSE section).
I understand that there are two sides of a connection and closing it on a server side does not mean that an other side will close it too. But I need to create a stabile server, immune to bad and half opened connections. For example, the user of web application can rapidly press F5 to reload page few times (yeah, some dudes do so :) ) - the connection will reopen few times, and the server must not lag or crash due to this actions.
How to handle this "bad" events in IOCP?
you have many wrong code here.
while(WAIT_OBJECT_0 != WaitForSingleObject(EventShutdown, 0)){
QCS = GetQueuedCompletionStatus(hIOCP, &transfered, (PULONG_PTR)&client, &overlapped, INFINITE);
this is not efficient and wrong code for stop WorkerThread. at first you do excess call WaitForSingleObject, use excess EventShutdown and main this anyway fail todo shutdown. if your code wait for packet inside GetQueuedCompletionStatus that you say EventShutdown - not break GetQueuedCompletionStatus call - you continue infinite wait here. correct way for shutdown - PostQueuedCompletionStatus(hIOCP, 0, 0, 0) instead call SetEvent(EventShutdown) and if worked thread view client == 0 - he break loop. and usually you need have multiple WorkerThread (not single). and multiple calls PostQueuedCompletionStatus(hIOCP, 0, 0, 0) - exactly count of working threads. also you need synchronize this calls with io - do this only after all io already complete and no new io packets will be queued to iocp. so "null packets" must be the last queued to port
if(!QCS || (QCS && !transfered)){
printf("Error %d\n", WSAGetLastError());
DeleteClient(client);
continue;
}
if !QCS - the value in client not initialized, you simply can not use it and call DeleteClient(client); is wrong under this condition
when object (client) used from several thread - who must delete it ? what be if one thread delete object, when another still use it ? correct solution will be if you use reference counting on such object (client). and based on your code - you have single client per hIOCP ? because you retriever pointer for client as completion key for hIOCP which is single for all I/O operation on sockets bind to the hIOCP. all this is wrong design.
you need store pointer to client in IO_Context. and add reference to client in IO_Context and release client in IO_Context destructor.
class IO_Context : public OVERLAPPED {
Client *client;
ULONG opcode;
// ...
public:
IO_Context(Client *client, ULONG opcode) : client(client), opcode(opcode) {
client->AddRef();
}
~IO_Context() {
client->Release();
}
void OnIoComplete(ULONG transfered) {
OnIoComplete(RtlNtStatusToDosError(Internal), transfered);
}
void OnIoComplete(ULONG error, ULONG transfered) {
client->OnIoComplete(opcode, error, transfered);
delete this;
}
void CheckIoError(ULONG error) {
switch(error) {
case NOERROR:
case ERROR_IO_PENDING:
break;
default:
OnIoComplete(error, 0);
}
}
};
then are you have single IO_Context ? if yes, this is fatal error. the IO_Context must be unique for every I/O operation.
if (IO_Context* ctx = new IO_Context(client, op))
{
ctx->CheckIoError(WSAxxx(ctx) == 0 ? NOERROR : WSAGetLastError());
}
and from worked threads
ULONG WINAPI WorkerThread(void * param)
{
ULONG_PTR key;
OVERLAPPED *overlapped;
ULONG transfered;
while(GetQueuedCompletionStatus(hIOCP, &transfered, &key, &overlapped, INFINITE)) {
switch (key){
case '_io_':
static_cast<IO_Context*>(overlapped)->OnIoComplete(transfered);
continue;
case 'stop':
// ...
return 0;
default: __debugbreak();
}
}
__debugbreak();
return GetLastError();
}
the code like while(!HasOverlappedIoCompleted(&overlapped)) Sleep(1); is always wrong. absolute and always. never write such code.
ctx = (IO_Context *)overlapped; despite in your concrete case this give correct result, not nice and can be break if you change definition of IO_Context. you can use CONTAINING_RECORD(overlapped, IO_Context, overlapped) if you use struct IO_Context{
OVERLAPPED overlapped; } but better use class IO_Context : public OVERLAPPED and static_cast<IO_Context*>(overlapped)
now about Why do IOCP select this events? How to handle this "bad" events in IOCP?
the IOCP nothing select. he simply signaling when I/O complete. all. which specific wsa errors you got on different network operation absolute independent from use IOCP or any other completion mechanism.
on graceful disconnect is normal when error code is 0 and 0 bytes transferred in recv operation. you need permanent have recv request active after connection done, and if recv complete with 0 bytes transferred this mean that disconnect happens
I'm currently working on a project using sockets via WinSock and have come across a peculiar problem. I'll attach the code before I start explaining.
#include "Connection.h"
Connection::Connection(SOCKET sock, int socketType)
: m_sock(sock), m_recvCount(0), m_sendCount(0), m_socketType(socketType)
{
printf("Succesfully created connection\n");
}
Connection::~Connection(void)
{
printf("Closing socket %d", m_sock);
closesocket(m_sock);
}
void Connection::ProcessMessage(const NetMessage *message){
printf("Got network message: type %d, data %s\n", message->type, message->data);
}
bool Connection::ReadSocket(){
// Call this when the socket is ready to read.
// Returns true if the socket should be closed.
// used to store count between the sockets
int count = 0;
if(m_socketType == SOCK_STREAM){
// attempt to read a TCP socket message
// Receive as much data from the client as will fit in the buffer.
count = recv(m_sock, &m_recvBuf[m_recvCount], sizeof(m_recvBuf) - m_recvCount, 0);
}
else if(m_socketType == SOCK_DGRAM){
// attempt to read UDP socket message
// temporarily stores details of the address which sent the message
// since UDP doesn't worry about whether it's connected to the
// sender or not
sockaddr_in fromAddr;
int fromAddrSize = sizeof(fromAddr);
count = recvfrom(m_sock, &m_recvBuf[m_recvCount], sizeof(m_recvBuf) - m_recvCount, 0, (sockaddr*) &fromAddr, &fromAddrSize);
}
else{
printf("Unknown socket type %d\n", m_socketType);
return true;
}
if (count <= 0)
{
printf("Tried to receive on socket %d and got %d bytes\n", m_sock, count);
printf("Client connection closed or broken\n");
return true;
}
// if we get to this point we have essentially received a complete message
// and must process it
printf("Received %d bytes from the client (total %d)\n", count, m_recvCount);
m_recvCount += count;
// Have we received a complete message?
// if so, process it
if (m_recvCount == sizeof NetMessage)
{
ProcessMessage((const NetMessage *) m_recvBuf);
m_recvCount = 0;
}
return false;
}
bool Connection::WriteSocket(){
// Sends the data in the send buffer through the socket
int count;
if(m_socketType == SOCK_STREAM){
// attempt to read TCP socket message
count = send(m_sock, m_sendBuf, m_sendCount, 0);
}
else if(m_socketType == SOCK_DGRAM){
// attempt to read UDP socket message
count = sendto(m_sock, m_sendBuf, m_sendCount, 0, 0, 0);
}
else{
// unhandled type of socket, kill server
printf("Unknown socket type %d", m_socketType);
return true;
}
if (count <= 0)
{
// we have received an error from the socket
printf("Client connection closed or broken\n");
return true;
}
m_sendCount -= count;
printf("Sent %d bytes to the client (%d left)\n", count, m_sendCount);
printf("Data: %s", m_sendBuf);
// Remove the sent data from the start of the buffer.
memmove(m_sendBuf, &m_sendBuf[count], m_sendCount);
return false;
}
bool Connection::WantWrite(){
if(m_sendCount > 0){
return true;
}
return false;
}
bool Connection::WantRead(){
return true;
}
bool Connection::SetMessage(const NetMessage *message){
// store contents of the message in the send buffer
// to allow us to send later
if (m_sendCount + sizeof(NetMessage) > sizeof(m_sendBuf))
{
return true;
}
memcpy(&m_sendBuf, message, sizeof(message));
m_sendCount += sizeof(NetMessage);
return false;
}
and the protocol
/* Definitions for the network protocol that the client and server use to communicate */
#ifndef PROTOCOL_H
#define PROTOCOL_H
// Message types.
enum MessageType
{
MT_UNKNOWN = 0,
MT_WELCOME = 1,
MT_KEYPRESS = 2,
MT_CHATMESSAGE = 3
};
// The message structure.
// This is a "plain old data" type, so we can send it over the network.
// (In a real program, we would want this structure to be packed.)
struct NetMessage
{
MessageType type;
char* data;
NetMessage()
: type(MT_UNKNOWN)
{
}
};
#endif
Essentially the protocol holds the definition of the messages that the client and server throw around to each other. The problem I am having is that, in connection.cpp line 132 (memcpy), the message becomes garbled in sendBuf.
http://imgur.com/MekQfgm,9ShRtHi
The image above shows exactly what is happening. As said in protocol.h the struct is a POD so when I do memcpy it should transfer the number of bytes as is held in the struct (so for example the message type should be 1 byte, followed by 7 or 8 bytes of data, in the example).
Can anyone shed some light on this? It's driving me crazy.
The line you wrote will copy 4 bytes (sizeof(pointer)) on 32bit systems:
memcpy(&m_sendBuf, message, sizeof(message));
what you probably meant is:
memcpy(&m_sendBuf, message, sizeof(NetMessage));
Edit:
In addition, as a commenter remarked, your data type is NOT a POD. It holds a pointer. You transfer that pointer. At the target system, it will point to the same place in RAM, but there will not be anything there. You need to actually make your datatype a POD by using an array or you need to find a way to transfer the data pointed to. You can achieve this by transfering the type, a length and a number of characters. That means that your receiver can NOT rely on messages being of fixed size.
I am responsible for a server that exports data over a TCP connection. With each data record that the server transmits, it requires the client to send a short "\n" acknowledgement message back. I have a customer who claims that the acknowledgement that he sends is not read from the web server. The following is code that I am using for I/O on the socket:
bool can_send = true;
char tx_buff[1024];
char rx_buff[1024];
struct pollfd poll_descriptor;
int rcd;
poll_descriptor.fd = socket_handle;
poll_descriptor.events = POLLIN | POLLOUT;
poll_descriptor.revents = 0;
while(!should_quit && is_connected)
{
// if we know that data can be written, we need to do this before we poll the OS for
// events. This will prevent the 100 msec latency that would otherwise occur
fill_write_buffer(write_buffer);
while(can_send && !should_quit && !write_buffer.empty())
{
uint4 tx_len = write_buffer.copy(tx_buff, sizeof(tx_buff));
rcd = ::send(
socket_handle,
tx_buff,
tx_len,
0);
if(rcd == -1 && errno != EINTR)
throw SocketException("socket write failure");
write_buffer.pop(rcd);
if(rcd > 0)
on_low_level_write(tx_buff, rcd);
if(rcd < tx_len)
can_send = false;
}
// we will use poll for up to 100 msec to determine whether the socket can be read or
// written
if(!can_send)
poll_descriptor.events = POLLIN | POLLOUT;
else
poll_descriptor.events = POLLIN;
poll(&poll_descriptor, 1, 100);
// check to see if an error has occurred
if((poll_descriptor.revents & POLLERR) != 0 ||
(poll_descriptor.revents & POLLHUP) != 0 ||
(poll_descriptor.revents & POLLNVAL) != 0)
throw SocketException("socket hung up or socket error");
// check to see if anything can be written
if((poll_descriptor.revents & POLLOUT) != 0)
can_send = true;
// check to see if anything can be read
if((poll_descriptor.revents & POLLIN) != 0)
{
ssize_t bytes_read;
ssize_t total_bytes_read = 0;
int bytes_remaining = 0;
do
{
bytes_read = ::recv(
socket_handle,
rx_buff,
sizeof(rx_buff),
0);
if(bytes_read > 0)
{
total_bytes_read += bytes_read;
on_low_level_read(rx_buff,bytes_read);
}
else if(bytes_read == -1)
throw SocketException("read failure");
ioctl(
socket_handle,
FIONREAD,
&bytes_remaining);
}
while(bytes_remaining != 0);
// recv() will return 0 if the socket has been closed
if(total_bytes_read > 0)
read_event::cpost(this);
else
{
is_connected = false;
closed_event::cpost(this);
}
}
}
I have written this code based upon the assumption that poll() is a level triggered function and will unblock immediately as long as there is data to be read from the socket. Everything that I have read seems to back up this assumption. Is there a reason that I may have missed that would cause the above code to miss a read event?
It is not edge triggered. It is always level triggered. I will have to read your code to answer your actual question though. But that answers the question in the title. :-)
I can see no clear reason in your code why you might be seeing the behavior you are seeing. But the scope of your question is a lot larger than the code you're presenting, and I cannot pretend that this is a complete problem diagnosis.
It is level triggered. POLLIN fires if there is data in the socket receive buffer when you poll, and POLLOUT fires if there is room in the socket send buffer (which there almost always is).
Based on your own assessment of the problem (that is, you are blocked on poll when you expect to be able to read the acknowledgement), then you will eventually get a timeout.
If the customer's machine is more than 50ms away from your server, then you will always timeout on the connection before receiving the acknowledgement, since you only wait 100ms. This is because it will take a minimum of 50ms for the data to reach the customer, and a minimum of 50ms for the acknowledgement to return.
This is how my server looks like:
-WorkerThread(s):
calls epoll_wait, accepts connections, sets fd nonblocking(EPOLLIN | EPOLLOUT | EPOLLET | EPOLLRDHUP)
calls recv until EAGAIN on EPOLLIN event and pushes all data to global RecvBuffer(pthread_mutex synced)
on EPOLLOUT event: accesses global SendBuffer and if there's data to be sent for current ready fd, do it (in while loop until EAGAIN or until all data is sent; when whole packet is sent, pop it from SendBuffer)
-IOThread(s)
takes data from global RecvBuffer, proccess them
sends response by first trying to call send right away. If not all data is sent, push rest of it onto global SendBuffer to be sent from WorkerThread)
Problem is, that server doesnt send all queued data(they are left in SendBuffer) and amount of 'not sent' data grows by increasing number of clients.
For the sake of testing im using only 1 workerthread and 1 iothread, but it doesnt seem to make any difference if i use more.
Accessing global buffers is protected with pthread_mutex.
Also, my response data size is 130k bytes(it needs 3 send calls at least to send this amount of data). On the other side is windows client using blocking sockets.
Thank you very much!
MJ
edit:
Yes, by default I'm waiting for EPOLLOUT events even tho I have nothing to send. For implementation simplicity and man page guide, i did it like this. Also, my undestanding of it was like this:
Even if I "miss" EPOLLOUT event at the time i dont want to send anything it's no problem because when i want to send data, I'll call send until EAGAIN and EPOLLOUT should be triggered in future(and it is most of time)
Now I modified code to switch between IN/OUT events:
On accept:
event.events = EPOLLIN | EPOLLET | EPOLLRDHUP;
epoll_ctl (pNetServer->m_EventFD, EPOLL_CTL_ADD, infd, &event);
when all data has been sent:
event.events = EPOLLIN | EPOLLET | EPOLLRDHUP;
epoll_ctl (pNetServer->m_EventFD, EPOLL_CTL_MOD, events[i].data.fd, &event);
when I reach EAGAIN by calling send in IOThread:
event.events = EPOLLOUT | EPOLLET | EPOLLRDHUP;
epoll_ctl (pNetServer->m_EventFD, EPOLL_CTL_MOD, events[i].data.fd, &event);
..and I get same behavior. Also, I tried removing EPOLLET flag and nothing's changed
One side question: Does epoll_ctl with EPOLL_CTL_MOD flag replaces events member or just ORs it with given argument?
EDIT3: Updated IOThread function to send continiuosly until all data has been sent, or until EAGAIN.
I also tried to send even if I sent all data, but most of time i was getting errno 88 Socket operation on non-socket
EDIT4: I fixed some bugs in my 'sending code' so I dont get any queued data not sent now.. But, I dont receive as much data as I should :)) Highest amount of 'missed'(not received) data I get when client calls recv right away when sending is complete, and it grows with number of clients. When I put 2 sec delay between send and recv call on client(blocking calls) I lose none to little data on server, depending how many clients im running( client test code includes simple for loop with 1 send and 1 recv call after it )
Again, tried with and without ET mode.. Below is updated WorkerThread function which is responsible for receiving data.
#Admins/Mods Maybe I should open new topic now as problem is a bit different?
void CNetServer::WorkerThread(void* param)
{
CNetServer* pNetServer =(CNetServer*)param;
struct epoll_event event;
struct epoll_event *events;
int s = 0;
// events = (epoll_event*)calloc (MAXEVENTS, sizeof event);
while (1)
{
int n, i;
// printf ("BLOCKING NOW! epoll_wait thread %d\n",pthread_self());
n = pNetServer->m_epollCtrl.Wait(-1);
// printf ("epoll_wait thread %d\n",pthread_self());
pthread_mutex_lock(&g_mtx_WorkerThd);
for (i = 0; i < n; i++)
{
if ((pNetServer->m_epollCtrl.Event(i)->events & EPOLLERR))
{
// An error has occured on this fd, or the socket is not ready for reading (why were we notified then?)
// g_SendBufferArray.RemoveAll( 0 );
char szFileName[30] = {0};
sprintf( (char*)szFileName,"fd_%d.txt",pNetServer->m_epollCtrl.Event(i)->data.fd );
remove(szFileName);
/* printf( "\n\n\n");
printf( "\tDATA LEFT COUNT:%d\n",g_SendBufferArray.size());
for (int k=0;k<g_SendBufferArray.size();k++)
printf( "\tSD: %d DATA LEFT:%d\n",g_SendBufferArray[i]->sd,g_SendBufferArray[i]->nBytesSent );
*/
// fprintf (stderr, "epoll error\n");
// fflush(stdout);
close (pNetServer->m_epollCtrl.Event(i)->data.fd);
continue;
}
else if (pNetServer->m_ListenSocket == pNetServer->m_epollCtrl.Event(i)->data.fd)
{
// We have a notification on the listening socket, which means one or more incoming connections.
while (1)
{
struct sockaddr in_addr;
socklen_t in_len;
int infd;
char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
in_len = sizeof in_addr;
infd = accept (pNetServer->m_ListenSocket, &in_addr, &in_len);
if (infd == -1)
{
if ((errno == EAGAIN) ||
(errno == EWOULDBLOCK))
{
// We have processed all incoming connections.
break;
}
else
{
perror ("accept");
break;
}
}
s = getnameinfo (&in_addr, in_len,
hbuf, sizeof hbuf,
sbuf, sizeof sbuf,
NI_NUMERICHOST | NI_NUMERICSERV);
if (s == 0)
{
printf("Accepted connection on descriptor %d "
"(host=%s, port=%s) thread %d\n", infd, hbuf, sbuf,pthread_self());
}
// Make the incoming socket non-blocking and add it to the list of fds to monitor.
CEpollCtrl::SetNonBlock(infd,true);
if ( !pNetServer->m_epollCtrl.Add( infd, EPOLLIN, NULL ))
{
perror ("epoll_ctl");
abort ();
}
}
continue;
}
if( (pNetServer->m_epollCtrl.Event(i)->events & EPOLLOUT) )
{
pNetServer->DoSend( pNetServer->m_epollCtrl.Event(i)->data.fd );
}
if( pNetServer->m_epollCtrl.Event(i)->events & EPOLLIN )
{
printf("EPOLLIN TRIGGERED FOR SD: %d\n",pNetServer->m_epollCtrl.Event(i)->data.fd);
// We have data on the fd waiting to be read.
int done = 0;
ssize_t count = 0;
char buf[512];
while (1)
{
count = read (pNetServer->m_epollCtrl.Event(i)->data.fd, buf, sizeof buf);
printf("recv sd %d size %d thread %d\n",pNetServer->m_epollCtrl.Event(i)->data.fd,count,pthread_self());
if (count == -1)
{
// If errno == EAGAIN, that means we have read all data. So go back to the main loop.
if ( errno != EAGAIN )
{
perror ("read");
done = 1;
}
break;
}
else if (count == 0)
{
//connection is closed by peer.. do a cleanup and close
done = 1;
break;
}
else if (count > 0)
{
static int nDataCounter = 0;
nDataCounter+=count;
printf("RECVDDDDD %d\n",nDataCounter);
CNetServer::s_pRecvContainer->OnData( pNetServer->m_epollCtrl.Event(i)->data.fd, buf, count );
}
}
if (done)
{
printf ("Closed connection on descriptor %d\n",pNetServer->m_epollCtrl.Event(i)->data.fd);
// Closing the descriptor will make epoll remove it from the set of descriptors which are monitored.
close (pNetServer->m_epollCtrl.Event(i)->data.fd);
}
}
}
//
pNetServer->IOThread( (void*)pNetServer );
pthread_mutex_unlock(&g_mtx_WorkerThd);
}
}
void CNetServer::IOThread(void* param)
{
BYTEARRAY* pbPacket = new BYTEARRAY;
int fd;
struct epoll_event event;
CNetServer* pNetServer =(CNetServer*)param;
printf("IOThread startin' !\n");
for (;;)
{
bool bGotIt = CNetServer::s_pRecvContainer->GetPacket( pbPacket, &fd );
if( bGotIt )
{
//process packet here
printf("Got 'em packet yo !\n");
BYTE* re = new BYTE[128000];
memset((void*)re,0xCC,128000);
buffer_t* responsebuff = new buffer_t( fd, re, 128000 ) ;
pthread_mutex_lock(&g_mtx_WorkerThd);
while( 1 )
{
int s;
int nSent = send( responsebuff->sd, ( responsebuff->pbBuffer + responsebuff->nBytesSent ),responsebuff->nSize - responsebuff->nBytesSent,0 );
printf ("IOT: Trying to send nSent: %d buffsize: %d \n",nSent,responsebuff->nSize - responsebuff->nBytesSent);
if (nSent == -1)
{
if (errno == EAGAIN || errno == EWOULDBLOCK )
{
g_vSendBufferArray.push_back( *responsebuff );
printf ("IOT: now waiting for EPOLLOUT\n");
event.data.fd = fd;
event.events = EPOLLIN | EPOLLOUT | EPOLLET | EPOLLRDHUP;
s = epoll_ctl (pNetServer->m_EventFD, EPOLL_CTL_MOD, fd, &event);
break;
if (s == -1)
{
perror ("epoll_ctl");
abort ();
}
}
else
{
printf( "%d\n",errno );
perror ("send");
break;
}
printf ("IOT: WOOOOT\n");
break;
}
else if (nSent == responsebuff->nSize - responsebuff->nBytesSent)
{
printf ("IOT:all is sent! wOOhOO\n");
responsebuff->sd = 0;
responsebuff->nBytesSent += nSent;
delete responsebuff;
break;
}
else if (nSent < responsebuff->nSize - responsebuff->nBytesSent)
{
printf ("IOT: partial send!\n");
responsebuff->nBytesSent += nSent;
}
}
delete [] re;
pthread_mutex_unlock(&g_mtx_WorkerThd);
}
}
}
Stop using EPOLLET. It's almost impossible to get right.
Don't ask for EPOLLOUT events if you have nothing to send.
When you have data to send on a connection, follow this logic:
A) If there's already data in your send queue for that connection, just add the new data. You're done.B) Try to send the data immediately. If you send it all, you're done.C) Save the leftover data in the send queue for this connection. Now ask for EPOLLOUT for this connection.
I did a few tests with an I/O-Completion port and winsock sockets.
I encountered, that sometimes after I received data from a connection and then adjacently call WSARecv again on that socket it returns immediately with the error 259 (ERROR_NO_MORE_ITEMS).
I am wondering why the system flags the overlapped transaction with this error instead of keeping the recv call blocking/waiting for incoming data.
Do You know what´s the sense of this ?
I would be glad to hear about your thoughts.
Edit: Code
do
{
OVERLAPPED* pOverlapped = nullptr;
DWORD dwBytes = 0; ULONG_PTR ulKey = 0;
//Dequeue a completion packet
if(!m_pIOCP->GetCompletionStatus(&dwBytes, &ulKey, &pOverlapped, INFINITE))
DebugBreak();
//Evaluate
switch(((MYOVERLAPPED*)pOverlapped)->WorkType)
{
case ACCEPT_OVERLAPPED_TYPE:
{
//cast
ACCEPT_OVERLAPPED* pAccept = (ACCEPT_OVERLAPPED*)pOverlapped;
//Associate the newly accepted connection with the IOCP
if(!m_pIOCP->AssociateHandle((HANDLE)(pAccept->pSockClient)->operator SOCKET(), 1))
{
//Association failed: close the socket and and delte the overlapped strucuture
}
//Call recv
RECV_OVERLAPPED* pRecvAction = new RECV_OVERLAPPED;
pRecvAction->pSockClient = pAccept->pSockClient;
short s = (pRecvAction->pSockClient)->Recv(pRecvAction->strBuf, pRecvAction->pWSABuf, 10, pRecvAction);
if(s == Inc::REMOTECONNECTION_CLOSED)
{
//Error stuff
}
//Call accept again (create a new ACCEPT_OVERLAPPED to ensure overlapped being zeroed out)
ACCEPT_OVERLAPPED *pNewAccept = new ACCEPT_OVERLAPPED;
pNewAccept->pSockListen = pAccept->pSockListen;
pNewAccept->pSockClient = new Inc::CSocket((pNewAccept->pSockListen)->Accept(nullptr, nullptr, pNewAccept));
//delete the old overlapped struct
delete pAccept;
}
break;
case RECV_OVERLAPPED_TYPE:
{
RECV_OVERLAPPED* pOldRecvAction = (RECV_OVERLAPPED*)pOverlapped;
if(!pOldRecvAction->InternalHigh)
{
//Connection has been closed: delete the socket(implicitly closes the socket)
Inc::CSocket::freewsabuf(pOldRecvAction->pWSABuf); //free the wsabuf
delete pOldRecvAction->pSockClient;
}
else
{
//Call recv again (create a new RECV_OVERLAPPED)
RECV_OVERLAPPED* pNewRecvAction = new RECV_OVERLAPPED;
pNewRecvAction->pSockClient = pOldRecvAction->pSockClient;
short sRet2 = (pNewRecvAction->pSockClient)->Recv(pNewRecvAction->strBuf, pNewRecvAction->pWSABuf, 10, pNewRecvAction);
//Free the old wsabuf
Inc::CSocket::freewsabuf(pOldRecvAction->pWSABuf);
delete pOldRecvAction;
}
Cutted error checkings...
The Recv-member-function is a simple wrapper around the WSARecv-call which creates the WSABUF and the receiving buffer itself (which needs to be cleaned up by the user via freewsabuf - just to mention)...
It looks like I was sending less data than was requested by the receiving side.
But since it´s an overlapped operation receiving a small junk of the requested bunch via the TCP-connection would trigger the completion indication with the error ERROR_NO_MORE_ITEMS, meaning there was nothing more to recv than what it already had.