Related
I am facing a very odd problem in my C++ project. I have a client and a server that send char arrays of data to each other. While the initial sending/reception works well (see: Client.firstConnection() in MVE), in the following messages the client appears to drop all but the 12th through 15th chars in the array as it sends it to the server. The server then only receives this partially filled array.
Example:
I want to send the buffer 4_user53:6134;*0/ from the client to the server.
Before sending, my variable, when printed character by character, shows 4_user53:6134;*0/
On the server side, the buffer is received but when printed character by character, shows 4;*0
On the client side, immediately after sending the buffer, the variable, when printed character by character, shows 4;*0.
I do not know why these characters are disappearing. All advice is appreciated, thank you.
Please find the minimal viable example below, I have done my best to annotate points of interest as well as illustrate the results of the print functions.
MVE:
Compiled with VS 2017, C++ 11.
//Header file information
// all imports:
#include <cstdlib>
#include <iostream>
#include <map>
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <random>
#include <windows.h>
#include <winsock2.h>
#include <ws2tcpip.h>
// Global vars
#define DEFAULT_BUFLEN 512
#define DEFAULT_PORT "6000"
#define MAX_CONS 1 //Total No of Clients allowed
//Client
class Client{
bool Client::firstConnection()
{
WSADATA wsaData;
SOCKET ConnectSocket = INVALID_SOCKET;
struct addrinfo *result = NULL,
*ptr = NULL,
hints;
int iResult;
// Initialize Winsock
iResult = WSAStartup( MAKEWORD( 2, 2 ), &wsaData );
if( iResult != 0 )
{
printf( "WSAStartup failed with error: %d\n", iResult );
return 1;
}
ZeroMemory( &hints, sizeof( hints ) );
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
// Resolve the server address and port
iResult = getaddrinfo( "127.0.0.1", DEFAULT_PORT, &hints, &result );
if( iResult != 0 )
{
printf( "getaddrinfo failed with error: %d\n", iResult );
WSACleanup();
return 1;
}
// Attempt to connect to an address until one succeeds
for( ptr = result; ptr != NULL; ptr = ptr->ai_next )
{
// Create a SOCKET for connecting to server
this->ConnectSocket = socket( ptr->ai_family, ptr->ai_socktype, ptr->ai_protocol );
if( this->ConnectSocket == INVALID_SOCKET )
{
printf( "socket failed with error: %ld\n", WSAGetLastError() );
WSACleanup();
return 1;
}
// Connect to server.
iResult = connect( this->ConnectSocket, ptr->ai_addr, (int)ptr->ai_addrlen );
if( iResult == SOCKET_ERROR )
{
closesocket( this->ConnectSocket );
this->ConnectSocket = INVALID_SOCKET;
continue;
}
break;
}
freeaddrinfo( result );
if( this->ConnectSocket == INVALID_SOCKET )
{
printf( "Unable to connect to server!\n" );
WSACleanup();
return 1;
}
std::cout << "Connection to server successful!\n";
// Send an initial buffer
int iResult = send( this->ConnectSocket, ( this->username ).c_str(), (int)strlen( ( this->username ).c_str() ), 0 ); //username is a perfectly fine string
// -> This instance of send does NOT drop any data! <- //
if( iResult == SOCKET_ERROR )
{
printf( "send failed with error: %d\n", WSAGetLastError() );
closesocket( this->ConnectSocket );
WSACleanup();
return false;
}
return true;
}
bool Client::sendCommand( std::string command )
{
// Send a buffer
char* tmp = this->gameinfo.SerializeToArray( DEFAULT_BUFLEN ); // see below
const char* send_buf = tmp; // have tried with and without const, results are the same
for( int i = 0; i < (int)strlen( send_buf ); ++i )
{
printf( "Buffer[%d] = [%c]\n", i, send_buf[i] );
}
// Output (literally copy/pasted):
/*
Buffer[0] = [4]
Buffer[1] = [_]
Buffer[2] = [u]
Buffer[3] = [s]
Buffer[4] = [e]
Buffer[5] = [r]
Buffer[6] = [5]
Buffer[7] = [3]
Buffer[8] = [:]
Buffer[9] = [6]
Buffer[10] = [1]
Buffer[11] = [3]
Buffer[12] = [4]
Buffer[13] = [;]
Buffer[14] = [*]
Buffer[15] = [0]
Buffer[16] = [/]
*/
// This is correct and the desired output!!! ^
printf( "Sending: %s, length = %d\n", send_buf, (int)strlen( send_buf ) );
// output (literally copy/pasted):
/*
Sending: 4_user53:6134;*0/, length = 17
*/
// This is correct and the desired output!!!
int iResult = send( this->ConnectSocket, send_buf, (int)strlen( send_buf ), 0 );
for( int i = 0; i < iResult; ++i )
{
printf( "Buffer[%d] = [%c]\n", i, send_buf[i] );
}
//Output (literally copy/pasted):
/*
Buffer[0] = [ ]
Buffer[1] = [ ]
Buffer[2] = [ ]
Buffer[3] = [ ]
Buffer[4] = [ ]
Buffer[5] = [ ]
Buffer[6] = [ ]
Buffer[7] = [ ]
Buffer[8] = [ ]
Buffer[9] = [ ]
Buffer[10] = [ ]
Buffer[11] = [ ]
Buffer[12] = [4]
Buffer[13] = [;]
Buffer[14] = [*]
Buffer[15] = [0]
Buffer[16] = [ ]
*/
// Here we only see the values 12-15. This is not the desired output. This is the issue I want to solve.
printf( "Iresult = %d\n", iResult );
//output= 17. This is correct.
if( iResult == SOCKET_ERROR )
{
printf( "\nSend failed with error: %d\n", WSAGetLastError() );
closesocket( ConnectSocket );
WSACleanup();
return false;
}
return true;
}
bool Client::receiveMessage()
{
char recvbuf[DEFAULT_BUFLEN];
int recvbuflen = DEFAULT_BUFLEN;
int iResult = recv( this->ConnectSocket, recvbuf, recvbuflen, 0 );
if( iResult > 0 )
{
printf("Message received OK!\n");
}
else if( iResult == 0 )
{
printf( "\nConnection closed\n" );
}
else
{
printf( "\nrecv failed with error: %d\n", WSAGetLastError() );
}
return true;
}
}
int main()
{
std::string in = "";
int n = 0;
bool rec = false;
if( this->firstConnection() )
{
while( in != "Q" )
{
std::cin >> in;
this->sendCommand( in );
while( !rec )
{
rec=this->receiveMessage();
}
}
return this->clientClose();
}
else
{
return false;
}
}
//Server
class Server {
Server::Server()
{
this->currentConnections = 0;
}
void Server::accept_clients()
{
for( int i = 0; i < MAX_CONS; i++ )
{
if( !this->client[i].con ) //i.e a client has not connected to this slot
{
if( this->accept_client( &( this->client[i] ) ) )
{
//update server status
this->Server_Status( CLIENT_CON );
}
}
}
}
int Server::accept_client( _client* x )
{
x->i = sizeof( sockaddr );
x->cs = accept( this->ListenSocket, (sockaddr*)&x->addr, &x->i );
if( x->cs == 0 || x->cs == SOCKET_ERROR )
{
printf( "SOCKET ERROR\n" );
return false;
}
x->con = true;
FD_ZERO( &x->set );
FD_SET( x->cs, &x->set );
printf( "Client accepted \n" );
// declaring character array
char char_array[DEFAULT_BUFLEN];
strcpy_s( char_array, result.c_str() );
this->send_clients( char_array );
return true;
}
void Server::send_clients( char* s )
{
int len = (int)strlen( s );
for( int i = 0; i < MAX_CONS; i++ )
{
if( this->client[i].con ) //valid slot,i.e a client has parked here
{
this->send_client( &( this->client[i] ), s, len );
}
}
}
int Server::send_client( _client* x, char* buffer, int sz )
{
char* send_buf;
if( strchr( buffer, ';' ) == NULL )
{
send_buf = x->gameinfo.SerializeToArray( DEFAULT_BUFLEN ); // see below
}
else
{
send_buf = buffer;
}
x->i = send( x->cs, send_buf, DEFAULT_BUFLEN, 0 );
if( x->i == SOCKET_ERROR || x->i == 0 )
{
printf( "Error: disconnecting client\n" );
disconnect_client( x );
return ( false );
}
else
{
return ( true );
}
}
void Server::recv_clients()
{
char buffer[DEFAULT_BUFLEN] = { 0 };
for( int i = 0; i < MAX_CONS; i++ )
{
if( client[i].con ) //valid slot,i.e a client has parked here
{
try
{
if( !this->recv_client( &( this->client[i] ), buffer, DEFAULT_BUFLEN ) )
{
//Error receiving
printf( "Error receiving message from client!\n" );
}
}
catch( ... )
{
printf( "Error\n" );
}
}
}
}
int Server::recv_client( _client* x, char* buffer, int sz )
{
if( FD_ISSET( x->cs, &x->set ) )
{
x->i = recv( x->cs, buffer, sz, 0 );
// checking contents
for( int i = 0; i < sz+1; ++i )
{
printf( "Buffer[%d] = [%c]\n", i, send_buf[i] );
}
// output is (literally copy/pasted):
/*
Buffer[0] = [ ]
Buffer[1] = [ ]
Buffer[2] = [ ]
Buffer[3] = [ ]
Buffer[4] = [ ]
Buffer[5] = [ ]
Buffer[6] = [ ]
Buffer[7] = [ ]
Buffer[8] = [ ]
Buffer[9] = [ ]
Buffer[10] = [ ]
Buffer[11] = [ ]
Buffer[12] = [4]
Buffer[13] = [;]
Buffer[14] = [*]
Buffer[15] = [0]
Buffer[16] = [ ]
*/
if( x->i == 0 )
{
disconnect_client( x );
return false;
}
std::string result = this->update( x, buffer, sz ); // returns a perfectly normal std::string - no issues here!!!
// declaring character array
char char_array[DEFAULT_BUFLEN];
strcpy_s( char_array, result.c_str() );
this->send_clients( char_array );
return true;
}
return false;
}
}
int main()
{
Server gameServer = Server();
while( true )
{
gameServer.accept_clients(); //Receive connections
gameServer.recv_clients(); //Receive and respond to data from clients
}
}
//Game info functions shared between client and server (only relevant ones)
char* GameInformation::SerializeToArray( int buflen )
{
std::string s_out = "Hello world;50*0/";
char out[512]; // more than enough space, no overflow issues here I promise
strcpy_s( out, s_out.c_str() );
return out;
}
In case you still don't believe me, here are screenshots of the prints:
Client:
(while debugging I set the first iterator length to 20, that's why there are 20 values, this is of course inaccurate and I apologise for any inconvenience)
Server:
The function GameInformation::SerializeToArray is bad because it is returning a pointer to non-static local array. The life of the array ends when returning from function and dereferencing pointers pointing at elements of the array after returning is illegal.
Instead of that, you should allocate a buffer on the heap and return a pointer to that.
//Game info functions shared between client and server (only relevant ones)
char* GameInformation::SerializeToArray( int buflen )
{
std::string s_out = "Hello world;50*0/";
char* out = new char[512]; // more than enough space, no overflow issues here I promise
strcpy_s( out, 512, s_out.c_str() );
return out;
}
Note that now you should free (via delete[]) the returned array after finished using that.
Another option is returning std::vector instead of plain array. This will eliminate needs to manually management memory regions.
//Game info functions shared between client and server (only relevant ones)
std::vector<char> GameInformation::SerializeToArray( int buflen )
{
std::string s_out = "Hello world;50*0/";
std::vector<char> out(512); // more than enough space, no overflow issues here I promise
strcpy_s( &out[0], 512, s_out.c_str() );
return out;
}
And the caller in this case will be like this:
// Send a buffer
std::vector<char> tmp = this->gameinfo.SerializeToArray( DEFAULT_BUFLEN );
const char* send_buf = tmp.data();
The declaration of GameInformation::SerializeToArray should also be changed to return std::vector<char> in this case.
Here's my function's I'm using to do the send() and recv()'ing.
bool Socket::send( const std::string s ) const noexcept {
int status = ::send ( m_sock, s.c_str(), s.size(), MSG_NOSIGNAL );
if(status == -1)
return false;
else
return true;
}
int Socket::recv( std::string& s ) const noexcept {
char buf [ MAXRECV + 1 ];
s = "";
memset ( buf, 0, MAXRECV + 1 );
int status = ::recv ( m_sock, buf, MAXRECV, 0 );
if ( status == -1 ) {
std::cout << "status == -1 errno == " << errno << " in Socket::recv\n";
return 0;
}
else if ( status == 0 ) {
return 0;
}
else {
s = buf;
return status;
}
}
The server accepts a client then calls send() with a test message.
std::cout << server.send(s) << std::endl;
Which is always returning false (writing 0 to cout). The client recv()'s by doing the following.
std::string s;
int i = 0;
while (s.empty()) {
i = k.read(s);
}
And it never leaves this loop. Both are set blocking by calling this function with false sent as the parameter.
void Socket::set_non_blocking ( const bool b ) noexcept {
int opts;
opts = fcntl(m_sock, F_GETFL);
if(opts < 0)
return;
if( b )
opts = ( opts | O_NONBLOCK );
else
opts = ( opts & ~O_NONBLOCK );
fcntl( m_sock, F_SETFL,opts );
}
It is not easy to write a iocp console server，socket pool and thread pool works well，but after some times leater, the server can not connect again,though nothing wrong happens, why? I use procexp_16.05.1446001339.exe to check the process properties, I found lots of close_wait status, after some times again, close_wait status disappears, but the server still can not connect.Why is that? And how to fix it ?
#include "stdafx.h"
#include "Winsock2.h"
#include "Windows.h"
#include "Winbase.h"
#include "tlhelp32.h"
#include "tchar.h"
#include "Psapi.h"
#include "Winternl.h"
#include "Shlwapi.h"
#include "mstcpip.h"
#include
#include "ws2tcpip.h"
#include "time.h"
#pragma comment( lib, "Kernel32.lib" )
#pragma comment( lib, "Shlwapi.lib" )
#pragma comment( lib, "Psapi.lib" )
#pragma comment( lib, "Winmm.lib" )
#pragma comment( lib, "Ws2_32.lib" )
#define DATA_BUFSIZE 10240
#define OP_ACCEPT 1
#define OP_RECV 2
#define OP_SEND 3
#define OP_DIS 4
#define OP_ONACCEPT 5
//iocp struct
struct iocp_overlapped{
OVERLAPPED m_ol; //
int m_iOpType; //do type
SOCKET m_skServer; //server socket
SOCKET m_skClient; //client
DWORD m_recvBytes; //recv msg bytes
char m_pBuf[DATA_BUFSIZE]; //recv buf
WSABUF m_DataBuf; //recv data buf
int m_recv_timeout; //recv timeout
int m_send_timeout;
SOCKADDR_IN m_addrClient; //client address
SOCKADDR_IN m_addrServer; //server address
int m_isUsed; //client is active 1 yes 0 not
time_t m_active; //the last active time
int m_isCrashed; //is crashed? 0 not 1 yes
int m_online; //is online 1 yes 0 not
int m_usenum; //
//void (*handler)(int,struct tag_socket_data*); data->handler(res, data);
};
static SOCKET m_sock_listen = INVALID_SOCKET; //the server listen socket
class WingIOCP{
private:
char* m_listen_ip; //listen ip
int m_port; //listen port
int m_max_connect; //max connection
int m_recv_timeout; //recv timeout
int m_send_timeout; //send timeout
unsigned long* m_povs; //clients
//iocp worker
static VOID CALLBACK worker(
DWORD dwErrorCode,
DWORD dwBytesTrans,
LPOVERLAPPED lpOverlapped
);
//accept ex
static BOOL accept(
SOCKET sAcceptSocket,
PVOID lpOutputBuffer,
DWORD dwReceiveDataLength,
DWORD dwLocalAddressLength,
DWORD dwRemoteAddressLength,
LPDWORD lpdwBytesReceived,
LPOVERLAPPED lpOverlapped
);
//disconnect a client socket and reuse it
static BOOL disconnect( SOCKET client_socket , LPOVERLAPPED lpOverlapped , DWORD dwFlags = TF_REUSE_SOCKET , DWORD reserved = 0);
//event callbacks
static void onconnect( iocp_overlapped *&povl );
static void ondisconnect( iocp_overlapped *&povl );
static void onclose( iocp_overlapped *&povl );
static void onrecv( iocp_overlapped *&povl );
static void onsend( iocp_overlapped *&povl );
static void onrun( iocp_overlapped *&povl, DWORD errorcode, int last_error );
static void onaccept(iocp_overlapped *&pOL);
public:
WingIOCP(
const char* listen = "0.0.0.0",
const int port = 6998,
const int max_connect = 10,
const int recv_timeout = 3000,
const int send_timeout = 3000
);
~WingIOCP();
BOOL start();
void wait();
};
/**
* # construct
*/
WingIOCP::WingIOCP(
const char* listen, //listen ip
const int port, //listen port
const int max_connect, //max connect
const int recv_timeout,//recv timeout in milliseconds
const int send_timeout //send timeout in milliseconds
)
{
this->m_listen_ip = _strdup(listen); //listen ip
this->m_port = port; //listen port
this->m_max_connect = max_connect; //max connect
this->m_recv_timeout = recv_timeout; //recv timeout
this->m_send_timeout = send_timeout; //send timeout
this->m_povs = new unsigned long[max_connect];//clients
}
/**
* # destruct
*/
WingIOCP::~WingIOCP(){
if( this->m_listen_ip )
{
free(this->m_listen_ip );
this->m_listen_ip = NULL;
}
if( this->m_povs )
{
delete[] this->m_povs;
this->m_povs = NULL;
}
if( m_sock_listen != INVALID_SOCKET )
{
closesocket( m_sock_listen );
m_sock_listen = INVALID_SOCKET;
}
WSACleanup();
}
/**
*#wait
*/
void WingIOCP::wait(){
while( true ){
Sleep(10);
}
}
//event callbacks
void WingIOCP::onconnect( iocp_overlapped *&pOL ){
printf("%ld onconnect\r\n",pOL->m_skClient);
pOL->m_online = 1;
pOL->m_active = time(NULL);
if( setsockopt( pOL->m_skClient, SOL_SOCKET,SO_UPDATE_ACCEPT_CONTEXT,(const char *)&pOL->m_skServer,sizeof(pOL->m_skServer) ) != 0 )
{
//setsockopt fail
//printf("1=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
WSASetLastError(0);
return;
}
// set send timeout
if( pOL->m_send_timeout > 0 )
{
if( setsockopt( pOL->m_skClient, SOL_SOCKET,SO_SNDTIMEO, (const char*)&pOL->m_send_timeout,sizeof(pOL->m_send_timeout)) !=0 )
{
//setsockopt fail
// printf("2=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
}
}
if( pOL->m_recv_timeout > 0 )
{
if( setsockopt( pOL->m_skClient, SOL_SOCKET,SO_RCVTIMEO, (const char*)&pOL->m_recv_timeout,sizeof(pOL->m_recv_timeout)) != 0 )
{
//setsockopt fail
// printf("3=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
}
}
linger so_linger;
so_linger.l_onoff = TRUE;
so_linger.l_linger = 0; // without close wait status
if( setsockopt( pOL->m_skClient,SOL_SOCKET,SO_LINGER,(const char*)&so_linger,sizeof(so_linger) ) != 0 ){
// printf("31=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
}
//get client ip and port
int client_size = sizeof(pOL->m_addrClient);
ZeroMemory( &pOL->m_addrClient , sizeof(pOL->m_addrClient) );
if( getpeername( pOL->m_skClient , (SOCKADDR *)&pOL->m_addrClient , &client_size ) != 0 )
{
//getpeername fail
// printf("4=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
}
// printf("%s %d connect\r\n",inet_ntoa(pOL->m_addrClient.sin_addr), pOL->m_addrClient.sin_port);
//keepalive open
int dt = 1;
DWORD dw = 0;
tcp_keepalive live ;
live.keepaliveinterval = 5000; //连接之后 多长时间发现无活动 开始发送心跳吧 单位为毫秒
live.keepalivetime = 1000; //多长时间发送一次心跳包 1分钟是 60000 以此类推
live.onoff = TRUE; //是否开启 keepalive
if( setsockopt( pOL->m_skClient, SOL_SOCKET, SO_KEEPALIVE, (char *)&dt, sizeof(dt) ) != 0 )
{
//setsockopt fail
// printf("5=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
}
if( WSAIoctl( pOL->m_skClient, SIO_KEEPALIVE_VALS, &live, sizeof(live), NULL, 0, &dw, &pOL->m_ol , NULL ) != 0 )
{
//WSAIoctl error
// printf("6=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
}
memset(pOL->m_pBuf,0,DATA_BUFSIZE);
//post recv
pOL->m_DataBuf.buf = pOL->m_pBuf;
pOL->m_DataBuf.len = DATA_BUFSIZE;
pOL->m_iOpType = OP_RECV;
DWORD RecvBytes = 0;
DWORD Flags = 0;
int code = WSARecv(pOL->m_skClient,&(pOL->m_DataBuf),1,&RecvBytes,&Flags,&(pOL->m_ol),NULL);
int error_code = WSAGetLastError();
if( 0 != code )
{
if( WSA_IO_PENDING != error_code )
{
// printf("7=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
return;
}
}
else
{
//recv complete
onrecv( pOL );
}
}
void WingIOCP::ondisconnect( iocp_overlapped *&pOL ){
// printf("ondisconnect error %d\r\n",WSAGetLastError());
WSASetLastError(0);
pOL->m_online = 0; //set offline
pOL->m_active = time(NULL); //the last active time
pOL->m_iOpType = OP_ONACCEPT; //reset status
pOL->m_isUsed = 0; //
ZeroMemory(pOL->m_pBuf,sizeof(char)*DATA_BUFSIZE); //clear buf
if( !BindIoCompletionCallback( (HANDLE)pOL->m_skClient ,worker,0) ){
// printf("BindIoCompletionCallback error %ld\r\n",WSAGetLastError());
}
//post acceptex
int error_code = accept( pOL->m_skClient,pOL->m_pBuf,0,sizeof(SOCKADDR_IN)+16,sizeof(SOCKADDR_IN)+16,NULL, (LPOVERLAPPED)pOL );
//printf("accept error %d\r\n",WSAGetLastError());
int last_error = WSAGetLastError() ;
if( !error_code && ERROR_IO_PENDING != last_error ){
}
//printf("2=>ondisconnect some error happened , error code %d \r\n================================================\r\n\r\n", WSAGetLastError());
//printf("21=>ondisconnect some error happened , error code %d \r\n================================================\r\n\r\n", WSAGetLastError());
WSASetLastError(0);
}
void WingIOCP::onaccept(iocp_overlapped *&pOL){
pOL->m_active = time(NULL); //the last active time
pOL->m_iOpType = OP_ACCEPT; //reset status
printf("%ld reuse socket real complete , error code %d \r\n", pOL->m_skClient,WSAGetLastError());
WSASetLastError(0);
}
void WingIOCP::onclose( iocp_overlapped *&pOL ){
// printf("%ld close\r\n", pOL->m_skClient);
SOCKET m_sockListen = pOL->m_skServer;
SOCKET m_client = pOL->m_skClient;
int send_timeout = pOL->m_send_timeout;
int recv_timeout = pOL->m_recv_timeout;
pOL->m_iOpType = OP_DIS;
shutdown( pOL->m_skClient, SD_BOTH );
//socket reuse
if( !disconnect( pOL->m_skClient , &pOL->m_ol ) && WSA_IO_PENDING != WSAGetLastError()) {
// printf("1=>onclose some error happened , error code %d \r\n", WSAGetLastError());
}
//printf("onclose complete %d \r\n", WSAGetLastError());
}
void WingIOCP::onrecv( iocp_overlapped *&pOL ){
pOL->m_active = time(NULL);
// printf("recv:\r\n%s\r\n\r\n",pOL->m_pBuf);
ZeroMemory(pOL->m_pBuf,DATA_BUFSIZE);
}
void WingIOCP::onsend( iocp_overlapped *&povl ){
}
void WingIOCP::onrun( iocp_overlapped *&povl, DWORD errorcode, int last_error ){}
/**
* # acceptex
*/
BOOL WingIOCP::accept(
SOCKET sAcceptSocket,
PVOID lpOutputBuffer,
DWORD dwReceiveDataLength,
DWORD dwLocalAddressLength,
DWORD dwRemoteAddressLength,
LPDWORD lpdwBytesReceived,
LPOVERLAPPED lpOverlapped
)
{
WSASetLastError(0);
if( m_sock_listen == INVALID_SOCKET || !lpOverlapped )
{
return 0;
}
GUID guidAcceptEx = WSAID_ACCEPTEX;
DWORD dwBytes = 0;
LPFN_ACCEPTEX lpfnAcceptEx;
int res= WSAIoctl( m_sock_listen, SIO_GET_EXTENSION_FUNCTION_POINTER, &guidAcceptEx,
sizeof(guidAcceptEx), &lpfnAcceptEx, sizeof(lpfnAcceptEx), &dwBytes, NULL, NULL );
if( 0 != res )
{
return 0;
}
return lpfnAcceptEx( m_sock_listen, sAcceptSocket, lpOutputBuffer, dwReceiveDataLength,
dwLocalAddressLength, dwRemoteAddressLength, lpdwBytesReceived, lpOverlapped );
}
/**
* # disconnect socket and reuse the socket
*/
BOOL WingIOCP::disconnect( SOCKET client_socket , LPOVERLAPPED lpOverlapped , DWORD dwFlags , DWORD reserved )
{
WSASetLastError(0);
if( client_socket == INVALID_SOCKET || !lpOverlapped )
{
return 0;
}
GUID GuidDisconnectEx = WSAID_DISCONNECTEX;
DWORD dwBytes = 0;
LPFN_DISCONNECTEX lpfnDisconnectEx;
if( 0 != WSAIoctl( client_socket,SIO_GET_EXTENSION_FUNCTION_POINTER,&GuidDisconnectEx,
sizeof(GuidDisconnectEx),&lpfnDisconnectEx,sizeof(lpfnDisconnectEx),&dwBytes,NULL,NULL))
{
return 0;
}
return lpfnDisconnectEx(client_socket,lpOverlapped,/*TF_REUSE_SOCKET*/dwFlags,reserved);
}
/**
* # iocp worker thread
*/
VOID CALLBACK WingIOCP::worker( DWORD dwErrorCode,DWORD dwBytesTrans,LPOVERLAPPED lpOverlapped )
{
//why here get the error code 87 ?
//printf("worker error %d\r\n",WSAGetLastError());
if( NULL == lpOverlapped )
{
//not real complete
SleepEx(20,TRUE);//set warn status
WSASetLastError(0);
return;
}
//get overlapped data
iocp_overlapped* pOL = CONTAINING_RECORD(lpOverlapped, iocp_overlapped, m_ol);
//just a test
onrun( pOL, dwErrorCode, WSAGetLastError() );
switch( pOL->m_iOpType )
{
case OP_DIS:
ondisconnect(pOL);
break;
case OP_ONACCEPT:
onaccept(pOL);
break;
case OP_ACCEPT:
{
//new client connect
onconnect( pOL );
}
break;
case OP_RECV:
{
pOL->m_recvBytes = dwBytesTrans;
//check client offline
if( 0 == dwBytesTrans || WSAECONNRESET == WSAGetLastError() || ERROR_NETNAME_DELETED == WSAGetLastError()){
onclose( pOL );
}
else
{ //recv msg from client
pOL->m_recvBytes = dwBytesTrans;
onrecv( pOL );
}
}
break;
case OP_SEND:
{
}
break;
}
WSASetLastError(0);
}
BOOL WingIOCP::start(){
do{
WSADATA wsaData;
if( WSAStartup(MAKEWORD(2,2), &wsaData) != 0 )
{
return FALSE;
}
if(LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2)
{
break;
}
m_sock_listen = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
if( INVALID_SOCKET == m_sock_listen )
{
break;
}
//bind the worker thread
BOOL bReuse = TRUE;
BOOL bind_status = ::BindIoCompletionCallback((HANDLE)( m_sock_listen ), worker, 0 );
if( !bind_status )
{
break;
}
//set option SO_REUSEADDR
if( 0 != ::setsockopt( m_sock_listen, SOL_SOCKET, SO_REUSEADDR,(LPCSTR)&bReuse, sizeof(BOOL) ) )
{
//some error happened
break;
}
struct sockaddr_in ServerAddress;
ZeroMemory(&ServerAddress, sizeof(ServerAddress));
ServerAddress.sin_family = AF_INET;
ServerAddress.sin_addr.s_addr = inet_addr( this->m_listen_ip );
ServerAddress.sin_port = htons( this->m_port );
if ( SOCKET_ERROR == bind( m_sock_listen, (struct sockaddr *) &ServerAddress, sizeof( ServerAddress ) ) )
{
break;
}
if( 0 != listen( m_sock_listen , SOMAXCONN ) )
{
break;
}
//printf("1=>start get error %d\r\n",WSAGetLastError());
WSASetLastError(0);
//socket pool
for( int i = 0 ; i m_max_connect ; i++ )
{
SOCKET client = WSASocket(AF_INET,SOCK_STREAM,IPPROTO_TCP,0,0,WSA_FLAG_OVERLAPPED);
if( INVALID_SOCKET == client )
{
continue;
}
if( !BindIoCompletionCallback( (HANDLE)client ,worker,0) )
{
closesocket(client);
continue;
}
iocp_overlapped *povl = new iocp_overlapped();
if( NULL == povl )
{
closesocket(client);
continue;
}
DWORD dwBytes = 0;
ZeroMemory(povl,sizeof(iocp_overlapped));
povl->m_iOpType = OP_ACCEPT;
povl->m_skServer = m_sock_listen;
povl->m_skClient = client;
povl->m_recv_timeout = m_recv_timeout;
povl->m_isUsed = 0;
povl->m_active = 0;
povl->m_isCrashed = 0;
povl->m_online = 0;
povl->m_usenum = 1;
int server_size = sizeof(povl->m_addrServer);
ZeroMemory(&povl->m_addrServer,server_size);
getpeername(povl->m_skServer,(SOCKADDR *)&povl->m_addrServer,&server_size);
int error_code = accept( povl->m_skClient, povl->m_pBuf, 0, sizeof(SOCKADDR_IN)+16, sizeof(SOCKADDR_IN)+16, NULL, (LPOVERLAPPED)povl );
int last_error = WSAGetLastError() ;
if( !error_code && ERROR_IO_PENDING != last_error )
{
closesocket( client );
client = povl->m_skClient = INVALID_SOCKET;
delete povl;
povl = NULL;
//printf("client=>crate error %d\r\n",WSAGetLastError());
}else{
this->m_povs[i] = (unsigned long)povl;
}
//here all the last error is 997 , means nothing error happened
//printf("client=>start get error %d\r\n",WSAGetLastError());
WSASetLastError(0);
}
//printf("last start get error %d\r\n",WSAGetLastError());
WSASetLastError(0);
return TRUE;
} while( 0 );
if( m_sock_listen != INVALID_SOCKET )
{
closesocket( m_sock_listen );
m_sock_listen = INVALID_SOCKET;
}
WSACleanup();
return FALSE;
}
int _tmain(int argc, _TCHAR* argv[])
{
WingIOCP *iocp = new WingIOCP();
iocp->start();
iocp->wait();
delete iocp;
return 0;
}
The solution to any CLOSE_WAIT issue is to close the socket. Evidently you are leaking sockets at end of stream or on an error.
Correct setsockopt usage like MS sample.
[EDIT] After some search I find this issue about AcceptEx.
Good lock.
I am trying to make a ZMQ_REP socket connect to a new address when connecting to a first socket did not work out.
The main question concerns a problem that occurs when the user provides an incorrect socket endpoint. Eg.: the server is located at endpoint: tcp://127.0.0.1:12345but the user tries to connect to endpoint tcp://127.0.0.1:12346 (note the 6 instead of 5 in the port number).Now I send a message to the server and the server should reply with another message. If this operation results in a timeout ( ETIMEOUT ), I want the program to connect to a new address, specified by the user.
Below is a lot of code but please focus on the thread and the main function. The other functions are included so the entire program is compilable. The thread function acts as a server for the main function which acts as a client. If you compile the program you can just run it and the main function will connect to the proper function. However, if you specify an endpoint as argument of the program it will fail (unless you specify the correct endpoint of course).
So I hope someone knows how I can make a socket connect to a new socket/endpoint?
I always keep either getting ETIMEOUT even when the user specifies the correct endpoint when he/she is prompted.
I compile the code like this:
g++ program.cpp -std=c++0x $(pkg-config --cflags --libs libzmq ) -Wall -pedantic -g
Where libzmq is version 2.2.0.
#include <iostream>
#include <thread>
#include <zmq.h>
#include <cstdlib>
#include <cstring>
#include <cassert>
using namespace std;
/*forward declarations.*/
char* recv_str( void* socket, int options=0 );
int send_string ( void* sock, const char* message, int opt = 0);
void
thread_func ( void* context, std::string address, string identifier )
{
int rc;
void* socket = zmq_socket( context, ZMQ_REP);
if ( !socket ){
perror( "__LINE__: " );
}
std::cout << identifier << " binding to: " << address << endl;
rc = zmq_bind ( socket, address.c_str() );
if (rc != 0){
char* error = strerror(errno);
cout << error << endl;
}
int running = 1;
while ( running ) {
char* message;
message = recv_str( socket );
cerr << "recieved: " << message << endl;
if ( strcmp( message, "stop" ) == 0 ){
rc = send_string( socket, "stopping" );
if (rc)
perror( "Unable to send string" );
assert ( rc == 0);
running = 0;
}
else if( strcmp ( message , "reply" ) == 0 ){
int rc = send_string( socket, (string("reply = ") + identifier).c_str() );
if (rc)
perror( "Unable to send string" );
assert ( rc == 0);
}
else
send_string ( socket, "error" );
}
zmq_close(socket);
cout << "Server is stopping."<< endl;
}
int main ( int argc, char** argv)
{
int rc; //returnvalue a to check whether function succeed
char* msg; //recieved messages.
/*The address the server binds to.*/
const char* addr1 = "tcp://127.0.0.1:12345";
/*If no argument is given the address to which we connect is equal
*to the server addres.*/
const char* caddr = addr1;
if ( argc == 2 ){ //so we can provide a custom addres.
caddr = argv[1];
}
void* c = zmq_init(1);
void* sock = zmq_socket( c, ZMQ_REQ);
//timeout of 500ms
int timeout = 500;
size_t s = sizeof(timeout);
rc = zmq_setsockopt(sock, ZMQ_RCVTIMEO, &timeout, s);
//rc = zmq_setsockopt(sock, ZMQ_SNDTIMEO, &timeout, s);
if ( rc ){
perror("Couldn't set sock opt: ");
return 1;
}
//starting server inside thread.
std::thread t1 ( &thread_func, c, addr1, "one" );
/*check if server exists and reconnect if not*/
int connected = 0;
/*connect to the server.*/
rc = zmq_connect( sock, caddr );
rc = send_string( sock, "reply");
assert ( rc == 0);
msg = recv_str( sock);
if ( msg != NULL) {
connected = 1;
free (msg);
}
/*if not connected retry with user input*/
string current = caddr;
while ( !connected ) {
/*connect to the server.*/
cout << "Unable to connect to " << current << "\nplease provide a new addres: " ;
cin >> current;
rc = zmq_connect( sock, current.c_str() );
assert ( rc == 0);
send_string(sock, "reply");
msg = recv_str(sock);
if ( msg ){
connected = 1;
free(msg);
}
}
/*do something hopefully more useful than this with the server.*/
for ( int i = 0; i < 10; i++) {
send_string ( sock , "reply" );
msg = recv_str( sock );
std::cout << msg << endl;
free(msg);
}
send_string ( sock , "stop" );
msg = recv_str (sock);
cout << "Got " << msg <<endl;
/*we are finished with msg.*/
if (msg)
free(msg);
t1.join();
cout << "Thread joined " << endl;
/*terminate zmq related resources.*/
zmq_close(sock);
zmq_term( c );
return 0;
}
/*Returns a string from a socket, the caller must free the string.*/
char* recv_str( void* socket, int options )
{
int rc = 0; //return value for functions that can faile
size_t size; //for message size
zmq_msg_t msg; //our message
if ( (rc =zmq_msg_init( &msg ) ) != 0 ){
return NULL;
}
//recieving message
rc = zmq_recv( socket, &msg, options);
if (rc) {
perror("While recieving msg");
zmq_msg_close(&msg);
return NULL;
}
size = zmq_msg_size( &msg );
char *str = (char*)malloc (size+1); //reserve sizeof message + '0'
if ( ! str){
zmq_msg_close(&msg);
return NULL;
}
/*copy message content to string */
memcpy( str, zmq_msg_data(&msg), zmq_msg_size(&msg) );
/*free msg*/
zmq_msg_close(&msg);
/*we can't assume caller sends a nullbyte on the line.*/
str[size] = '\0';
return str;
}
int send_string ( void* sock, const char* message, int opt ) {
int rc;
size_t size = strlen( message );
zmq_msg_t msg;
zmq_msg_init_size(&msg, size);
memcpy ( zmq_msg_data(&msg), message, size);
rc = zmq_send( sock, &msg, opt);
zmq_msg_close(&msg);
return rc;
}
EDIT: I am guessing the problem is I have to associate the OVERLAPPED or WSAOVERLAPPED in the container with my completion port. Is that correct?
I can get IO completions when someone connects to my server. I then use CreateIoCompletionPort on the new socket, with the completionport that original was used. But when they send me data, it does not get set off. Although, it still gets set off if someone else connects. My question is, why would this happen? I also make sure CreateIoCompletionPort returns the same handle as was the original. What gives?
EDIT:
DWORD WINAPI worker_thread(LPVOID lpParam) {
client_information_class *cicc = NULL;
HANDLE CompletionPort = (HANDLE)lpParam;
ULONG_PTR Key;
DWORD BytesTransfered;
OVERLAPPED *lpOverlapped = NULL;
DWORD error = NULL;
while(1) {
error = GetQueuedCompletionStatus(CompletionPort, &BytesTransfered, (PULONG_PTR)&Key, &lpOverlapped, 0);
cicc = CONTAINING_RECORD ( lpOverlapped, client_information_class, ol );
if ( error == TRUE ) {
cout << endl << "IO TRIGGERED" << endl;
switch ( cicc->operation ) {
/*#define OP_ACCEPT 0
#define OP_READ 1
#define OP_WRITE 2*/
case 0:{
if ( check_auth_progress ( cicc->client_socket , cicc->client_buff , BytesTransfered ) ) {
cout << "Client " << cicc->client_socket << " connected." << endl;
client_information_class *k = NULL;
SOCKADDR_STORAGE *LocalSockaddr=NULL, *RemoteSockaddr=NULL;
int LocalSockaddrLen,RemoteSockaddrLen;
k = (client_information_class *)Key;
k->lpfnGetAcceptExSockaddrs(
cicc->client_buff,
cicc->client_len - ((sizeof(SOCKADDR_STORAGE) + 16) * 2),
sizeof(SOCKADDR_STORAGE) + 16,
sizeof(SOCKADDR_STORAGE) + 16,
(SOCKADDR **)&cicc->LocalSockaddr,
&cicc->LocalSockaddrLen,
(SOCKADDR **)&cicc->RemoteSockaddr,
&cicc->RemoteSockaddrLen
);
client_information_class *cicc2 = NULL;
cicc2 = ( client_information_class *)HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, sizeof(client_information_class) + (sizeof(BYTE) * 4096));
if (cicc2 == NULL) {
fprintf(stderr, "Out of memory!\n");
}
cicc2->client_socket = cicc->client_socket;
cicc2->client_socketaddr_in = cicc->client_socketaddr_in;
cicc2->LocalSockaddr = cicc->LocalSockaddr;
cicc2->LocalSockaddrLen = cicc->LocalSockaddrLen;
cicc2->RemoteSockaddr = cicc->RemoteSockaddr;
cicc2->RemoteSockaddrLen = cicc->RemoteSockaddrLen;
HANDLE hrc = CreateIoCompletionPort( (HANDLE)cicc2->client_socket, CompletionPort, (ULONG_PTR)cic, 0 );
if (hrc == NULL) {
fprintf(stderr, "CompletionThread: CreateIoCompletionPort failed: %d\n", GetLastError());
return 0;
} else {
fprintf(stderr, "CompletionThread: CreateIoCompletionPort: %d\n", hrc);
}
cic->deleteNode ( cicc->client_socket , cic );
cic->addNode ( cicc2 );
} else {
cout << endl << "Something Happened ... " << endl;
}
}break;
case 1:{
if ( ParsePacket ( cicc->client_socket , data ) ) {
cout << "Client " << cicc->client_socket << " connected." << endl;
} else {
cout << endl << "Something Happened ... " << endl;
}
}break;
default:{
cout << endl << "Didnt catch that operation ... " << cicc->operation << endl;
}break;
}
} else if ( error == FALSE && &lpOverlapped == NULL ) {
// no packet was dequed...
fprintf(stderr, "[error == FALSE && &lpOverlapped == NULL] CompletionThread: GetQueuedCompletionStatus failed: %d [0x%x]\n", GetLastError(), &lpOverlapped->Internal);
} else if ( error == FALSE && &lpOverlapped != NULL ) {
if((DWORD)&lpOverlapped->Internal == 0x0) { // a timeout...
} else {
fprintf(stderr, "[error == FALSE && &lpOverlapped != NULL] CompletionThread: GetQueuedCompletionStatus failed: %d [0x%x]\n", GetLastError(), &lpOverlapped->Internal);
}
}
}
ExitThread(0);
return 0;
}
Id hate to do this again, but I was correct, you have to place the socket into a new mode (much like acceptex) using WSARECV: I did not know this, and its not very clear on the MSDN, and one of the sources I was looking at to learn IOCP, doesn't talk about it. Hopefully this helps someone :/
WSABUF wbuf;
DWORD bytes, flags;
wbuf.buf = cicc2->client_buff;
wbuf.len = cicc2->client_len;
flags = 0;
int rr = WSARecv ( cicc2->client_socket , &wbuf , 1 , &bytes , &flags , &cicc2->ol , NULL );
if (rr == FALSE) {
if (WSAGetLastError() != WSA_IO_PENDING) {
printf("PostRecv: WSARecv* failed: %d\n", WSAGetLastError());
closesocket(cicc2->client_socket);
cic->deleteNode ( cicc2->client_socket , cic );
}
fprintf(stderr, "PostRecv: WSARecv* failed: %d\n", GetLastError());
}