I learned how to create sockets using the Windows Message Proc and switched on FD_CONNECT, FD_ACCEPT, FD_CLOSE, etc.. I used: WSAAsyncSelect(socket, WindowHandle, WM_SOCKET, FD_READ | FD_WRITE | FD_CONNECT | FD_CLOSE | FD_ACCEPT).
This let me know when a socket: accepted, closed, received data all without having to poll for it.
Now I'm trying to learn to do the same for console applications but using IOCP.
I have written the following:
#include <Winsock2.h>
#include <Ws2tcpip.h>
#include <Windows.h>
#include <iostream>
#include <thread>
#include <stdexcept>
#include <cstring>
class Socket
{
private:
typedef struct
{
OVERLAPPED Overlapped;
WSABUF DataBuf;
char Buffer[1024];
unsigned int BytesSent;
unsigned int BytesReceived;
} PER_IO_OPERATION_DATA;
typedef struct
{
SOCKET Sock;
} PER_HANDLE_DATA;
protected:
void Close();
std::function<void(HANDLE)> Worker;
public:
Socket();
bool Start(std::string Address, unsigned int Port, bool Listen);
void Read(char* Buffer, int bufflen); //reads from a socket.
void Write(char* Buffer, int bufflen); //writes to a socket.
bool Accept(); //accepts a socket.
virtual void OnRead(); //Called when Reading.
virtual void OnWrite(); //Called when Writing.
virtual void OnAccept(); //Called when a socket has been accepted.
virtual void OnConnect(); //Called when connected.
virtual void OnDisconnect(); //Called when disconnected.
};
bool Socket::Start(std::string Address, unsigned int Port, bool Listen)
{
WSADATA wsaData;
SOCKET sock = 0;
struct sockaddr_in* sockaddr_ipv4;
//WSA Startup and getaddrinfo, etc.. here..
//Create IOCP Handle and worker threads.
HANDLE IOCPPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, nullptr, 0, 0);
if (!IOCPPort)
{
this->Close();
throw std::runtime_error("Error: Creating IOCP Failed With Error: " + std::to_string(GetLastError()));
}
SYSTEM_INFO SystemInfo = {0};
GetSystemInfo(&SystemInfo);
for (std::size_t I = 0; I < SystemInfo.dwNumberOfProcessors * 2; ++I)
{
std::thread(this->Worker, IOCPPort).detach();
}
//Set the socket to overlapped.
if ((sock = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, nullptr, 0, WSA_FLAG_OVERLAPPED)) == INVALID_SOCKET)
{
this->Close();
throw std::runtime_error("Error: ");
}
struct sockaddr_in SockAddr;
std::memset(&SockAddr, 0, sizeof(SockAddr));
SockAddr.sin_port = htons(Port);
SockAddr.sin_family = AF_INET;
SockAddr.sin_addr.s_addr = (Address == "INADDR_ANY" ? htonl(INADDR_ANY) : inet_addr(Address.c_str()));
//If it is a server socket being created, bind it.
if (Listen && (bind(sock, reinterpret_cast<SOCKADDR*>(&SockAddr), sizeof(SockAddr)) == SOCKET_ERROR))
{
this->Close();
throw std::runtime_error("Error: ");
}
//If it is a server socket, start listenening.
if (Listen && (listen(sock, SOMAXCONN) == SOCKET_ERROR))
{
this->Close();
throw std::runtime_error("Error: ");
}
//Otherwise it is a client socket so just connected..
//if(!Listen && (connect(this->socket, reinterpret_cast<SOCKADDR*>(&SockAddr), sizeof(SockAddr)) == SOCKET_ERROR))
//Associate this socket with a completion port.
if (CreateIoCompletionPort(reinterpret_cast<HANDLE>(sock), IOCPPort, 0, 0) != IOCPPort)
{
this->Close();
throw std::runtime_error("Error: ");
}
//setsockopt(sock, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, (char *)sdListen, sizeof(sdListen));
}
However, now I'm stuck. I'm not sure what I need to do after creating the IoCompletionPort. How would I know when the socket has been accepted so I can call OnAccept or how can I tell when there is data to be read so I can call OnRead? I've went through all pages on google and I cannot find anything that is similar to OnRead, OnAccept or OnWrite.
I just want to make it scalable and have callbacks for when something happens without using events or message loop. The only thing I saw on google that interested me was IOCP but I'm completely lost. Any ideas what I need to do next?
I wrote some articles on IOCP server design a long time ago and I also have some code you can download which does the basics for you.
You can get the code and find links to the articles from here. They may help you to understand what happens next and how to structure an IOCP client or server.
Related
I am tying to evaluate the performance of this simple client-server program setting it up in a way where a test client makes a fixed number of requests, say; 10000 and then see how long it takes the server to deal with these and then trying it with multiple clients in parallel since it runs with threads. Now I am wondering how to program this? (sorry just started with Winsock). I was also wondering if this is a proper implementation of threading and if not what could be improved and why. Thanks in advance
Test server code;
Server:
#define _WINSOCK_DEPRECATED_NO_WARNINGS
#pragma comment(lib,"ws2_32.lib") //header file
#include <WinSock2.h>
#include <string>
#include <iostream>
#include <iomanip>
using namespace std;
SOCKET Connections[100];
int ConnectionCounter = 0;
void ClientHandlerThread(int index) //index = the index in the SOCKET Connections array
{
char buffer[256]; //Buffer to receive and send out messages from/to the clients
while (true)
{
recv(Connections[index], buffer, sizeof(buffer), NULL); //get message from client
for (int i = 0; i < ConnectionCounter; i++) //For each client connection
{
if (i == index) //Don't send the chat message to the same user who sent it
continue; //Skip user
send(Connections[i], buffer, sizeof(buffer), NULL);//send the chat message to this client
}
}
}
int main()
{
std::cout << "\n\n\n\n\n Server sucessfully turned on, awaiting for clients...\n\n\n\n\n";
//Winsock Startup
WSAData wsaData;
WORD DllVersion = MAKEWORD(2, 1);
if (WSAStartup(DllVersion, &wsaData) != 0) //initialise winsock library, if WSAStartup returns anything other than 0, then that means an error has occured in the WinSock Startup.
{
MessageBoxA(NULL, "WinSock startup failed", "Error", MB_OK | MB_ICONERROR);
return 0;
}
SOCKADDR_IN addr; //Address that we will bind our listening socket to
int addrlen = sizeof(addr); //length of the address (required for accept call)
addr.sin_addr.s_addr = inet_addr("127.0.0.1"); //Broadcast locally, using inet_address funtion that converts to correct long format.
addr.sin_port = htons(1111); //Port
addr.sin_family = AF_INET; //IPv4 Socket
SOCKET sListen = socket(AF_INET, SOCK_STREAM, NULL); //Create socket to listen for new connections
bind(sListen, (SOCKADDR*)&addr, sizeof(addr)); //Bind the address to the socket
listen(sListen, SOMAXCONN); //Places sListen socket in a state in which it is listening for an incoming connection. Note:SOMAXCONN = Socket Oustanding Max Connections
SOCKET newConnection; //Socket to hold the client's connection
int ConnectionCounter = 0; //# of client connections
for (int i = 0; i < 100; i++)
{
newConnection = accept(sListen, (SOCKADDR*)&addr, &addrlen); //Accept a new connection
if (newConnection == 0) //If accepting the client connection failed
{
std::cout << "Failed to accept the client's connection." << std::endl;
}
else //If client connection properly accepted
{
std::cout << "\n\n\nClient Connected!\n\n" << std::endl;
/* char MOTD[256] = "Welcome! This is the Message of the Day."; //Create buffer with message of the day
send(newConnection, MOTD, sizeof(MOTD), NULL); //Send MOTD buffer */
Connections[i] = newConnection; //Set socket in array to be the newest connection before creating the thread to handle this client's socket.
ConnectionCounter += 1; //Incremenent total # of clients that have connected
cout << "\nConnected Clients: ";
cout << ConnectionCounter;
CreateThread(NULL, NULL, (LPTHREAD_START_ROUTINE)ClientHandlerThread, (LPVOID)(i), NULL, NULL); //Create Thread to handle this client. The index in the socket array for this thread is the value (i).
}
}
system("pause");
return 0;
}
Client:
#define _WINSOCK_DEPRECATED_NO_WARNINGS
#pragma comment(lib,"ws2_32.lib") //Required for WinSock
#include <WinSock2.h> //For win sockets
#include <string> //For std::string
#include <iostream> //For std::cout, std::endl, std::cin.getline
SOCKET Connection;//This client's connection to the server
void ClientThread()
{
char buffer[256]; //Create buffer to hold messages up to 256 characters
while (true)
{
recv(Connection, buffer, sizeof(buffer), NULL); //receive buffer
std::cout << buffer << std::endl; //print out buffer
}
}
int main()
{
//Winsock Startup
WSAData wsaData;
WORD DllVersion = MAKEWORD(2, 1);
if (WSAStartup(DllVersion, &wsaData) != 0)
{
MessageBoxA(NULL, "Winsock startup failed", "Error", MB_OK | MB_ICONERROR);
return 0;
}
SOCKADDR_IN addr; //Address to be binded to our Connection socket
int sizeofaddr = sizeof(addr); //Need sizeofaddr for the connect function
addr.sin_addr.s_addr = inet_addr("127.0.0.1"); //Address = localhost (this pc)
addr.sin_port = htons(1111); //Port = 1111
addr.sin_family = AF_INET; //IPv4 Socket
Connection = socket(AF_INET, SOCK_STREAM, NULL); //Set Connection socket
if (connect(Connection, (SOCKADDR*)&addr, sizeofaddr) != 0) //If we are unable to connect...
{
MessageBoxA(NULL, "Failed to Connect", "Error", MB_OK | MB_ICONERROR);
return 0; //Failed to Connect
}
std::cout << "Connected!" << std::endl;
CreateThread(NULL, NULL, (LPTHREAD_START_ROUTINE)ClientThread, NULL, NULL, NULL); //Create the client thread that will receive any data that the server sends.
char buffer[256]; //256 char buffer to send message
while (true)
{
std::cin.getline(buffer, sizeof(buffer)); //Get line if user presses enter and fill the buffer
send(Connection, buffer, sizeof(buffer), NULL); //Send buffer
Sleep(10);
}
return 0;
}
As #JerryCoffin said, using thread-per-client model burden the server by too many context switches between threads. Modern servers like Nginx web server use an asynchronous non-blocking model to reach scalability. I have developed such an architecture recently. To get more familiar with this model, I provide some useful links for you.
Start with the following to get more familiar with the concept:
Inside NGINX: How We Designed for Performance & Scale
You need to know re-actor and pro-actor design patterns. You can implement them using select() or similar facilities.
Example: Nonblocking I/O and select()
There are also some nice implementations of them in libraries like boost, POCO, and ACE. Personally, I recommend you take a look at the boost asio library. You can find full documentation, tutorial, and examples here.
Boost.Asio
I'm trying to start QLocalServer in my local OSX 10.11.
I have client which tries connection to server in loop:
int connect(const char* filename) {
int sock;
struct sockaddr_un serv_addr;
memset(&serv_addr, 0x00, sizeof(serv_addr));
serv_addr.sun_family = AF_LOCAL;
strncpy(serv_addr.sun_path, filename, sizeof(serv_addr.sun_path) - 1);
if ((sock = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
return sock;
}
if (connect(sock, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) == -1) {
close(sock);
sock = -1;
return sock;
}
return sock;
}
int main() {
int sock;
while((sock = connect("my_socket_server")) == -1) {
usleep(3000);
}
// The code never reaches this line
const char* buffer = "hello";
if (send(sock, buffer, strlen(buffer), 0) < 0) {
exit(1);
}
return 0;
}
When this code is running, I try to start QLocalServer in another application:
// Starting server:
QString socket_path = "my_socket_server";
QLocalServer::removeServer(socket_path);
if (!server.listen(socket_path)) {
return false;
}
connect(&server, &QLocalServer::newConnection, this, &MyServerClass::newConnection);
...
void MyServerClass::newConnection() {
socket = server.nextPendingConnection(); // socket - member of MyServerClass
connect(socket, &QLocalSocket::disconnected, socket, &QLocalSocket::deleteLater);
connect(socket, &QLocalSocket::readyRead, this, &MyServerClass::readyRead);
}
...
void MyServerClass::readyRead() {
if (!socket->bytesAvailable()) {
exit(1); // THIS CODE WAS CALLED. WHY?
}
...
}
Why when readyRead was called, bytes are not available?
You might be getting multiple connections, in which case the socket variable might be pointing to a different object from the one which emits the readyRead signal. Either use QObject::sender to get the correct object in the slot, or use the QSignalMapper. If you are using Qt5, you can also use a lambda function and capture the socket object.
I have UDP Server program which receives data from port 7888. The server code is below.
//UDPIPV4Server.h
#pragma once
#include <iostream>
#include <string>
#include <winsock2.h>
#include <windows.h>
#include <Ws2tcpip.h>
using std::string;
using std::cout;
using std::endl;
using std::cerr;
class UDPIPV4Server
{
public:
UDPIPV4Server();
~UDPIPV4Server(void);
UINT type;
string mac_address;
UINT port;
int socket_var;
struct sockaddr_in si_server, si_client;
int Config(void);
int RecvData(char* recv_buffer, int buf_size,string *ip_addr);
};
//UDPIPV4Server.cpp
int UDPIPV4Server::Config(void) {
WSADATA wsadata;
int error = WSAStartup(0X0202, &wsadata);
if(error) {
cerr<<"UdpIPV4Server.cpp:- WSAStartup failed"<<endl;
return -1;
}
if ((socket_var = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) == -1) {
cerr<<"UdpIPV4Server.cpp:- socket function failed"<<endl;
return -1;
}
memset((char *) &si_server, 0, sizeof(si_server));
si_server.sin_family = AF_INET;
si_server.sin_port = htons(7888);
char host[NI_MAXHOST] = "10.8.0.2";
if(inet_pton(AF_INET, host, &si_server.sin_addr) != 1) {
cerr<<"UdpIPV4Server.cpp: inet_pton() failed\n";
return -1;
}
if(bind(socket_var,(struct sockaddr *)&si_server,sizeof(si_server)) == -1) {
cerr<<"UdpIPV4Server.cpp:- bind failed: "<<endl;
return -1;
}
return 0;
}
//recv data from the UDP client
//recv_buffer - [out] receive buffer
//buf_size - [in] size of receive buffer in bytes
//ip_addr - [out] ip address of the remote client
int UDPIPV4Server::RecvData(char* recv_buffer, int buf_size, string *ip_addr) {
int recv_len;
cout<<"waiting for data\n";
memset((char*)&si_client, 0, sizeof(struct sockaddr_in));
int si_client_len = sizeof(si_client);
if((recv_len = recvfrom(socket_var, recv_buffer, buf_size, 0, (struct sockaddr *)&si_client, &si_client_len)) == -1) {
cerr<<"udpipv4server.cpp:- recvfrom failed"<<endl;
return recv_len;
}
char client_addr[INET_ADDRSTRLEN];
cout<<"Received packets from "<<inet_ntoa(si_client.sin_addr)<<":"<<ntohs(si_client.sin_port)<<endl;
*ip_addr = inet_ntoa(si_client.sin_addr);
return recv_len;
}
//main.cpp
#include "UDPIPV4Server.h"
int main() {
UDPIPV4Server udp_server;
udp_server.Config();
char recv_frame_buffer[65534] = "";
memset(recv_frame_buffer,0,sizeof(recv_frame_buffer));
int retval;
string ip_addr;
if((retval = udp_server.RecvData(recv_frame_buffer,sizeof(recv_frame_buffer),&ip_addr)) == -1) {
cerr<<"ReceiverCommModule:- Error in receving data"<<endl;
continue;
}
}
The above program receives data on 10.8.0.2:7888. But this code is not working when data is received. I have checked with wireshark, the data is being received at 10.8.0.2:7888. But socket is unable to read the data from the port.The UDPIPV4Server.config() function passed successfully. But the UDPIPV4Server.RecvData() is not returning. The UDP recvfrom is waiting as such there is no data received. Is it anything wrong with the code? Thank you.
I had the same issue. in UDP, recvfrom behaves as if no data were available dispite wireshark confirmed that a valid UDP packet arrived on the correct port. It happens on my compagny's computer but it works fine on my personnal one. I guess this is linked to windows fire wall or antivir that filters incoming packets if the associated port is not allowed for your software. I noticed that recvfrom works once, just after having sent packet through the same port. May be this is linked to my computer's config.
I have a program that serves both as client and as server without multi-threading (as far as I know accept should let the program continue up until a certain connection is occurs).
The thing is, that my friend has a very similar program (not multithreaded) that also serves as both client AND server and it totally works, I'm trying to accomplish the same thing and accept() stops the program.
The code is as the following:
//main.cpp
#include <iostream>
#include "Client.h"
#include "Server.h"
#pragma comment(lib, "Ws2_32.lib")
int main()
{
WSADATA wsaData;
WSAStartup(MAKEWORD(2,2), &wsaData);
Server s(6666);
Client c("127.0.0.1", 6666);
cout << "Done";
WSACleanup();
return 0;
}
Server.cpp (two variables, SOCKET _socket and struct sockaddr_in _details):
Server::Server(unsigned short Port) : _socket(0)
{
this->_socket = socket(AF_INET, SOCK_STREAM, 0);
if (_socket < 0)
throw "Invalid socket";
ZeroMemory(&this->_details, sizeof(this->_details));
this->_details.sin_family = AF_INET;
this->_details.sin_port = htons(Port);
this->_details.sin_addr.S_un.S_addr = htonl(INADDR_ANY);
if (bind(this->_socket, (const struct sockaddr*)&this->_details, sizeof(this->_details)) != 0)
{
throw "Bind Unsuccessful";
}
this->AcceptConnections();
}
void Server::AcceptConnections()
{
if (listen(this->_socket, SOMAXCONN) != 0)
throw "Listen Unsuccessful";
void* buf = NULL;
string ans("Accepted");
int client;
struct sockaddr_in client_addr;
int addrlen = sizeof(client_addr);
client = accept(this->_socket, (struct sockaddr*)&client_addr, &addrlen);
/*THIS IS WHERE THE PROGRAM STOPS... AWAITING CONNECTIONS*/
//NEVER REACHING THE CODE HERE
int recvBytes = this->Receive(buf, MY_MAX_LEN);
if (recvBytes <= 0)
{
throw "Client disconnected";
}
this->Send((void*)ans.c_str(), ans.length());
closesocket(client);
closesocket(this->_socket);
}
And client.cpp is irrelevant as it doesn't even encounter its code.
Why does this happen? How come my friend has a code with no multi-threading that has both client and server. By the way, Send and Receive are functions implemented by me.
Here is an example of echo TCP server from libevent book.
How can I modify it to close a socket after each write. I know that I can close a socket connection by calling bufferevent_free(), but I don't understand how use it to close connection every time I send an echo to the socket.
#include <event2/listener.h>
#include <event2/bufferevent.h>
#include <event2/buffer.h>
#include <arpa/inet.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
static void
echo_read_cb(struct bufferevent *bev, void *ctx)
{
/* This callback is invoked when there is data to read on bev. */
struct evbuffer *input = bufferevent_get_input(bev);
struct evbuffer *output = bufferevent_get_output(bev);
/* Copy all the data from the input buffer to the output buffer. */
evbuffer_add_buffer(output, input);
}
static void
echo_event_cb(struct bufferevent *bev, short events, void *ctx)
{
if (events & BEV_EVENT_ERROR)
perror("Error from bufferevent");
if (events & (BEV_EVENT_EOF | BEV_EVENT_ERROR)) {
bufferevent_free(bev);
}
}
static void
accept_conn_cb(struct evconnlistener *listener,
evutil_socket_t fd, struct sockaddr *address, int socklen,
void *ctx)
{
/* We got a new connection! Set up a bufferevent for it. */
struct event_base *base = evconnlistener_get_base(listener);
struct bufferevent *bev = bufferevent_socket_new(
base, fd, BEV_OPT_CLOSE_ON_FREE);
bufferevent_setcb(bev, echo_read_cb, NULL, echo_event_cb, NULL);
bufferevent_enable(bev, EV_READ|EV_WRITE);
}
static void
accept_error_cb(struct evconnlistener *listener, void *ctx)
{
struct event_base *base = evconnlistener_get_base(listener);
int err = EVUTIL_SOCKET_ERROR();
fprintf(stderr, "Got an error %d (%s) on the listener. "
"Shutting down.\n", err, evutil_socket_error_to_string(err));
event_base_loopexit(base, NULL);
}
int
main(int argc, char **argv)
{
struct event_base *base;
struct evconnlistener *listener;
struct sockaddr_in sin;
int port = 9876;
if (argc > 1) {
port = atoi(argv[1]);
}
if (port<=0 || port>65535) {
puts("Invalid port");
return 1;
}
base = event_base_new();
if (!base) {
puts("Couldn't open event base");
return 1;
}
/* Clear the sockaddr before using it, in case there are extra
* platform-specific fields that can mess us up. */
memset(&sin, 0, sizeof(sin));
/* This is an INET address */
sin.sin_family = AF_INET;
/* Listen on 0.0.0.0 */
sin.sin_addr.s_addr = htonl(0);
/* Listen on the given port. */
sin.sin_port = htons(port);
listener = evconnlistener_new_bind(base, accept_conn_cb, NULL,
LEV_OPT_CLOSE_ON_FREE|LEV_OPT_REUSEABLE, -1,
(struct sockaddr*)&sin, sizeof(sin));
if (!listener) {
perror("Couldn't create listener");
return 1;
}
evconnlistener_set_error_cb(listener, accept_error_cb);
event_base_dispatch(base);
return 0;
}
From my understanding of documentation - you should put bufferevent_free(bev); at the end of echo_event_cb() function, this should close the connection after echoing user data back w/o waiting the client to close it. This should work this way till you use BEV_OPT_CLOSE_ON_FREE when creating bev buffer event.