I have project where multiple ZMQ PUSHers (impemented in nodejs) push data to different tcp ports. I need to PULL those data into a single C++ application. My current implementation uses different threads to PULL from each port. Is there any built-in method to connect to multiple ports to a same socket in zeromq?
pull thread (each thread connects to a different pull port)
uint16_t port = 4001;
void *context_pull = zmq_ctx_new();
void *requester_pull = zmq_socket(context_pull, ZMQ_PULL);
zmq_setsockopt(requester_pull, ZMQ_RCVTIMEO, &timeout, sizeof(int));
int len = sprintf(pub_url, "%s%s%u", (char *) "tcp://", "127.0.0.1:", port);
pub_url[len] = '\0';
int pull = zmq_connect(requester_pull, pub_url);
if (pull < 0) {
printf("CAN NOT BIND TO PORT %s\n", pub_url);
} else {
printf("PULL PROCESS CONNECTED TO PORT %s\n", pub_url);
}
char buf[256];
while (1) {
int rep = zmq_recv(requester_pull, buf, sizeof (buf), 0);
if (rep > 0) {
add_log(json::parse(std::string(buf))); //this method handles the received json data
} else {
printf("DATA PULL ERROR!"\n");
}
}
Instead of using threads for each socket, you can use zmq_poll() to wait on multiple sockets in a single thread. When data is available on a socket, zmq_poll() returns and you can process all the sockets that have data available.
zmq_poll() can also wait on "normal" file descriptors if that is required.
See the documentation for more details, including an example.
Related
I am relatively new to network programming and multithreading in C++. Currently my recv() call returns an unknown error. I'm not quite sure where the error coming from at the moment and would appreciate some help.
I used putty to connect to the server locally
class Threaded_TCPListener{
int Threaded_TCPListener::Init()
{
// Initializing WinSock
WSADATA wsData;
WORD ver = MAKEWORD(2,2);
int winSock = WSAStartup(ver, &wsData);
if(winSock != 0)
return winSock;
// Creating listening socket
this->socket = ::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if(this->socket == INVALID_SOCKET)
return WSAGetLastError();
// Fill sockaddr with ip addr and port
sockaddr_in hint;
hint.sin_family = AF_INET;
hint.sin_port = htons(this->port);
inet_pton(AF_INET, this->ipAddress, &hint.sin_addr);
// Bind hint to socket
if(bind(this->socket, (sockaddr*)&hint, sizeof(hint)) == SOCKET_ERROR)
return WSAGetLastError();
// Start listening on socket
if(listen(this->socket, SOMAXCONN) == SOCKET_ERROR)
return WSAGetLastError();
// Accept first client
this->createAcceptThread();
return 0;
}
int Threaded_TCPListener::Run()
{
bool isRunning = true;
// Read from all clients
std::vector<std::thread> threads;
threads.reserve(this->clients.size());
// Recv from client sockets
for (int i=0; i < this->clients.size(); ++i)
{
threads.emplace_back(std::thread(&Threaded_TCPListener::receiveFromSocket, this, socket));
}
// Wait for all threads to finish
for(std::thread& t : threads)
{
t.detach();
}
return 0;
}
void Threaded_TCPListener::onMessageReceived(int clientSocket, const char* msg, int length)
{
Threaded_TCPListener::broadcastToClients(clientSocket, msg, length);
std::thread t(&Threaded_TCPListener::receiveFromSocket, this, clientSocket);
t.detach();
return;
}
void Threaded_TCPListener::sendMessageToClient(int clientSocket, const char * msg, int length)
{
send(clientSocket, msg, length, 0);
return;
}
void Threaded_TCPListener::broadcastToClients(int senderSocket, const char * msg, int length)
{
std::vector<std::thread> threads;
threads.reserve(clients.size());
// Iterate over all clients
for (int sendSock : this->clients)
{
if(sendSock != senderSocket)
threads.emplace_back(std::thread(&Threaded_TCPListener::sendMessageToClient, this,sendSock, msg, length));
}
// Wait for all threads to finish
for(std::thread& t : threads)
t.join();
return;
}
void Threaded_TCPListener::createAcceptThread()
{
// Start accepting clients on a new thread
this->listeningThread = std::thread(&Threaded_TCPListener::acceptClient, this);
this->listeningThread.detach();
return;
}
void Threaded_TCPListener::acceptClient()
{
int client = accept(this->socket, nullptr, nullptr);
// Error
if(client == INVALID_SOCKET)
{
std::printf("Accept Err: %d\n", WSAGetLastError());
}
// Add client to clients queue
else
{
// Add client to queue
this->clients.emplace(client);
// Client Connect Confirmation
onClientConnected(client); // Prints msg on server
// Create another thread to accept more clients
this->createAcceptThread();
}
return;
}
void Threaded_TCPListener::receiveFromSocket(int receivingSocket)
{
// Byte storage
char buff[MAX_BUFF_SIZE];
// Clear buff
memset(buff, 0, sizeof(buff));
// Receive msg
int bytesRecvd = recv(receivingSocket, buff, MAX_BUFF_SIZE, 0);
if(bytesRecvd <= 0)
{
char err_buff[1024];
strerror_s(err_buff, bytesRecvd);
std::cerr << err_buff;
// Close client
this->clients.erase(receivingSocket);
closesocket(receivingSocket);
onClientDisconnected(receivingSocket); // Prints msg on server
}
else
{
onMessageReceived(receivingSocket, buff, bytesRecvd);
}
}
}
I am trying to create a multithreaded TCP 'server' that will handle incoming clients by having an accept thread continuously running (listening for new connections), and a thread waiting with a recv block for each client connected to the server.
Your Init looks fine:
create socket, bind it, listen on it, start accept thread
In your accept thread's acceptClient looks sort of OK:
print some message
add the client socket to clients queue
create a new accept thread
Your Run makes no sense:
create one thread per element in clients to receive from the listening socket
It looks like you are spawning a new thread for every single socket action. That is a pretty wasteful design. As soon as the thread is done it can go back to doing something else.
So creating a new accept thread in acceptClient is a waste, you could just loop back to the beginning to ::accept the next client. Like so:
acceptClient() {
while (alive) {
int client = accept(socket, ...);
createClientHandler(client);
}
}
What seems to be missing is spawning a new client thread to service the client socket. You currently do this in Run, but that's before any of the clients are actually accepted. And you do it for the wrong socket! Instead, you should be spawning the receiveFromSocket threads in acceptClient, and passing it the client socket. So that's a bug.
In your receiveFromSocket you also need not create another thread to receiveFromSocket again -- just loop back to the beginning.
The biggest concern with this thread-per-action design is that you are spawning sender threads on every incoming message. This means you could actually have several sender threads attempting to ::send on the same TCP socket. That's not very safe.
The order of calls made to WSASend is also the order in which the buffers are transmitted to the transport layer. WSASend should not be called on the same stream-oriented socket concurrently from different threads, because some Winsock providers may split a large send request into multiple transmissions, and this may lead to unintended data interleaving from multiple concurrent send requests on the same stream-oriented socket.
https://learn.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsasend
Similarly, instead of spawning threads in broadcastToClients, I suggest you just spawn one persistent sender thread per client socket in acceptClient (together with the receiveFromSocket thread within some createClientHandler).
To communicate with the sender threads you should use thread-safe blocking queues. Each sender thread would look like this:
while (alive) {
msg = queue.next_message();
send(client_socket, msg);
}
Then on message received you just do:
for (client : clients) {
client.queue.put_message(msg);
}
So to summarize, to handle each client you need a structure like this:
struct Client {
int client_socket;
BlockingQueue queue;
// optionally if you want to keep track of your threads
// to be able to safely clean up
std::thread recv_thread, send_thread;
};
Safe cleanup is a whole other story.
Finally, a remark on this comment in your code:
// Wait for all threads to finish
for(std::thread& t : threads)
{
t.detach();
}
That's almost the opposite to what std::thread::detach does:
https://en.cppreference.com/w/cpp/thread/thread/detach
It allows you to destroy the thread object without having to wait for the thread to finish execution.
There is a misconception in the code in how a TCP server has to be implemented:
You seem to assume that you can have a single server socket file descriptor which can handle all communication. This is not the case. You must have a single dedicated socket file descriptor which is just used for listening and accepting incoming connections, and then you have one additional file descriptor for each existing connection.
In your code I see that you invoke receiveFromSocket() always with the listening socket. This is wrong. Also invoking receiveFromSocket() in a loop for all clients is wrong.
What you rather need to do is:
- Have one dedicated thread which call accept() in a loop. There is no performance benefit in calling accept() from multiple threads.
- One accept() returns a new connection you spawn a new thread which calls recv() in a loop. This will then block and wait for new data as you expect in your question.
You also need to drop the habit of calling individual functions from new threads. This is not multithreaded programming. A thread usually contains a loop. Everything else is usually a design flaw.
Also note that multithreaded programming is still rocket science in 2019, especially in C++. If you are not an absolute expert you will not be able to do it. Also note that absolute experts in multithreaded programming will try to avoid multithreaded programming whenever possible. A lot seemingly concurrent tasks which are I/O bound can better be handled by a single threaded event based system.
I'm trying to build a simulation for multiple socket clients.
My server has the following code to listen to multiple clients
My socket are from a very simple class drive from CAsyncSocket and my environment is windows MFC.
m_server.Create(....); // with the correct values
if (m_server.Listen()==FALSE)
and later on the OnSocketAccept() function
if (m_server.Accept(tempSock))
{
CSocketThread* pThread = (CSocketThread*)AfxBeginThread(RUNTIME_CLASS(CSocketThread), THREAD_PRIORITY_NORMAL, 0, CREATE_SUSPENDED);
...
My simulation apps has the following code:
for (int i = 0; i < numOfClients; i++)
{
m_sConnected[i].Create();
int rVal = m_sConnected[i].Connect(csIPAddress.GetString(), m_port);
That doesn't work.
In WireShark I can see that my (numOfClients = 10 for example) 10 clients are connected with different client source port.
But each new socket of m_sConnected[i] is becoming NULL after the second connection to all sockets including m_sConnected[0].
Closing the sockets or destroy the simulation app create socket close at the server side for all open threads for the listen sockets.
What is the problem?
Can I use the same process/thread for all my socket clients?
10x
UrAv.
your problem is that you are not using the CSocketThread object the right way.
as mentiend in microsoft documention
after the accept function you need to do the following :
CSockThread* pSockThread = (CSockThread*)AfxBeginThread( RUNTIME_CLASS(CSockThread), THREAD_PRIORITY_NORMAL, 0, CREATE_SUSPENDED);
if (NULL != pSockThread) {
// Detach the newly accepted socket and save
//the SOCKET handle in our new thread object.
//After detaching it, it should no longer be
//used in the context of this thread.
pSockThread->m_hConnected = sConnected.Detach();
pSockThread->ResumeThread();
} }
when you attach your socket to the thread then it will run.
link to microsoft doc:
https://msdn.microsoft.com/en-us/library/wxzt95kb.aspx
your solution has worked for me. I have used multiple threads to stress test the server in c++ under linux. Pasting my code, it will be helpful to somebody...Experts can improve my code, if they find any flaws in my handling of code. I know, I am doing something wrong but no other go to test the server as no one has provided the solution for this till now. I am able to test the server for 100000 clients using this code. - Kranti.
include //for threading , link with lpthread
void *connection_handler(void *);
#define PORT 9998
#define SERVER_IP "127.0.0.1"
#define MAXSZ 100
#define MAXSOCK 70000
int main()
{
int sockfd[MAXSOCK];//to create socket
int socket_desc , new_socket[MAXSOCK], *new_sock;
struct sockaddr_in serverAddress;//client will connect on this
int n;
char msg1[MAXSZ];
char msg2[MAXSZ];
int NoOfClients = MAXSOCK;
memset(msg2,0,100);
void *ret;
for(int i=0;i<NoOfClients;i++){
//create socket
sockfd[i]=socket(AF_INET,SOCK_STREAM,0);
//initialize the socket addresses
memset(&serverAddress,0,sizeof(serverAddress));
serverAddress.sin_family=AF_INET;
serverAddress.sin_addr.s_addr=inet_addr(SERVER_IP);
serverAddress.sin_port=htons(PORT);
//client connect to server on port
new_socket[i] = connect(sockfd[i],(struct sockaddr *)&serverAddress,sizeof(serverAddress));
printf("new socket connected= %d",new_socket[i]);
pthread_t sniffer_thread[MAXSOCK];
new_sock = malloc(sizeof(int));
*new_sock = new_socket[i];
int p=-1;
if( p = pthread_create( &sniffer_thread[i] , NULL , connection_handler , (void*) new_sock) < 0)
{
perror("could not create thread");
return 1;
}
}
return 0;
}
/*
* This will handle connection for each client
* */
void *connection_handler(void *socket_desc)
{
//Get the socket descriptor
int sock = *(int*)socket_desc;
int read_size;
char *message , client_message[50];
printf("we are in connection handler");
//Send some messages to the server
message = "Greetings! I am your connection handler\n";
int wlen = write(sock , message , strlen(message));
printf("write length is %d", wlen);
//Free the socket pointer
//close(sock);
free(sock);
return 0;
}
I have a simple socket server set up using sys/socket and OpenSSL. For each connection, the client is required to send a message to the server, receive a response and then reply to that response.
I can't find any clear mechanism for making these sockets non-blocking? The system has to be able to handle multiple sockets concurrently...
My server code for listening for connections:
while(1)
{
struct sockaddr_in addr;
uint len = sizeof(addr);
SSL *ssl;
int client = accept(sock, (struct sockaddr*)&addr, &len);
if (client > 0)
{
std::cout<<"Client accepted..."<<std::endl;
}
else
{
perror("Unable to accept");
exit(EXIT_FAILURE);
}
ssl = SSL_new(ctx);
SSL_set_fd(ssl, client);
if (SSL_accept(ssl) <= 0)
{
std::cout<<"ERROR"<<std::endl;
}
else
{
char buff[1024];
SSL_read(ssl, buff, 1024);
std::cout<<buff<<std::endl;
std::string reply="Thanks from the server";
char buff_response[1024];
reply.copy(buff_response, 1024);
const void *buf=&buff_response;
SSL_write(ssl, buf, 1024);
char another_buff[1024];
SSL_read(ssl,another_buff,1024);
std::cout<<another_buff<<std::endl;
}
}
I've looked into 'select()', however this doesn't seem to allow concurrency as such, but allows the system to know when a socket is freed?
Does anyone have any experience in solving this basic problem?
First, with server code, it's important to differentiate between concurrency and parallelism. A reasonable server will typically handle many more connections concurrently than its number of cores. Consequently, it's important to make the code concurrent in the sense that it can (efficiently) handle many concurrent connections, in a way that does not rely on parallelism (in the sense of having each connection handled by a thread).
In this sense, select is actually a reasonable choice for concurrency, and gives you the effect of being non-blocking.
When your system handles multiple sockets concurrently, select indicates on which socket(s) you can perform operations such as send and recv without their blocking when you do so. If you use select well you won't have cases where your thread is idling, waiting indefinitely for some operation to proceed, while other sockets are ready.
The minimal example from gnu.org shows a reasonably efficient server which it seems you can adapt to your needs.
fd_set active_fd_set, read_fd_set;
FD_ZERO (&active_fd_set);
FD_ZERO (&read_fd_set);
// Use FD_SET to add sockets according to what you want to do with them
/* This call (checking to see who can be read) is the
* only thing that blocks. But if it does, no socket is ready for reading. */
if (select (FD_SETSIZE, &read_fd_set, NULL, NULL, NULL) < 0) {
// Handle error;
for (i = 0; i < FD_SETSIZE; ++i)
if (FD_ISSET (i, &read_fd_set))
// Here you can read without its blocking.
Firstly I'm coding in c++ and running in Linux/CentOS 6.4
So after a really long time and reading a lot of different books on sockets, I finally have at least my client and my server partially working.
First I want to continuously accept messages from different clients, I have already setup the client, and it finally successfully compiled at least. Now I need to set up my server so that I can properly test.
What I'm doing is implementing the dining philosopher problem with sockets, with each client/philosopher representing a different process. I was going to go through this whole thing, where the server was going to keep track of everything, like the states of all the client. That was too difficult, I have now just created the client just to send their status to the server and the server prints it out.
I was thinking of putting a do/while loop to continuously accept messages, but not sure what I should use to stop the loop. Note that I will have a while loop set up in my client, which is signaled to stop after an elapsed amount of time. It should then close that particular client. I do have a signal in my serve, but I am not sure it works.
#include "helper.h"
char buffer[4096];
void sigchld_handler(int signo)
{
while (waitpid(-1, NULL, WNOHANG) > 0);
}
void client(int &newsock, int nread)
{
do
{
int nread = recv(newsock, buffer,sizeof(buffer), 0);
puts(buffer);
}while(nread!=0);
}
int main(int argc, char *argv[])
{
struct sockaddr_in sAddr, cli_addr;
socklen_t client_len;
int listensock;
int newsock;
int result;
int nread=1;
pid_t childid; ;
int status;
if((listensock = socket(AF_INET, SOCK_STREAM, 0))<0)
{
perror("Problem in creating socket");
exit(2);
}
sAddr.sin_family = AF_INET;
sAddr.sin_port = htons(3333);
sAddr.sin_addr.s_addr = htonl(INADDR_ANY);
bind(listensock, (struct sockaddr *) &sAddr, sizeof(sAddr));
if (result < 0) {
perror("exserver2");
return 0;
}
result = listen(listensock, 5);
if (result < 0) {
perror("exserver2");
return 0;
}
signal(SIGCHLD, sigchld_handler);
while (1) {
client_len = sizeof(cli_addr);
newsock = accept(listensock,(struct sockaddr *)&cli_addr, &client_len);
if ((childid = fork()) == 0) {
printf("child process %i created.\n", getpid());
close(listensock);
client(newsock, nread);
}
if(status<0)
{
printf("%s\n" "Read error");
exit(1);
}
close(newsock);
}
}
You need a multiplexing syscall like poll(2) (or the old, nearly obsolete, select(2)syscall). You may want to use some (or implement your own) event loop. See this & that answer. Read about the C10K problem.
Every server needs an event loop.
Read Advanced Linux Programming (or some Posix network programming book).
You may want to simply run your server program under tcpserver (see http://cr.yp.to/ucspi-tcp.html). This will spawn a new instance of your program each time a client connects to your program (on the port that you specify). This way, you can focus on the core logic of your program, and let tcpserver handle all of the heavy lifting as far as the socket programming, etc. tcpserver will pass input from the client to your program's stdin, and output from your programs stdout will be sent back to the client.
i have a server and client classes but the problem is: when i make infinite loop to accept incoming connection i cant receive all the data received from the client while accepting the connections because accept blocks until the connection is accepted, my code:
for (;;)
{
boost::thread thread(boost::bind(&Irc::Server::startAccept, &s));
thread.join();
for (ClientsMap::const_iterator it = s.begin(); it != s.end(); ++it)
{
std::string msg = getData(it->second->recv());
std::clog << "Msg: " << msg << std::endl;
}
}
You need either multiple threads or a call to select/poll to find out which connections have unprocessed data. IBM has a nice example here, which will work on any flavor of Unix, Linux, BSD, etc. (you might need different header files depending on the OS).
Right now you're starting a thread and then waiting for it immediately, which results in sequential execution and completely defeats the purpose of threads.
Take a look here : http://www.boost.org/doc/libs/1_38_0/doc/html/boost_asio/examples.html
especially the HTTP Server 3 example, thats exactly what you are looking for , all you have to do is change that code a little bit for your needs :) and your done
A good approach would be to create one thread that only accepts new connections. That's where you have a listener socket. Then, for every connection that gets accepted, you have a new connected socket, so you can spawn another thread, giving it the connected socket as a parameter. That way, your thread that accepts connections doesn't get blocked, and can connect to many clients very fast. The processing threads deal with the clients and then they exit.
I don't even know why need to wait for them, but if you do, you may deal with it in some other way, depending on the OS and/or libraries that you use (messages, signals etc can be used).
If you don't want to spawn a new thread for each connected client, then as Ben Voigt suggested, you can use select. That is another good approach if you want to make it single threaded. Basically, all your sockets will be in an array of socket descriptors and using select you will know what happened (someone connected, socket is ready for read/write, socket got disconnected etc) and act accordingly.
Here's one example Partial one, but it works. you just accept connections in the acceptConnections(), which will then spawn a separate thread for each client. That's where you communicate with the clients. It's from a windows code that i have lying around, but it's very easy to be reimplemented for any platform.
typedef struct SOCKET_DATA_ {
SOCKET sd;
/* other parameters that you may want to pass to the clientProc */
} SOCKET_DATA;
/* In this function you communicate with the clients */
DWORD WINAPI clientProc(void * param)
{
SOCKET_DATA * pSocketData = (SOCKET_DATA *)param;
/* Communicate with the new client, and at the end deallocate the memory for
SOCKET_DATA and return.
*/
delete pSocketData;
return 0;
}
int acceptConnections(const char * pcAddress, int nPort)
{
sockaddr_in sinRemote;
int nAddrSize;
SOCKET sd_client;
SOCKET sd_listener;
sockaddr_in sinInterface;
SOCKET_DATA * pSocketData;
HANDLE hThread;
sd_listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (INVALID_SOCKET == sd_listener) {
fprintf(stderr, "Could not get a listener socket!\n");
return 1;
}
sinInterface.sin_family = AF_INET;
sinInterface.sin_port = nPort;
sinInterface.sin_addr.S_un.S_addr = INADDR_ANY;
if (SOCKET_ERROR != bind(sd_listener, (sockaddr*)&sinInterface, sizeof(sockaddr_in))) {
listen(sd_listener, SOMAXCONN);
} else {
fprintf(stderr, "Could not bind the listening socket!\n");
return 1;
}
while (1)
{
nAddrSize = sizeof(sinRemote);
sd_client = accept(sd_listener, (sockaddr*)&sinRemote, &nAddrSize);
if (INVALID_SOCKET == sd_client) {
fprintf(stdout, "Accept failed!\n");
closesocket(sd_listener);
return 1;
}
fprintf(stdout, "Accepted connection from %s:%u.\n", inet_ntoa(sinRemote.sin_addr), ntohs(sinRemote.sin_port));
pSocketData = (SOCKET_DATA *)malloc(sizeof(SOCKET_DATA));
if (!pSocketData) {
fprintf(stderr, "Could not allocate memory for SOCKET_DATA!\n");
return 1;
}
pSocketData->sd = sd_client;
hThread = CreateThread(0, 0, clientProc, pSocketData, 0, &nThreadID);
if (hThread == INVALID_HANDLE_VALUE) {
fprintf(stderr, "An error occured while trying to create a thread!\n");
delete pSocketData;
return 1;
}
}
closesocket(sd_listener);
return 0;
}