Related
I made a single server socket that i want to allow multiple connections in a multithreaded fasion but there's an issue. It drops messages from clients for no apparent reason
Each socket is handled by their own thread so my guess was that it shouldn't be an issue (may be it is).
Here is the code
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>
#include <WinSock2.h>
#include <WS2tcpip.h>
#include <iostream>
#include <thread>
#include <mutex>
#include <chrono>
#include <vector>
#include <sstream>
#include <cassert>
// Taken from: https://stackoverflow.com/a/46104456/6119618
static std::string wsa_error_to_string(int wsa_error)
{
char msgbuf [256]; // for a message up to 255 bytes.
msgbuf [0] = '\0'; // Microsoft doesn't guarantee this on man page.
FormatMessage(
FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, // flags
nullptr, // lpsource
wsa_error, // message id
MAKELANGID (LANG_NEUTRAL, SUBLANG_DEFAULT), // languageid
msgbuf, // output buffer
sizeof (msgbuf), // size of msgbuf, bytes
nullptr
);
if (! *msgbuf)
sprintf (msgbuf, "%d", wsa_error); // provide error # if no string available
return msgbuf;
}
#define PRINT_ERROR_AND_TERMINATE(MSG) do { std::cerr << (MSG) << std::endl; assert(0); } while(0)
struct wsa_lifetime
{
wsa_lifetime()
{
int result = ::WSAStartup(MAKEWORD(2,2), &wsa_data);
assert(result == 0);
is_initialized = true;
}
~wsa_lifetime()
{
::WSACleanup();
}
WSAData wsa_data {};
bool is_initialized {false};
};
static wsa_lifetime wsa_lifetime;
static SOCKET socket_create()
{
SOCKET socket = ::socket(AF_INET, SOCK_STREAM, 0);
assert(socket != INVALID_SOCKET);
return socket;
}
static void socket_destroy(SOCKET socket)
{
::closesocket(socket);
socket = INVALID_SOCKET;
}
static void socket_bind(SOCKET socket, const char *address, uint16_t port)
{
sockaddr_in addr {};
addr.sin_family = AF_INET;
inet_pton(AF_INET, address, &addr.sin_addr.s_addr);
addr.sin_port = htons(port);
int bind_result = ::bind(socket, reinterpret_cast<SOCKADDR *>(&addr), sizeof(addr));
if (bind_result == SOCKET_ERROR)
PRINT_ERROR_AND_TERMINATE(WSAGetLastError());
}
static void socket_connect(SOCKET socket, const char *address, uint16_t port)
{
sockaddr_in addr {};
addr.sin_family = AF_INET;
inet_pton(AF_INET, address, &addr.sin_addr.s_addr);
addr.sin_port = htons(port);
int connect_result = ::connect(socket, reinterpret_cast<SOCKADDR *>(&addr), sizeof(addr));
if (connect_result == SOCKET_ERROR)
PRINT_ERROR_AND_TERMINATE(WSAGetLastError());
}
static void socket_listen(SOCKET socket)
{
int listen_result = ::listen(socket, SOMAXCONN);
if (listen_result == SOCKET_ERROR)
PRINT_ERROR_AND_TERMINATE(WSAGetLastError());
}
static SOCKET socket_accept(SOCKET socket)
{
SOCKET accepted_socket = ::accept(socket, nullptr, nullptr);
if (accepted_socket == INVALID_SOCKET)
PRINT_ERROR_AND_TERMINATE(WSAGetLastError());
return accepted_socket;
}
static size_t socket_recv(SOCKET socket, char *buffer, size_t buffer_size, int flags = 0)
{
int bytes_received = ::recv(socket, buffer, static_cast<int>(buffer_size), flags);
if (bytes_received == SOCKET_ERROR)
{
int err = WSAGetLastError();
if (err == WSAECONNRESET)
return 0; // Disconnected client
PRINT_ERROR_AND_TERMINATE(WSAGetLastError());
}
return bytes_received;
}
static size_t socket_send(SOCKET socket, const char *data, size_t data_size, int flags = 0)
{
int bytes_sent = ::send(socket, data, static_cast<int>(data_size), flags);
if (bytes_sent == SOCKET_ERROR)
{
int err = WSAGetLastError();
if (err == WSAECONNRESET)
return 0; // Disconnected client
PRINT_ERROR_AND_TERMINATE(WSAGetLastError());
}
return bytes_sent;
}
static std::mutex output_mutex;
int main()
{
const char *server_address = "127.0.0.1";
uint16_t server_port = 23456;
bool server_terminate = false;
std::thread server_thread([server_address, server_port, &server_terminate](){
SOCKET server = socket_create();
socket_bind(server, server_address, server_port);
socket_listen(server);
std::vector<SOCKET> clients;
std::vector<std::thread> client_threads;
while (!server_terminate)
{
SOCKET incoming_client = socket_accept(server);
if (server_terminate)
break;
clients.push_back(incoming_client);
size_t client_id = clients.size();
std::thread incoming_client_thread([&incoming_client, client_id](){
const size_t data_size = 1024;
char data[data_size];
while (true)
{
size_t bytes_received = socket_recv(incoming_client, data, data_size, 0);
if (bytes_received == 0)
break;
std::string_view client_message(data, bytes_received);
{
std::unique_lock lock(output_mutex);
std::cout << "Client (" << client_id << "): " << client_message << std::endl;
}
}
});
client_threads.push_back(std::move(incoming_client_thread));
}
for (std::thread &client_thread: client_threads)
if (client_thread.joinable())
client_thread.join();
});
std::vector<SOCKET> clients;
std::vector<std::thread> client_threads;
for (int i = 0; i < 4; i++)
{
SOCKET client = socket_create();
clients.push_back(client);
}
for (SOCKET client : clients)
{
std::thread client_thread([server_address, server_port, client](){
socket_connect(client, server_address, server_port);
for (int i = 0; i < 10; i++)
{
std::string data_str = (std::stringstream() << "hello " << i).str();
socket_send(client, data_str.c_str(), data_str.size());
using namespace std::chrono_literals;
std::this_thread::sleep_for(100ms + 1ms * (rand() % 100));
}
});
client_threads.push_back(std::move(client_thread));
}
for (std::thread &client_thread : client_threads)
if (client_thread.joinable())
client_thread.join();
for (SOCKET client: clients)
socket_destroy(client);
clients.clear();
server_terminate = true;
SOCKET dummy_socket = socket_create();
socket_connect(dummy_socket, server_address, server_port); // just to unblock server's socket_accept() blocking call
socket_destroy(dummy_socket);
if (server_thread.joinable())
server_thread.join();
return 0;
}
Possible output:
Client (2): hello 0
Client (2): hello 0
Client (3): hello 1
Client (2): hello 2
Client (1): hello 3
Client (4): hello 4
Client (3): hello 5
Client (2): hello 6
Client (1): hello 7
Client (4): hello 8
Client (3): hello 9
I expected each client to send 10 messages, 40 in total but some messages are dropped as you can see. I think it shouldn't drop even with UDP transport because all job is done on my loopback network
Wireshark registers all the messages
When constructing the lambda incoming_client_thread, you capture incoming_client by reference and not by copy.
Since this variable is reset at the start of each loop by socket_accept, a thread might not be calling socket_recv on the same socket once another socket_accept succeeds.
I am trying to understand how to use the epoll library (Linux) in combination with socket file descriptors. There is limited information about epoll available online as far as I can tell. So far the most useful resource I have found is this:
https://www.suchprogramming.com/epoll-in-3-easy-steps/
Which gives a complete working example of epoll however it is used with the stdin file descriptor. (Elsewhere I read that epoll can not be used with stdin, stdout and stderr. I assume that information was wrong.)
I have tried to build on this with a MWE.
#include <string>
#include <sstream>
#include <iostream>
#include <fstream>
#include <string.h>
#include <unistd.h>
#include <future>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/epoll.h>
int thread_function(int accept_sock_fd)
{
char *read_buffer = new char[1024];
unsigned int read_length = 0;
std::string read_string = "";
for(;;)
{
unsigned int t_read_length = read(accept_sock_fd, read_buffer, 1024 - 1);
//std::cout << "t_read_length=" << t_read_length << std::endl;
//std::cout << read_buffer << std::endl;
read_string += std::string(read_buffer);
read_length += t_read_length;
// dodgy ?
// tries to detect end of data using new lines
// but what if client does not send new line char?
if(read_buffer[t_read_length - 1] == '\n')
{
break;
}
}
std::cout << read_string << std::endl;
//std::cout << read_length << std::endl;
//std::cout << read_buffer << std::endl;
std::string status_line = "HTTP/1.0 200 OK\r\n\r\n";
std::string html_lines = "Hello World\r\n\r\n";
std::string write_string = status_line + html_lines;
//std::cout << write_string << std::endl;
unsigned int write_length = write(accept_sock_fd, write_string.data(), write_string.length());
//std::cout << "write_length=" << write_length << std::endl;
close(accept_sock_fd);
delete [] read_buffer;
return 0;
}
int main()
{
// read configuration file
//Config config;
//config.ReadFromFile("config.txt");
//std::cout << config << std::endl;
unsigned int num_threads = 4; //config.GetNumThreads();
///////////////////////////////////////////////////////////////////////////
// init epoll
//
//
int epoll_fd = epoll_create(num_threads);
if(epoll_fd < 0)
{
std::cout << "epoll error" << std::endl;
}
struct epoll_event event;
struct epoll_event *events = new struct epoll_event[num_threads * sizeof(struct epoll_event)];
///////////////////////////////////////////////////////////////////////////
// create socket fd
//
//
int sock_fd = socket(AF_INET, SOCK_STREAM, 0);
if(sock_fd == 0)
{
std::cout << "Error creating sock_fd" << std::endl;
}
// int setsockopt(sockfd,
struct sockaddr_in server_address;
memset(&server_address, 0, sizeof(struct sockaddr_in));
server_address.sin_family = AF_INET;
server_address.sin_addr.s_addr = INADDR_ANY;
server_address.sin_port = htons(55555 /*config.GetPort()*/);
bind(sock_fd, (struct sockaddr *)&server_address, sizeof(server_address));
if(bind < 0)
{
std::cout << "Error: bind" << std::endl;
}
if(listen(sock_fd, 10) < 0)
{
std::cout << "Error: listen" << std::endl;
}
///////////////////////////////////////////////////////////////////////////
// continue epoll setup
//
//
event.events = EPOLLIN; // | EPOLLPRI | EPOLLERR | EPOLLHUP;
event.data.fd = 0; //client_sock;
if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sock_fd, &event)) // not sure about this
{
std::cout << "Failed to add epoll" << std::endl;
return 1;
}
for(;;)
{
int n_fd = epoll_wait(epoll_fd, events, num_threads, 0);
if(n_fd < 0)
{
std::cout << "die" << std::endl;
break;
}
for(int i = 0; i < n_fd; ++ i)
{
int fd = events[i].data.fd;
// accpt connection
int accept_sock_fd = 0;
struct sockaddr client_address;
int addrlen = sizeof(client_address);
accept_sock_fd = accept(fd, NULL, NULL); // not sure what goes here
accept_sock_fd = accept(sock_fd, NULL, NULL); // not sure what goes here
if(accept_sock_fd < 0)
{
std::cout << "Error: accept" << std::endl;
}
// do something to handle data on accept_sock_fd
std::future<int> thread_future = std::async(thread_function, accept_sock_fd);
int ret = thread_future.get();
std::cout << ret << std::endl;
// close connection
close(accept_sock_fd);
}
}
delete [] events;
close(sock_fd);
if(close(epoll_fd))
{
std::cout << "error: failed to close epoll fd" << std::endl;
}
return 0;
}
I have searched pretty extensively for further information on how to combine epoll with sockets, but there isn't much information about this. I suppose it is quite a niche area.
Ideally if anyone knows of any good resources (even a book) where I can find more information about this and thus solve the problem myself that would be great, otherwise some advice on how to proceed would be appreciated also.
I am mostly confused as to how I watch for events on the socket fd and also how I then accept events and handle the new fd returned by accept.
See these lines in the above code:
// correct fd
epoll_ctl(epoll_fd, EPOLL_CTL_ADD, /*the_server_socket_fd*/, &event);
...
// not sure about this one, particularly the waiting time of 0
for(;;)
epoll_wait(epoll_fd, events, /*num_threads*/, 0);
for(i ...)
fd = events[i].data.fd; // what fd is this? what does it mean?
accept(/*something here, but what*/ fd OR the_server_socket_fd ?, NULL, NULL);
// launch thread with retuned value from accept, probably
Using epoll is not so much different with using poll. The tricky thing is that keep track which client owns file descriptor returned by epoll (if there are a lot of clients).
How to handle socket file asynchronously with epoll (in this case as TCP socket server).
Open an epoll file descriptor with epoll_create(2).
Create a TCP socket with socket(2), bind(2) and listen(2).
Add the main TCP socket file descriptor to epoll with epoll_ctl + EPOLL_CTL_ADD.
Call epoll_wait inside a loop, the program will sleep on epoll_wait, the kernel will wake the program up when there is event coming from monitored file descriptors or when timeout is reached.
If epoll_wait returns a value greater than zero, then you need to decide, what file descriptor returned by epoll.
5.1. If it is main TCP file descriptor, then you need to accept(2). And then add the client file descriptor returned by accept(2) to epoll with epoll_ctl + EPOLL_CTL_ADD. Other
5.2. If it is client file descriptor, then you need to call recv(2) and do any action you want with that client.
At step 5.2, if you see EPOLLHUP in the events[i].events, then the client has closed its connection and you need to call epoll_ctl + EPOLL_CTL_DEL and close the client file descriptor (it may be safe not to call epoll_ctl + EPOLL_CTL_DEL and just close the client file descriptor, but I prefer to delete it from epoll first).
Goto step 4.
Detailed mechanism about how to determine which client owns file descriptor returned by epoll can be seen from server.c code below.
Flowchart
Flowchart can make it more clear.
Working example you can try at home
server.c
/*
* https://stackoverflow.com/questions/66916835/c-confused-by-epoll-and-socket-fd-on-linux-systems-and-async-threads
*/
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <stdbool.h>
#include <sys/epoll.h>
#include <arpa/inet.h>
#include <netinet/tcp.h>
#define PRERF "(errno=%d) %s\n"
#define PREAR(NUM) NUM, strerror(NUM)
#define EPOLL_MAP_TO_NOP (0u)
#define EPOLL_MAP_SHIFT (1u) /* Shift to cover reserved value MAP_TO_NOP */
struct client_slot {
bool is_used;
int client_fd;
char src_ip[sizeof("xxx.xxx.xxx.xxx")];
uint16_t src_port;
uint16_t my_index;
};
struct tcp_state {
bool stop;
int tcp_fd;
int epoll_fd;
uint16_t client_c;
struct client_slot clients[10];
/*
* Map the file descriptor to client_slot array index
* Note: We assume there is no file descriptor greater than 10000.
*
* You must adjust this in production.
*/
uint32_t client_map[10000];
};
static int my_epoll_add(int epoll_fd, int fd, uint32_t events)
{
int err;
struct epoll_event event;
/* Shut the valgrind up! */
memset(&event, 0, sizeof(struct epoll_event));
event.events = events;
event.data.fd = fd;
if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, fd, &event) < 0) {
err = errno;
printf("epoll_ctl(EPOLL_CTL_ADD): " PRERF, PREAR(err));
return -1;
}
return 0;
}
static int my_epoll_delete(int epoll_fd, int fd)
{
int err;
if (epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, NULL) < 0) {
err = errno;
printf("epoll_ctl(EPOLL_CTL_DEL): " PRERF, PREAR(err));
return -1;
}
return 0;
}
static const char *convert_addr_ntop(struct sockaddr_in *addr, char *src_ip_buf)
{
int err;
const char *ret;
in_addr_t saddr = addr->sin_addr.s_addr;
ret = inet_ntop(AF_INET, &saddr, src_ip_buf, sizeof("xxx.xxx.xxx.xxx"));
if (ret == NULL) {
err = errno;
err = err ? err : EINVAL;
printf("inet_ntop(): " PRERF, PREAR(err));
return NULL;
}
return ret;
}
static int accept_new_client(int tcp_fd, struct tcp_state *state)
{
int err;
int client_fd;
struct sockaddr_in addr;
socklen_t addr_len = sizeof(addr);
uint16_t src_port;
const char *src_ip;
char src_ip_buf[sizeof("xxx.xxx.xxx.xxx")];
const size_t client_slot_num = sizeof(state->clients) / sizeof(*state->clients);
memset(&addr, 0, sizeof(addr));
client_fd = accept(tcp_fd, (struct sockaddr *)&addr, &addr_len);
if (client_fd < 0) {
err = errno;
if (err == EAGAIN)
return 0;
/* Error */
printf("accept(): " PRERF, PREAR(err));
return -1;
}
src_port = ntohs(addr.sin_port);
src_ip = convert_addr_ntop(&addr, src_ip_buf);
if (!src_ip) {
printf("Cannot parse source address\n");
goto out_close;
}
/*
* Find unused client slot.
*
* In real world application, you don't want to iterate
* the whole array, instead you can use stack data structure
* to retrieve unused index in O(1).
*
*/
for (size_t i = 0; i < client_slot_num; i++) {
struct client_slot *client = &state->clients[i];
if (!client->is_used) {
/*
* We found unused slot.
*/
client->client_fd = client_fd;
memcpy(client->src_ip, src_ip_buf, sizeof(src_ip_buf));
client->src_port = src_port;
client->is_used = true;
client->my_index = i;
/*
* We map the client_fd to client array index that we accept
* here.
*/
state->client_map[client_fd] = client->my_index + EPOLL_MAP_SHIFT;
/*
* Let's tell to `epoll` to monitor this client file descriptor.
*/
my_epoll_add(state->epoll_fd, client_fd, EPOLLIN | EPOLLPRI);
printf("Client %s:%u has been accepted!\n", src_ip, src_port);
return 0;
}
}
printf("Sorry, can't accept more client at the moment, slot is full\n");
out_close:
close(client_fd);
return 0;
}
static void handle_client_event(int client_fd, uint32_t revents,
struct tcp_state *state)
{
int err;
ssize_t recv_ret;
char buffer[1024];
const uint32_t err_mask = EPOLLERR | EPOLLHUP;
/*
* Read the mapped value to get client index.
*/
uint32_t index = state->client_map[client_fd] - EPOLL_MAP_SHIFT;
struct client_slot *client = &state->clients[index];
if (revents & err_mask)
goto close_conn;
recv_ret = recv(client_fd, buffer, sizeof(buffer), 0);
if (recv_ret == 0)
goto close_conn;
if (recv_ret < 0) {
err = errno;
if (err == EAGAIN)
return;
/* Error */
printf("recv(): " PRERF, PREAR(err));
goto close_conn;
}
/*
* Safe printing
*/
buffer[recv_ret] = '\0';
if (buffer[recv_ret - 1] == '\n') {
buffer[recv_ret - 1] = '\0';
}
printf("Client %s:%u sends: \"%s\"\n", client->src_ip, client->src_port,
buffer);
return;
close_conn:
printf("Client %s:%u has closed its connection\n", client->src_ip,
client->src_port);
my_epoll_delete(state->epoll_fd, client_fd);
close(client_fd);
client->is_used = false;
return;
}
static int event_loop(struct tcp_state *state)
{
int err;
int ret = 0;
int timeout = 3000; /* in milliseconds */
int maxevents = 32;
int epoll_ret;
int epoll_fd = state->epoll_fd;
struct epoll_event events[32];
printf("Entering event loop...\n");
while (!state->stop) {
/*
* I sleep on `epoll_wait` and the kernel will wake me up
* when event comes to my monitored file descriptors, or
* when the timeout reached.
*/
epoll_ret = epoll_wait(epoll_fd, events, maxevents, timeout);
if (epoll_ret == 0) {
/*
*`epoll_wait` reached its timeout
*/
printf("I don't see any event within %d milliseconds\n", timeout);
continue;
}
if (epoll_ret == -1) {
err = errno;
if (err == EINTR) {
printf("Something interrupted me!\n");
continue;
}
/* Error */
ret = -1;
printf("epoll_wait(): " PRERF, PREAR(err));
break;
}
for (int i = 0; i < epoll_ret; i++) {
int fd = events[i].data.fd;
if (fd == state->tcp_fd) {
/*
* A new client is connecting to us...
*/
if (accept_new_client(fd, state) < 0) {
ret = -1;
goto out;
}
continue;
}
/*
* We have event(s) from client, let's call `recv()` to read it.
*/
handle_client_event(fd, events[i].events, state);
}
}
out:
return ret;
}
static int init_epoll(struct tcp_state *state)
{
int err;
int epoll_fd;
printf("Initializing epoll_fd...\n");
/* The epoll_create argument is ignored on modern Linux */
epoll_fd = epoll_create(255);
if (epoll_fd < 0) {
err = errno;
printf("epoll_create(): " PRERF, PREAR(err));
return -1;
}
state->epoll_fd = epoll_fd;
return 0;
}
static int init_socket(struct tcp_state *state)
{
int ret;
int err;
int tcp_fd = -1;
struct sockaddr_in addr;
socklen_t addr_len = sizeof(addr);
const char *bind_addr = "0.0.0.0";
uint16_t bind_port = 1234;
printf("Creating TCP socket...\n");
tcp_fd = socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP);
if (tcp_fd < 0) {
err = errno;
printf("socket(): " PRERF, PREAR(err));
return -1;
}
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(bind_port);
addr.sin_addr.s_addr = inet_addr(bind_addr);
ret = bind(tcp_fd, (struct sockaddr *)&addr, addr_len);
if (ret < 0) {
ret = -1;
err = errno;
printf("bind(): " PRERF, PREAR(err));
goto out;
}
ret = listen(tcp_fd, 10);
if (ret < 0) {
ret = -1;
err = errno;
printf("listen(): " PRERF, PREAR(err));
goto out;
}
/*
* Add `tcp_fd` to epoll monitoring.
*
* If epoll returned tcp_fd in `events` then a client is
* trying to connect to us.
*/
ret = my_epoll_add(state->epoll_fd, tcp_fd, EPOLLIN | EPOLLPRI);
if (ret < 0) {
ret = -1;
goto out;
}
printf("Listening on %s:%u...\n", bind_addr, bind_port);
state->tcp_fd = tcp_fd;
return 0;
out:
close(tcp_fd);
return ret;
}
static void init_state(struct tcp_state *state)
{
const size_t client_slot_num = sizeof(state->clients) / sizeof(*state->clients);
const uint16_t client_map_num = sizeof(state->client_map) / sizeof(*state->client_map);
for (size_t i = 0; i < client_slot_num; i++) {
state->clients[i].is_used = false;
state->clients[i].client_fd = -1;
}
for (uint16_t i = 0; i < client_map_num; i++) {
state->client_map[i] = EPOLL_MAP_TO_NOP;
}
}
int main(void)
{
int ret;
struct tcp_state state;
init_state(&state);
ret = init_epoll(&state);
if (ret != 0)
goto out;
ret = init_socket(&state);
if (ret != 0)
goto out;
state.stop = false;
ret = event_loop(&state);
out:
/*
* You should write a cleaner here.
*
* Close all client file descriptors and release
* some resources you may have.
*
* You may also want to set interrupt handler
* before the event_loop.
*
* For example, if you get SIGINT or SIGTERM
* set `state->stop` to true, so that it exits
* gracefully.
*/
return ret;
}
test.php (PHP script to simulate client)
I am too lazy to write a client TCP socket in C. So I use PHP.
<?php
function main(): int
{
$sock = socket_create(AF_INET, SOCK_STREAM, 0);
$conn = socket_connect($sock, "127.0.0.1", 1234);
if (!$conn)
return 1;
socket_write($sock, "AAAAAAAA\n", 9);
sleep(1);
socket_write($sock, "BBBBBBBB\n", 9);
sleep(1);
socket_write($sock, "CCCCCCCC\n", 9);
sleep(1);
socket_close($sock);
return 0;
}
exit(main());
Compile and run the server
ammarfaizi2#integral:~/ex$ gcc -Wall -Wextra -ggdb3 server.c -o server
ammarfaizi2#integral:~/ex$ ./server
Initializing epoll_fd...
Creating TCP socket...
Listening on 0.0.0.0:1234...
Entering event loop...
Simulate multiple clients with test.php
ammarfaizi2#integral:~$ for i in {1..10}; do php test.php & done;
[1] 14214
[2] 14215
[3] 14216
[4] 14217
[5] 14218
[6] 14219
[7] 14220
[8] 14221
[9] 14222
[10] 14223
ammarfaizi2#integral:~$
Server output when clients connect
ammarfaizi2#integral:~/ex$ ./server
Initializing epoll_fd...
Creating TCP socket...
Listening on 0.0.0.0:1234...
Entering event loop...
Client 127.0.0.1:60866 has been accepted!
Client 127.0.0.1:60866 sends: "AAAAAAAA"
Client 127.0.0.1:60868 has been accepted!
Client 127.0.0.1:60868 sends: "AAAAAAAA"
Client 127.0.0.1:60870 has been accepted!
Client 127.0.0.1:60872 has been accepted!
Client 127.0.0.1:60870 sends: "AAAAAAAA"
Client 127.0.0.1:60872 sends: "AAAAAAAA"
Client 127.0.0.1:60874 has been accepted!
Client 127.0.0.1:60874 sends: "AAAAAAAA"
Client 127.0.0.1:60876 has been accepted!
Client 127.0.0.1:60878 has been accepted!
Client 127.0.0.1:60878 sends: "AAAAAAAA"
Client 127.0.0.1:60876 sends: "AAAAAAAA"
Client 127.0.0.1:60880 has been accepted!
Client 127.0.0.1:60880 sends: "AAAAAAAA"
Client 127.0.0.1:60882 has been accepted!
Client 127.0.0.1:60882 sends: "AAAAAAAA"
Client 127.0.0.1:60884 has been accepted!
Client 127.0.0.1:60884 sends: "AAAAAAAA"
Client 127.0.0.1:60866 sends: "BBBBBBBB"
Client 127.0.0.1:60868 sends: "BBBBBBBB"
Client 127.0.0.1:60870 sends: "BBBBBBBB"
Client 127.0.0.1:60872 sends: "BBBBBBBB"
Client 127.0.0.1:60874 sends: "BBBBBBBB"
Client 127.0.0.1:60878 sends: "BBBBBBBB"
Client 127.0.0.1:60876 sends: "BBBBBBBB"
Client 127.0.0.1:60880 sends: "BBBBBBBB"
Client 127.0.0.1:60882 sends: "BBBBBBBB"
Client 127.0.0.1:60884 sends: "BBBBBBBB"
Client 127.0.0.1:60866 sends: "CCCCCCCC"
Client 127.0.0.1:60868 sends: "CCCCCCCC"
Client 127.0.0.1:60870 sends: "CCCCCCCC"
Client 127.0.0.1:60872 sends: "CCCCCCCC"
Client 127.0.0.1:60874 sends: "CCCCCCCC"
Client 127.0.0.1:60878 sends: "CCCCCCCC"
Client 127.0.0.1:60876 sends: "CCCCCCCC"
Client 127.0.0.1:60880 sends: "CCCCCCCC"
Client 127.0.0.1:60882 sends: "CCCCCCCC"
Client 127.0.0.1:60884 sends: "CCCCCCCC"
Client 127.0.0.1:60866 has closed its connection
Client 127.0.0.1:60868 has closed its connection
Client 127.0.0.1:60870 has closed its connection
Client 127.0.0.1:60872 has closed its connection
Client 127.0.0.1:60874 has closed its connection
Client 127.0.0.1:60878 has closed its connection
Client 127.0.0.1:60876 has closed its connection
Client 127.0.0.1:60880 has closed its connection
Client 127.0.0.1:60882 has closed its connection
Client 127.0.0.1:60884 has closed its connection
Assuming you have already open server socket you can use code below. It is based on man epoll accessible here.
// assume you have opened listen_sock properly
// int listen_sock;
//
int epollfd = epoll_create1(0);
if (epollfd == -1) {
exit(1);
}
struct epoll_event ev;
ev.events = EPOLLIN;
ev.data.fd = listen_sock;
if (epoll_ctl(epollfd, EPOLL_CTL_ADD, listen_sock, &ev) == -1) {
exit(2);
}
for (;;) {
#define MAX_EVENTS 64
struct epoll_event events[MAX_EVENTS];
int events_count = epoll_wait(epollfd, events, MAX_EVENTS, -1);
if (n == -1) {
exit(3);
}
for (int n = 0; n < events_count; ++ n) {
if (events[n].data.fd == listen_sock) {
struct sockaddr_un addr;
socklen_t addrlen;
int socket = accept(listen_sock, (struct sockaddr *) &addr, &addrlen);
}
}
}
In a project I am currently doing in group, we have to build a card game from scratch that uses sockets (Linux). We also have to build a chat room that every player can use.
So far so good. The chat is implemented using three separate threads, one that receives incoming connections (up to 50) and stores them in a client list, one that constantly waits for messages from all connected clients, and one that is created each time a client sends a message, sending that message to all clients in the client list. All of this works, except when a single client disconnects.
I managed to keep the server alive (with a sig handler for SIGPIPE) when a client disconnects, but now, when a client disconnects, I keep getting the error Bad file descriptor. But that's not the only problem, since the server keeps receiving empty messages and sends them to the remaining clients, effectively flooding the whole chat in a matter of milliseconds with empty messages.
I believe that if I can fix the problem on the server side, there won't be any problems on the client side.
So my question is: What is the right way (or any way) to manage a Bad file descriptor in my case. I've already tried closing the socket FD and setting the value to -1 in the client list, but that created even more problems and didn't fix the initial ones.
Here is the code, if necessary. The most important function (for the chat) are reception_thread, chat_thread, receive_string, send_string and connect_to_chat on the client side.
Here is the client:
//includes
const int PORT = 2477;
const int CHAT_PORT = 2478;
#define DEBUG
//error()
// Sets up the connection to the server.
//connect_to_server()
int connect_to_chat(char * hostname)
{
#ifdef DEBUG
printf("[DEBUG] Initiating connection to chat server.\n");
#endif
struct sockaddr_in serv_addr;
struct hostent *server;
// Get a socket.
int sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd < 0)
error("Error opening socket for server.");
// Get the address of the server.
server = gethostbyname(hostname);
if (server == NULL) {
fprintf(stderr, "ERROR, no such host\n");
exit(0);
}
// Zero out memory for server info.
memset(&serv_addr, 0, sizeof (serv_addr));
// Set up the server info.
serv_addr.sin_family = AF_INET;
memmove(server->h_addr, &serv_addr.sin_addr.s_addr, server->h_length);
serv_addr.sin_port = htons(CHAT_PORT);
// Make the connection.
if (connect(sockfd, (struct sockaddr *) &serv_addr, sizeof (serv_addr)) < 0)
error("Error connecting to chat server");
#ifdef DEBUG
printf("[DEBUG] Connected to server.\n");
#endif
return sockfd;
}
//-------------------------------- Messages ------------------------------------
// Bunch of send/recv functions that are not important to chat
int send_string(int sockfd, std::string myString)
{
#ifdef DEBUG
printf("[DEBUG] Sending string: %s.\n", myString.c_str());
#endif
//send size
uint32_t stringLen = myString.size();
uint32_t sendLen = htonl(stringLen);
int n = send(sockfd, &sendLen, sizeof (uint32_t), 0);
if (n < 0) {
error("Error sending message (string size). Removing client from list.");
return -1;
}
//send string
n = send(sockfd, myString.c_str(), stringLen, 0);
if (n < 0) {
error("Error sending message (string). Removing client from list.");
return -1;
}
return 0;
}
std::string receive_string(int sockfd)
{
//get string length
uint32_t stringLen;
int n = recv(sockfd, &stringLen, sizeof (uint32_t), 0);
if (n < 0) {
perror("Error receiving message(string size).");
}
stringLen = ntohl(stringLen);
std::vector<uint8_t> buffer;
buffer.resize(stringLen, 0x00);
//get string
n = recv(sockfd, &(buffer[0]), stringLen, 0);
if (n < 0) {
perror("Error receiving message(string).");
}
std::string returnString;
returnString.assign(reinterpret_cast<const char*> (&(buffer[0])), buffer.size()); //might be a bad idea, but it works
#ifdef DEBUG
printf("[DEBUG] Received message: %s\n", returnString.c_str());
#endif
return returnString;
}
//----------------------------- Printing functions------------------------------
void print_menu_guest()
{
// some visual function
}
void print_menu_user()
{
// some visual function
}
void print_info()
{
std::cout << " No information available on the game yet." << std::endl;
}
//---------------------------- Account functions -------------------------------
// Not necessary for chat functions
//--------------------------- Chat thread functions ----------------------------
void reception_thread(int sockfd)
{
#ifdef DEBUG
printf("[DEBUG] Reception thread started.\n");
#endif
std::string stringToPrint;
while (1) {
stringToPrint = receive_string(sockfd);
std::cout << stringToPrint << std::endl;
}
}
void chat_thread(int sockfd, char* host)
{
#ifdef DEBUG
printf("[DEBUG] Chat thread started.\n");
#endif
std::string myString, myUsername, blank;
std::cout << "Enter your username (NO SPACES): ";
std::cin >> myUsername;
myUsername += ": ";
int chat_sockfd = connect_to_chat(host);
std::thread reception_thr(reception_thread, chat_sockfd);
reception_thr.detach();
while (1) {
getline(std::cin, myString);
if (!myString.empty()) {
if (myString != "/quit") {
send_string(chat_sockfd, (myUsername + myString));
}
else {
printf("On peut pas encore quitter :( ");
}
}
}
}
//---------------------- Menu management functions -----------------------------
// Main menu function
//---------------------------- Main function -----------------------------------
int main(int argc, char** argv)
{
/* Make sure host and port are specified. */
if (true) {
char* hostname = "localhost";
/* Connect to the server. */
int sockfd = connect_to_server(hostname);
#ifdef DEBUG
printf("[DEBUG] Client ID: Not yet implemented. ");
#endif
login_prompt(sockfd);
user_menu_loop(sockfd);
}
return 0;
}
And here is the server: Its most important functions (for the chat) are setup_user_fetcher, message_receiver, send_string_to_all, receive_string, send_string, get_chat_user, setup_chat_listener.
// Bunch of includes
const int PORT = 2477;
const int CHAT_PORT = 2478;
const int BACKLOG = 10;
const int MAX_CLIENTS = 20;
int clients_list[50] = {-1};
#define DEBUG
void error(const char *msg)
{
perror(msg);
}
/* Catch Signal Handler functio */
void signal_callback_handler(int signum){
printf("Caught signal SIGPIPE %d\n",signum);
}
//-------------------------- Server set-up functions ---------------------------
// Not necessary for chat
//--------------------------- Chat server functions ---------------------------
int setup_chat_listener()
{
int sockfd;
struct sockaddr_in serv_addr;
// Get a socket to listen on
sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd < 0)
error("ERROR opening listener socket.");
// Zero out the memory for the server information
memset(&serv_addr, 0, sizeof (serv_addr));
// set up the server info
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = htons(CHAT_PORT);
// Bind the server info to the listener socket.
if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof (serv_addr)) < 0)
error("Error binding listener socket.");
#ifdef DEBUG
printf("[DEBUG] Chat listener set.\n");
#endif
// Return the socket number.
return sockfd;
}
int get_chat_user(int sockfd)
{
#ifdef DEBUG
printf("[DEBUG] Getting chat user.\n");
#endif
struct sockaddr_in their_addr;
socklen_t sin_size;
if (listen(sockfd, BACKLOG) < 0) {
perror("Error while listening.");
exit(EXIT_FAILURE);
}
sin_size = sizeof (struct sockaddr_in);
// Mise a zero de la memoire pour le client.
memset(&their_addr, 0, sin_size);
int new_fd = accept(sockfd, (struct sockaddr *) &their_addr, &sin_size);
if (new_fd < 0)
error("Error while accepting.");
printf("Chat server: Connection received from: %s\n",
inet_ntoa(their_addr.sin_addr));
return new_fd;
}
int send_string(int sockfd, std::string myString)
{
#ifdef DEBUG
printf("[DEBUG] Sending string to client %d.\n", sockfd);
#endif
uint32_t stringLen = myString.size();
uint32_t sendLen = htonl(stringLen);
int n = send(sockfd, &sendLen, sizeof (uint32_t), 0);
if (n < 0) {
error("Error sending message (string size). Removing client from list.");
return -1;
}
//send string
n = send(sockfd, myString.c_str(), stringLen, 0);
if (n < 0) {
error("Error sending message (string). Removing client from list.");
return -1;
}
return 0;
}
std::string receive_string(int sockfd)
{
#ifdef DEBUG
printf("[DEBUG] Receiving string.\n");
printf("Current chat user sockfd: %d\n", sockfd);
#endif
uint32_t stringLen;
int n = recv(sockfd, &stringLen, sizeof (uint32_t), 0);
#ifdef DEBUG
printf("[DEBUG] String size received: %d.\n", stringLen);
#endif
if (n < 0) {
perror("Error receiving message(string size).");
}
stringLen = ntohl(stringLen);
std::vector<uint8_t> buffer;
buffer.resize(stringLen, 0x00);
//get string
n = recv(sockfd, &(buffer[0]), stringLen, 0);
if (n < 0) {
perror("Error receiving message(string).");
close(sockfd);
}
std::string returnString;
returnString.assign(reinterpret_cast<const char*> (&(buffer[0])), buffer.size()); //might be a bad idea, but it works
#ifdef DEBUG
printf("[DEBUG] Received message: %s\n", returnString.c_str());
#endif
return returnString;
}
void send_string_to_all(std::string myString)
{
#ifdef DEBUG
printf("[DEBUG] Sending string to all clients.\n");
#endif
int n;
for (int i = 0; i < 50; ++i) {
if (clients_list[i] != -1) {
n = send_string(clients_list[i], myString);
if (n < 0) {
close(clients_list[i]);
clients_list[i] = -1;
}
}
}
}
void message_receiver(int sockfd)
{
#ifdef DEBUG
printf("[DEBUG] Setting up message receiver.\n");
printf("Current chat user sockfd: %d", sockfd);
#endif
std::string message;
int n;
while (1) {
message = receive_string(sockfd);
std::thread t1(send_string_to_all, message);
t1.detach();
}
}
//------------------------------------------------------------------------------
// Bunch of send/recv functions, not necessary to chat
//----------------------------Account Functions---------------------------------
// Not necessary to chat
//------------------------------------------------------------------------------
// Main menu function
void setup_user_fetcher(int lis_chat_sockfd)
{
#ifdef DEBUG
printf("[DEBUG] Gotta catch'em all.\n");
#endif
while (1) {
int chat_user_sockfd = get_chat_user(lis_chat_sockfd);
for (int i = 0; i < 50; ++i)
if (clients_list[i] == -1) {
clients_list[i] = chat_user_sockfd;
break;
}
std::thread message_receiver_thread(message_receiver, chat_user_sockfd);
message_receiver_thread.detach();
}
}
int main(int argc, char** argv)
{
signal(SIGPIPE, signal_callback_handler);
int lis_sockfd = setup_listener();
int lis_chat_sockfd = setup_chat_listener();
std::thread chat_thread(setup_user_fetcher, lis_chat_sockfd);
chat_thread.detach();
while (1) {
int user_sockfd = get_user(lis_sockfd);
int* user_sockfd_ptr = (int*) malloc(sizeof (int));
memset(user_sockfd_ptr, 0, sizeof (int));
user_sockfd_ptr[0] = user_sockfd;
#ifdef DEBUG
printf("[DEBUG] Starting main menu...\n");
#endif
pthread_t thread;
int result = pthread_create(&thread, NULL, main_menu,
(void *) user_sockfd_ptr);
if (result) {
printf("Thread creation failed with return code %d\n", result);
exit(-1);
}
#ifdef DEBUG
printf("[DEBUG] New main menu thread started.\n");
#endif
}
close(lis_sockfd);
pthread_exit(NULL);
return 0;
}
If you wish to reproduce the error, you could compile the code using the following lines
g++ client.cpp -o client -std=c++14 -pthread
g++ server.cpp -o server -std=c++14 -pthread
and run both without any arguments. The client is set to connect on "localhost".
I would be really glad if anyone could help me out with this.
I recomment getting rid of the SIGPIPE signal itself.
signal(SIGPIPE, SIG_IGN);
Now, write()s on killed sockets will simply return -1. It should be easier to deal with that, instead of an asynchronous signal.
If you need SIGPIPE for other reasons, replace write()s with sendto()s with the MSG_NOSIGNAL option. See the sendto(2) manual page for more information.
You have UB. &(buffer[0]) will fail if the number of bytes read is 0 (which I believe will happen if client disconnects). You should test for 0 and return early before building your string.
Also you do not return after finding errors so you build your string from bad data in case of errors.
Maybe something more like:
std::string receive_string(int sockfd)
{
uint32_t stringLen;
int n = recv(sockfd, &stringLen, sizeof (uint32_t), 0);
if (n < 0) {
close(sockfd);
// exit early
throw std::runtime_error("Error receiving message(string size): "
+ std::string(std::strerror(errno)));
}
// test for zero
if(!n)
return {}; // empty string
stringLen = ntohl(stringLen);
std::vector<uint8_t> buffer(stringLen);
// buffer.resize(stringLen, 0x00);
//get string
n = recv(sockfd, &(buffer[0]), stringLen, 0);
if (n < 0) {
close(sockfd);
// exit early
throw std::runtime_error("Error receiving message(string): "
+ std::string(std::strerror(errno)));
}
// only build string if no errors
return {buffer.begin(), buffer.begin() + n};
}
Here is an example of echo TCP server from libevent book.
How can I modify it to close a socket after each write. I know that I can close a socket connection by calling bufferevent_free(), but I don't understand how use it to close connection every time I send an echo to the socket.
#include <event2/listener.h>
#include <event2/bufferevent.h>
#include <event2/buffer.h>
#include <arpa/inet.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
static void
echo_read_cb(struct bufferevent *bev, void *ctx)
{
/* This callback is invoked when there is data to read on bev. */
struct evbuffer *input = bufferevent_get_input(bev);
struct evbuffer *output = bufferevent_get_output(bev);
/* Copy all the data from the input buffer to the output buffer. */
evbuffer_add_buffer(output, input);
}
static void
echo_event_cb(struct bufferevent *bev, short events, void *ctx)
{
if (events & BEV_EVENT_ERROR)
perror("Error from bufferevent");
if (events & (BEV_EVENT_EOF | BEV_EVENT_ERROR)) {
bufferevent_free(bev);
}
}
static void
accept_conn_cb(struct evconnlistener *listener,
evutil_socket_t fd, struct sockaddr *address, int socklen,
void *ctx)
{
/* We got a new connection! Set up a bufferevent for it. */
struct event_base *base = evconnlistener_get_base(listener);
struct bufferevent *bev = bufferevent_socket_new(
base, fd, BEV_OPT_CLOSE_ON_FREE);
bufferevent_setcb(bev, echo_read_cb, NULL, echo_event_cb, NULL);
bufferevent_enable(bev, EV_READ|EV_WRITE);
}
static void
accept_error_cb(struct evconnlistener *listener, void *ctx)
{
struct event_base *base = evconnlistener_get_base(listener);
int err = EVUTIL_SOCKET_ERROR();
fprintf(stderr, "Got an error %d (%s) on the listener. "
"Shutting down.\n", err, evutil_socket_error_to_string(err));
event_base_loopexit(base, NULL);
}
int
main(int argc, char **argv)
{
struct event_base *base;
struct evconnlistener *listener;
struct sockaddr_in sin;
int port = 9876;
if (argc > 1) {
port = atoi(argv[1]);
}
if (port<=0 || port>65535) {
puts("Invalid port");
return 1;
}
base = event_base_new();
if (!base) {
puts("Couldn't open event base");
return 1;
}
/* Clear the sockaddr before using it, in case there are extra
* platform-specific fields that can mess us up. */
memset(&sin, 0, sizeof(sin));
/* This is an INET address */
sin.sin_family = AF_INET;
/* Listen on 0.0.0.0 */
sin.sin_addr.s_addr = htonl(0);
/* Listen on the given port. */
sin.sin_port = htons(port);
listener = evconnlistener_new_bind(base, accept_conn_cb, NULL,
LEV_OPT_CLOSE_ON_FREE|LEV_OPT_REUSEABLE, -1,
(struct sockaddr*)&sin, sizeof(sin));
if (!listener) {
perror("Couldn't create listener");
return 1;
}
evconnlistener_set_error_cb(listener, accept_error_cb);
event_base_dispatch(base);
return 0;
}
From my understanding of documentation - you should put bufferevent_free(bev); at the end of echo_event_cb() function, this should close the connection after echoing user data back w/o waiting the client to close it. This should work this way till you use BEV_OPT_CLOSE_ON_FREE when creating bev buffer event.
I have to make an app using C sockets on Mac-OS that sends data from one socket to other socket, like this.
Server waits for connections
Client connect to server(from 1). -> socket1
Server connects to an external server and obtains an socket. -> socket2
From now on the server job is finish. The data exchange should be made only between the client socket (from 2) and socket obtained from 3.
Current implementation:
Server makes the connection and then reads data from one socket and sends to other.
Any ides how after step 3 to pipe line the two sockets socket1 and socket2.
Well your problem can be solved in two ways:
1) You need to code the part related to the connection formation between client and external server. But this puts an extra overload on the client, because it needs to make two connections, to both the servers (and I strongly feel the middle server in this case is useless).
2) Second way of solving it is passing the sockets between the servers:
Client connects to the server, this middle server sends this socket to the external server. Now external server starts communication with the client. This can be done only if both the server processes run on the same machine. And the file-descriptors are usually passed using Unix Domain Sockets.
Here is the code which I have. You can use these two functions to either send or receive the file-descriptors. It works on my Linux machine. I don't know about Mac-OS.
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/uio.h>
#include <unistd.h>
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
/* this function passes 'fd_to_send'
file descriptor via
a unix domain socket 'sfd'...
*/
void pass_fd(int sfd, int fd_to_send)
{
struct msghdr msg;
/*allocate memory to 'msg_control' field in msghdr struct */
char buf[CMSG_SPACE(sizeof(int))];
/*the memory to be allocated should include data + header..
this is calculated by the above macro...(it merely adds some
no. of bytes and returs that number..*/
struct cmsghdr *cmsg;
struct iovec ve;
/*must send/receive atleast one byte...
main purpose is to have some error
checking.. but this is completely
irrelevant in the current context..*/
char *st ="I";
/*jst let us allocate 1 byte for formality
and leave it that way...*/
ve.iov_base = st;
ve.iov_len =1;
/*attach this memory to our main msghdr struct...*/
msg.msg_iov = &ve;
msg.msg_iovlen = 1;
/*these are optional fields ..
leave these fields with zeros..
to prevent unnecessary SIGSEGVs..*/
msg.msg_name = NULL;
msg.msg_namelen = 0;
/*here starts the main part..*/
/*attach the 'buf' to msg_control..
and fill in the size field correspondingly..
*/
msg.msg_control = buf;
msg.msg_controllen = sizeof(buf);
/*actually msg_control field must
point to a struct of type 'cmsghdr'
we just allocated the memory, yet we need to
set all the corresponding fields..
It is done as follows:
*/
cmsg = CMSG_FIRSTHDR(&msg);
/* this macro returns the address in the buffer..
from where the first header starts..
*/
/*set all the fields appropriately..*/
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
cmsg->cmsg_len = CMSG_LEN(sizeof(fd_to_send));
/*in the above field we need to store
the size of header + data(in this case 4 bytes(int) for our fd..
this is returned by the 'CMSG_LEN' macro..*/
*(int*)CMSG_DATA(cmsg) = fd_to_send;
/*after the above three fields we keep the actual data..
the macro 'CMSG_DATA' returns pointer to this location
and we set it to the file descriptor to be sent..
*/
msg.msg_controllen = cmsg->cmsg_len;
/*now that we have filled the 'cmsg' struct
we store the size of this struct..*/
/*this one isn't required when you
pass a single fd..
but useful when u pass multiple fds.*/
msg.msg_flags = 0;
/*leave the flags field zeroed..*/
if(sendmsg( sfd, &msg, 0)==-1){ perror("snd:\n"); exit(1); }
/*send this over the UNIX deomain socoket..*/
printf("sent fd:%d\n", fd_to_send);
close(fd_to_send);
/*close the fd which was sent..*/
}
/*returns the received fd over the unix domain socket 'sfd'..*/
int recv_fd(int sfd)
{
struct msghdr msg;
/*do all the unwanted things first...
same as the send_fd function..*/
struct iovec io;
char ptr[1];
io.iov_base = ptr;
io.iov_len = 1;
msg.msg_name = 0;
msg.msg_namelen = 0;
msg.msg_iov = &io;
msg.msg_iovlen = 1;
/*-----------------------*/
char buf[CMSG_SPACE(sizeof(int))];
msg.msg_control = buf;
msg.msg_controllen = sizeof(buf);
/*reasoning is same..as above*/
/*now here comes the main part..*/
if(recvmsg( sfd, &msg, 0)==-1)
{
/*some shit has happened*/
perror("recv\n");
exit(1);
}
struct cmsghdr *cm;
cm = CMSG_FIRSTHDR(&msg);
/*get the first message header..*/
if(cm->cmsg_type != SCM_RIGHTS)
{
/*again some shit has happened..*/
perror("unknown type..\n");
exit(1);
}
/*if control has reached here.. this means
we have got the correct message..and when you
extract the fd out of this message
this need not be same as the one which was sent..
allocating a new fd is all done by the kernel
and our job is jst to use it..*/
printf("received fd:%d\n", *(int*)CMSG_DATA(cm));
return *(int*)CMSG_DATA(cm);
}
In the below example:
ClientOne and ClientTwo connect to the server.
When the server receives both ClientOne and ClientTwo's socket descriptor, it sends ClientOne's information to ClientTwo and vice-versa.
The information it sends is the IP and the client is coming from. Server shuts down.
When each client receives their info, a socket is created and they connect to eachother. The server socket is then shutdown.
Socket Class:
#include <winsock2.h>
#include <Ws2tcpip.h>
#include <windows.h>
#include <cstdint>
#include <string>
#include <stdexcept>
#include <iostream>
#include <thread>
#include <vector>
class Socket
{
private:
SOCKET socket;
std::uint32_t Port;
std::string Address;
bool Listen, Initialized, Asynchronous;
void Swap(Socket &S);
void UnInitialized();
public:
Socket();
Socket(std::uint32_t Port, std::string Address, bool Listen = false, bool Asynchronous = false);
Socket(const Socket &S) = delete;
Socket(Socket && S);
~Socket();
Socket& operator = (const Socket &S) = delete;
Socket& operator = (Socket && S);
int Recv(void* Buffer, std::uint32_t BufferLength);
int Recv(SOCKET S, void* Buffer, std::uint32_t BufferLength);
std::uint32_t RecvEx(void* Buffer, std::uint32_t BufferLength);
std::uint32_t RecvEx(SOCKET S, void* Buffer, std::uint32_t BufferLength);
int Send(void* Buffer, std::size_t BufferSize);
int Send(SOCKET S, void* Buffer, std::size_t BufferSize);
void Connect();
void Connect(std::uint32_t Port, std::string Address, bool Listen, bool Asynchronous);
SOCKET Accept(sockaddr* ClientInfo, int* ClientInfoSize);
void Close();
SOCKET GetSocket() const;
};
Socket::~Socket()
{
Close();
}
void Socket::Close()
{
if (socket)
{
shutdown(socket, SD_BOTH);
closesocket(socket);
socket = 0;
}
if (Initialized)
{
WSACleanup();
}
}
SOCKET Socket::GetSocket() const
{
return this->socket;
}
Socket::Socket(Socket && S) : socket(std::move(S.socket)), Port(std::move(S.Port)), Address(std::move(S.Address)), Listen(std::move(S.Listen)), Initialized(std::move(S.Initialized)), Asynchronous(std::move(S.Asynchronous)) {}
Socket::Socket() : socket(0), Port(0), Address(std::string()), Listen(false), Initialized(false), Asynchronous(false) {}
Socket::Socket(std::uint32_t Port, std::string Address, bool Listen, bool Asynchronous) : socket(0), Port(Port), Address(Address), Listen(Listen), Initialized(true), Asynchronous(Asynchronous)
{
Connect(Port, Address, Listen, Asynchronous);
}
void Socket::Connect()
{
UnInitialized();
Connect(Port, Address, Listen, Asynchronous);
}
void Socket::Connect(std::uint32_t Port, std::string Address, bool Listen, bool Asynchronous)
{
if (!socket)
{
this->Port = Port;
this->Address = Address;
this->Asynchronous = Asynchronous;
this->Initialized = true;
WSADATA wsaData;
struct sockaddr_in* sockaddr_ipv4;
if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0)
{
throw std::runtime_error("Error: WSAStartup Failed");
}
if (Address != "INADDR_ANY")
{
if (Address.find("http://") != std::string::npos)
{
Address = Address.substr(7);
}
std::size_t Position = Address.find("/");
if (Position != std::string::npos)
{
Address = Address.substr(0, Position);
}
struct addrinfo *it = nullptr, *result = nullptr;
getaddrinfo(Address.c_str(), nullptr, nullptr, &result);
for (it = result; it != nullptr; it = it->ai_next)
{
sockaddr_ipv4 = reinterpret_cast<sockaddr_in*>(it->ai_addr);
Address = inet_ntoa(sockaddr_ipv4->sin_addr);
if (Address != "0.0.0.0") break;
}
freeaddrinfo(result);
}
if ((this->socket = ::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) == INVALID_SOCKET)
{
this->Close();
throw std::runtime_error("Error: Failed to create socket");
}
struct sockaddr_in SockAddr;
memset(&SockAddr, 0, sizeof(SockAddr));
SockAddr.sin_port = htons(Port);
SockAddr.sin_family = AF_INET;
SockAddr.sin_addr.s_addr = (Address == "INADDR_ANY" ? htonl(INADDR_ANY) : inet_addr(Address.c_str()));
if (Listen && (bind(this->socket, reinterpret_cast<SOCKADDR*>(&SockAddr), sizeof(SockAddr)) == SOCKET_ERROR))
{
this->Close();
throw std::runtime_error("Error: Socket binding failed");
}
if (Listen && (listen(this->socket, SOMAXCONN) == SOCKET_ERROR))
{
this->Close();
throw std::runtime_error("Error: Socket Listening Failed");
}
if(!Listen && (connect(this->socket, reinterpret_cast<SOCKADDR*>(&SockAddr), sizeof(SockAddr)) == SOCKET_ERROR))
{
if(Asynchronous && WSAGetLastError() != WSAEWOULDBLOCK)
{
this->Close();
throw std::runtime_error("Error: Socket Connection failed");
}
else if (!Asynchronous)
{
this->Close();
throw std::runtime_error("Error: Socket Connection failed");
}
}
}
}
SOCKET Socket::Accept(sockaddr* ClientInfo, int* ClientInfoSize)
{
static int Size = sizeof(sockaddr);
return accept(this->socket, ClientInfo, (ClientInfo && ClientInfoSize ? ClientInfoSize : &Size));
}
Socket& Socket::operator = (Socket && S)
{
S.Swap(*this);
return *this;
}
int Socket::Recv(void* Buffer, std::uint32_t BufferLength)
{
return recv(this->socket, reinterpret_cast<char*>(Buffer), BufferLength, 0);
}
int Socket::Recv(SOCKET S, void* Buffer, std::uint32_t BufferLength)
{
return recv(S, reinterpret_cast<char*>(Buffer), BufferLength, 0);
}
std::uint32_t Socket::RecvEx(void* Buffer, std::uint32_t BufferLength)
{
return this->RecvEx(this->socket, Buffer, BufferLength);
}
std::uint32_t Socket::RecvEx(SOCKET S, void* Buffer, std::uint32_t BufferLength)
{
UnInitialized();
char* Pointer = reinterpret_cast<char*>(Buffer);
std::uint32_t TotalRead = 0;
while (BufferLength > 0)
{
int BytesRead = recv(S, Pointer, std::min(1024 * 1024, static_cast<int>(BufferLength)), 0);
if (BytesRead < 0)
{
if ((BytesRead == SOCKET_ERROR) && (WSAGetLastError() == WSAEWOULDBLOCK))
continue;
throw std::runtime_error("Error! RecvEx: Failed To Read Bytes.");
}
if (BytesRead == 0) break;
Pointer += BytesRead;
BufferLength -= BytesRead;
TotalRead += BytesRead;
}
return TotalRead;
}
int Socket::Send(void* Buffer, std::size_t BufferSize)
{
return send(this->socket, reinterpret_cast<char*>(Buffer), BufferSize, 0);
}
int Socket::Send(SOCKET S, void* Buffer, std::size_t BufferSize)
{
return send(S, reinterpret_cast<char*>(Buffer), BufferSize, 0);
}
void Socket::Swap(Socket &S)
{
using std::swap;
swap(socket, S.socket);
swap(Port, S.Port);
swap(Address, S.Address);
swap(Listen, S.Listen);
swap(Initialized, S.Initialized);
swap(Asynchronous, S.Asynchronous);
}
void Socket::UnInitialized()
{
if (!Initialized)
{
throw std::runtime_error("\nError! Socket Not Constructed!");
std::cout << "Socket Not Constructed!\n";
ExitProcess(0);
}
}
Server.cpp:
#include "Sockets.hpp"
#define PORT 27015
#define ADDRESS INADDR_ANY
#define CLIENTCOUNT 2
typedef struct
{
std::string ip;
int port;
SOCKET sock;
} ClientInfo;
template <typename T>
inline T ReadPointer(TCHAR* &Pointer)
{
T Result = *(reinterpret_cast<T*>(Pointer));
Pointer += sizeof(T) / sizeof(TCHAR);
return Result;
}
template <typename T>
inline void WritePointer(TCHAR* &Pointer, const T& Value)
{
*(reinterpret_cast<T*>(Pointer)) = Value;
Pointer += sizeof(T) / sizeof(TCHAR);
}
bool SendClient(ClientInfo* client, ClientInfo* receiver)
{
int datasize = sizeof(client->ip.size()) + client->ip.size() + sizeof(client->port);
std::vector<char> buffer(datasize, 0);
char* ptr = &buffer[0];
WritePointer(ptr, client->ip.size());
for (std::size_t i = 0; i < client->ip.size(); ++i)
WritePointer(ptr, client->ip[i]);
WritePointer(ptr, client->port);
std::cout << "Sending: " << &buffer[0] << "\n";
return send(receiver->sock, &buffer[0], datasize, 0) == datasize;
}
bool ReadClient(SOCKET sock, ClientInfo* client)
{
std::size_t ip_size = 0;
recv(sock, (char*) &ip_size, sizeof(client->ip.size()), 0);
client->ip.resize(ip_size);
recv(sock, &client->ip[0], ip_size, 0);
recv(sock, (char*) &client->port, sizeof(int), 0);
std::cout<<client->ip<<"\n";
return true;
}
int main()
{
Socket s;
s.Connect(PORT, "localhost", true, false);
char buffer[1024] = {0};
std::vector<ClientInfo> clients;
while(true)
{
if (clients.size() < CLIENTCOUNT)
{
sockaddr_in ClientAddressInfo = {0};
SOCKET sock = s.Accept(reinterpret_cast<sockaddr*>(&ClientAddressInfo), nullptr);
char* ip = inet_ntoa(ClientAddressInfo.sin_addr);
int port = (int) ntohs(ClientAddressInfo.sin_port);
ClientInfo info = {ip, port, sock};
clients.push_back(info);
std::cout << "Client Connected From: " << ip << " on port: " << port << "\n";
}
if (ReadAsync(s, buffer))
{
std::cout << "Connected\n";
}
std::this_thread::sleep_for(std::chrono::milliseconds(100));
if (clients.size() >= CLIENTCOUNT)
{
SendClient(&clients[0], &clients[1]);
SendClient(&clients[1], &clients[0]);
return 0;
}
}
}
Client.cpp:
#define PORT 27015
#define ADDRESS INADDR_ANY
#define CLIENTCOUNT 2
typedef struct
{
std::string ip;
int port;
SOCKET sock;
} ClientInfo;
template <typename T>
inline T ReadPointer(TCHAR* &Pointer)
{
T Result = *(reinterpret_cast<T*>(Pointer));
Pointer += sizeof(T) / sizeof(TCHAR);
return Result;
}
template <typename T>
inline void WritePointer(TCHAR* &Pointer, const T& Value)
{
*(reinterpret_cast<T*>(Pointer)) = Value;
Pointer += sizeof(T) / sizeof(TCHAR);
}
bool SendClient(ClientInfo* client, ClientInfo* receiver)
{
int datasize = sizeof(client->ip.size()) + client->ip.size() + sizeof(client->port);
std::vector<char> buffer(datasize, 0);
char* ptr = &buffer[0];
WritePointer(ptr, client->ip.size());
for (std::size_t i = 0; i < client->ip.size(); ++i)
WritePointer(ptr, client->ip[i]);
WritePointer(ptr, client->port);
std::cout << "Sending: " << &buffer[0] << "\n";
return send(receiver->sock, &buffer[0], datasize, 0) == datasize;
}
bool ReadClient(SOCKET sock, ClientInfo* client)
{
std::size_t ip_size = 0;
recv(sock, (char*) &ip_size, sizeof(client->ip.size()), 0);
client->ip.resize(ip_size);
recv(sock, &client->ip[0], ip_size, 0);
recv(sock, (char*) &client->port, sizeof(int), 0);
return true;
}
bool ReadAsync(const Socket &sock, ClientInfo* client)
{
struct timeval tv;
tv.tv_sec = 0;
tv.tv_usec = 100000;
fd_set rfds;
FD_ZERO(&rfds);
FD_SET(sock.GetSocket(), &rfds);
for (int i = 0; i < 600; ++i)
{
if (select(sock.GetSocket(), &rfds, &rfds, NULL, &tv))
{
return ReadClient(sock.GetSocket(), client);
}
tv.tv_sec = 0;
tv.tv_usec = 100000;
}
return false;
}
int main()
{
Socket s;
s.Connect(PORT, "localhost", false, false);
std::vector<SOCKET> clients;
ClientInfo client = {};
while(true)
{
if (ReadAsync(s, &client))
{
std::cout<<"IP: "<<client.ip<<" PORT: "<<client.port<<"\n";
s = std::move(Socket(client.port, client.ip, true, false));
}
std::this_thread::sleep_for(std::chrono::milliseconds(100));
system("CLS");
std::cout<<"Connecting..\n";
}
}