In a project I am currently doing in group, we have to build a card game from scratch that uses sockets (Linux). We also have to build a chat room that every player can use.
So far so good. The chat is implemented using three separate threads, one that receives incoming connections (up to 50) and stores them in a client list, one that constantly waits for messages from all connected clients, and one that is created each time a client sends a message, sending that message to all clients in the client list. All of this works, except when a single client disconnects.
I managed to keep the server alive (with a sig handler for SIGPIPE) when a client disconnects, but now, when a client disconnects, I keep getting the error Bad file descriptor. But that's not the only problem, since the server keeps receiving empty messages and sends them to the remaining clients, effectively flooding the whole chat in a matter of milliseconds with empty messages.
I believe that if I can fix the problem on the server side, there won't be any problems on the client side.
So my question is: What is the right way (or any way) to manage a Bad file descriptor in my case. I've already tried closing the socket FD and setting the value to -1 in the client list, but that created even more problems and didn't fix the initial ones.
Here is the code, if necessary. The most important function (for the chat) are reception_thread, chat_thread, receive_string, send_string and connect_to_chat on the client side.
Here is the client:
//includes
const int PORT = 2477;
const int CHAT_PORT = 2478;
#define DEBUG
//error()
// Sets up the connection to the server.
//connect_to_server()
int connect_to_chat(char * hostname)
{
#ifdef DEBUG
printf("[DEBUG] Initiating connection to chat server.\n");
#endif
struct sockaddr_in serv_addr;
struct hostent *server;
// Get a socket.
int sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd < 0)
error("Error opening socket for server.");
// Get the address of the server.
server = gethostbyname(hostname);
if (server == NULL) {
fprintf(stderr, "ERROR, no such host\n");
exit(0);
}
// Zero out memory for server info.
memset(&serv_addr, 0, sizeof (serv_addr));
// Set up the server info.
serv_addr.sin_family = AF_INET;
memmove(server->h_addr, &serv_addr.sin_addr.s_addr, server->h_length);
serv_addr.sin_port = htons(CHAT_PORT);
// Make the connection.
if (connect(sockfd, (struct sockaddr *) &serv_addr, sizeof (serv_addr)) < 0)
error("Error connecting to chat server");
#ifdef DEBUG
printf("[DEBUG] Connected to server.\n");
#endif
return sockfd;
}
//-------------------------------- Messages ------------------------------------
// Bunch of send/recv functions that are not important to chat
int send_string(int sockfd, std::string myString)
{
#ifdef DEBUG
printf("[DEBUG] Sending string: %s.\n", myString.c_str());
#endif
//send size
uint32_t stringLen = myString.size();
uint32_t sendLen = htonl(stringLen);
int n = send(sockfd, &sendLen, sizeof (uint32_t), 0);
if (n < 0) {
error("Error sending message (string size). Removing client from list.");
return -1;
}
//send string
n = send(sockfd, myString.c_str(), stringLen, 0);
if (n < 0) {
error("Error sending message (string). Removing client from list.");
return -1;
}
return 0;
}
std::string receive_string(int sockfd)
{
//get string length
uint32_t stringLen;
int n = recv(sockfd, &stringLen, sizeof (uint32_t), 0);
if (n < 0) {
perror("Error receiving message(string size).");
}
stringLen = ntohl(stringLen);
std::vector<uint8_t> buffer;
buffer.resize(stringLen, 0x00);
//get string
n = recv(sockfd, &(buffer[0]), stringLen, 0);
if (n < 0) {
perror("Error receiving message(string).");
}
std::string returnString;
returnString.assign(reinterpret_cast<const char*> (&(buffer[0])), buffer.size()); //might be a bad idea, but it works
#ifdef DEBUG
printf("[DEBUG] Received message: %s\n", returnString.c_str());
#endif
return returnString;
}
//----------------------------- Printing functions------------------------------
void print_menu_guest()
{
// some visual function
}
void print_menu_user()
{
// some visual function
}
void print_info()
{
std::cout << " No information available on the game yet." << std::endl;
}
//---------------------------- Account functions -------------------------------
// Not necessary for chat functions
//--------------------------- Chat thread functions ----------------------------
void reception_thread(int sockfd)
{
#ifdef DEBUG
printf("[DEBUG] Reception thread started.\n");
#endif
std::string stringToPrint;
while (1) {
stringToPrint = receive_string(sockfd);
std::cout << stringToPrint << std::endl;
}
}
void chat_thread(int sockfd, char* host)
{
#ifdef DEBUG
printf("[DEBUG] Chat thread started.\n");
#endif
std::string myString, myUsername, blank;
std::cout << "Enter your username (NO SPACES): ";
std::cin >> myUsername;
myUsername += ": ";
int chat_sockfd = connect_to_chat(host);
std::thread reception_thr(reception_thread, chat_sockfd);
reception_thr.detach();
while (1) {
getline(std::cin, myString);
if (!myString.empty()) {
if (myString != "/quit") {
send_string(chat_sockfd, (myUsername + myString));
}
else {
printf("On peut pas encore quitter :( ");
}
}
}
}
//---------------------- Menu management functions -----------------------------
// Main menu function
//---------------------------- Main function -----------------------------------
int main(int argc, char** argv)
{
/* Make sure host and port are specified. */
if (true) {
char* hostname = "localhost";
/* Connect to the server. */
int sockfd = connect_to_server(hostname);
#ifdef DEBUG
printf("[DEBUG] Client ID: Not yet implemented. ");
#endif
login_prompt(sockfd);
user_menu_loop(sockfd);
}
return 0;
}
And here is the server: Its most important functions (for the chat) are setup_user_fetcher, message_receiver, send_string_to_all, receive_string, send_string, get_chat_user, setup_chat_listener.
// Bunch of includes
const int PORT = 2477;
const int CHAT_PORT = 2478;
const int BACKLOG = 10;
const int MAX_CLIENTS = 20;
int clients_list[50] = {-1};
#define DEBUG
void error(const char *msg)
{
perror(msg);
}
/* Catch Signal Handler functio */
void signal_callback_handler(int signum){
printf("Caught signal SIGPIPE %d\n",signum);
}
//-------------------------- Server set-up functions ---------------------------
// Not necessary for chat
//--------------------------- Chat server functions ---------------------------
int setup_chat_listener()
{
int sockfd;
struct sockaddr_in serv_addr;
// Get a socket to listen on
sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd < 0)
error("ERROR opening listener socket.");
// Zero out the memory for the server information
memset(&serv_addr, 0, sizeof (serv_addr));
// set up the server info
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = htons(CHAT_PORT);
// Bind the server info to the listener socket.
if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof (serv_addr)) < 0)
error("Error binding listener socket.");
#ifdef DEBUG
printf("[DEBUG] Chat listener set.\n");
#endif
// Return the socket number.
return sockfd;
}
int get_chat_user(int sockfd)
{
#ifdef DEBUG
printf("[DEBUG] Getting chat user.\n");
#endif
struct sockaddr_in their_addr;
socklen_t sin_size;
if (listen(sockfd, BACKLOG) < 0) {
perror("Error while listening.");
exit(EXIT_FAILURE);
}
sin_size = sizeof (struct sockaddr_in);
// Mise a zero de la memoire pour le client.
memset(&their_addr, 0, sin_size);
int new_fd = accept(sockfd, (struct sockaddr *) &their_addr, &sin_size);
if (new_fd < 0)
error("Error while accepting.");
printf("Chat server: Connection received from: %s\n",
inet_ntoa(their_addr.sin_addr));
return new_fd;
}
int send_string(int sockfd, std::string myString)
{
#ifdef DEBUG
printf("[DEBUG] Sending string to client %d.\n", sockfd);
#endif
uint32_t stringLen = myString.size();
uint32_t sendLen = htonl(stringLen);
int n = send(sockfd, &sendLen, sizeof (uint32_t), 0);
if (n < 0) {
error("Error sending message (string size). Removing client from list.");
return -1;
}
//send string
n = send(sockfd, myString.c_str(), stringLen, 0);
if (n < 0) {
error("Error sending message (string). Removing client from list.");
return -1;
}
return 0;
}
std::string receive_string(int sockfd)
{
#ifdef DEBUG
printf("[DEBUG] Receiving string.\n");
printf("Current chat user sockfd: %d\n", sockfd);
#endif
uint32_t stringLen;
int n = recv(sockfd, &stringLen, sizeof (uint32_t), 0);
#ifdef DEBUG
printf("[DEBUG] String size received: %d.\n", stringLen);
#endif
if (n < 0) {
perror("Error receiving message(string size).");
}
stringLen = ntohl(stringLen);
std::vector<uint8_t> buffer;
buffer.resize(stringLen, 0x00);
//get string
n = recv(sockfd, &(buffer[0]), stringLen, 0);
if (n < 0) {
perror("Error receiving message(string).");
close(sockfd);
}
std::string returnString;
returnString.assign(reinterpret_cast<const char*> (&(buffer[0])), buffer.size()); //might be a bad idea, but it works
#ifdef DEBUG
printf("[DEBUG] Received message: %s\n", returnString.c_str());
#endif
return returnString;
}
void send_string_to_all(std::string myString)
{
#ifdef DEBUG
printf("[DEBUG] Sending string to all clients.\n");
#endif
int n;
for (int i = 0; i < 50; ++i) {
if (clients_list[i] != -1) {
n = send_string(clients_list[i], myString);
if (n < 0) {
close(clients_list[i]);
clients_list[i] = -1;
}
}
}
}
void message_receiver(int sockfd)
{
#ifdef DEBUG
printf("[DEBUG] Setting up message receiver.\n");
printf("Current chat user sockfd: %d", sockfd);
#endif
std::string message;
int n;
while (1) {
message = receive_string(sockfd);
std::thread t1(send_string_to_all, message);
t1.detach();
}
}
//------------------------------------------------------------------------------
// Bunch of send/recv functions, not necessary to chat
//----------------------------Account Functions---------------------------------
// Not necessary to chat
//------------------------------------------------------------------------------
// Main menu function
void setup_user_fetcher(int lis_chat_sockfd)
{
#ifdef DEBUG
printf("[DEBUG] Gotta catch'em all.\n");
#endif
while (1) {
int chat_user_sockfd = get_chat_user(lis_chat_sockfd);
for (int i = 0; i < 50; ++i)
if (clients_list[i] == -1) {
clients_list[i] = chat_user_sockfd;
break;
}
std::thread message_receiver_thread(message_receiver, chat_user_sockfd);
message_receiver_thread.detach();
}
}
int main(int argc, char** argv)
{
signal(SIGPIPE, signal_callback_handler);
int lis_sockfd = setup_listener();
int lis_chat_sockfd = setup_chat_listener();
std::thread chat_thread(setup_user_fetcher, lis_chat_sockfd);
chat_thread.detach();
while (1) {
int user_sockfd = get_user(lis_sockfd);
int* user_sockfd_ptr = (int*) malloc(sizeof (int));
memset(user_sockfd_ptr, 0, sizeof (int));
user_sockfd_ptr[0] = user_sockfd;
#ifdef DEBUG
printf("[DEBUG] Starting main menu...\n");
#endif
pthread_t thread;
int result = pthread_create(&thread, NULL, main_menu,
(void *) user_sockfd_ptr);
if (result) {
printf("Thread creation failed with return code %d\n", result);
exit(-1);
}
#ifdef DEBUG
printf("[DEBUG] New main menu thread started.\n");
#endif
}
close(lis_sockfd);
pthread_exit(NULL);
return 0;
}
If you wish to reproduce the error, you could compile the code using the following lines
g++ client.cpp -o client -std=c++14 -pthread
g++ server.cpp -o server -std=c++14 -pthread
and run both without any arguments. The client is set to connect on "localhost".
I would be really glad if anyone could help me out with this.
I recomment getting rid of the SIGPIPE signal itself.
signal(SIGPIPE, SIG_IGN);
Now, write()s on killed sockets will simply return -1. It should be easier to deal with that, instead of an asynchronous signal.
If you need SIGPIPE for other reasons, replace write()s with sendto()s with the MSG_NOSIGNAL option. See the sendto(2) manual page for more information.
You have UB. &(buffer[0]) will fail if the number of bytes read is 0 (which I believe will happen if client disconnects). You should test for 0 and return early before building your string.
Also you do not return after finding errors so you build your string from bad data in case of errors.
Maybe something more like:
std::string receive_string(int sockfd)
{
uint32_t stringLen;
int n = recv(sockfd, &stringLen, sizeof (uint32_t), 0);
if (n < 0) {
close(sockfd);
// exit early
throw std::runtime_error("Error receiving message(string size): "
+ std::string(std::strerror(errno)));
}
// test for zero
if(!n)
return {}; // empty string
stringLen = ntohl(stringLen);
std::vector<uint8_t> buffer(stringLen);
// buffer.resize(stringLen, 0x00);
//get string
n = recv(sockfd, &(buffer[0]), stringLen, 0);
if (n < 0) {
close(sockfd);
// exit early
throw std::runtime_error("Error receiving message(string): "
+ std::string(std::strerror(errno)));
}
// only build string if no errors
return {buffer.begin(), buffer.begin() + n};
}
Related
#include <stdio.h>
#include <time.h>
#include <WinSock2.h>
#include <WS2tcpip.h>
#pragma comment(lib, "WS2_32.lib")
#define IP_ADDRESS "127.0.0.1"
#define PORT 20000
#define BUF_SIZE 64
#undef FD_SETSIZE
#define FD_SETSIZE 10000
void shuffle_buffer(char* buf, size_t size);
SOCKET create_socket();
void send_data(SOCKET sock);
int main()
{
WSADATA ws;
if (WSAStartup(MAKEWORD(2, 2), &ws) != 0)
{
printf("Init Windows Socket Failed::%d\n", GetLastError());
return -1;
}
const int CLIENT_SIZE = 1;
SOCKET socks[CLIENT_SIZE];
struct timeval tv = { 0, 10 };
fd_set fd_read, fd_write;
FD_ZERO(&fd_read);
FD_ZERO(&fd_write);
for (int i = 0; i < CLIENT_SIZE; i++) {
SOCKET sock = create_socket();
socks[i] = sock;
FD_SET(sock, &fd_write);
FD_SET(sock, &fd_read);
}
Sleep(1000);
int number_to_recv = CLIENT_SIZE;
while (number_to_recv > 0) {
int ret = select(CLIENT_SIZE, &fd_read, &fd_write, NULL, &tv);
for (int i = 0; i < CLIENT_SIZE; i++) {
if (FD_ISSET(socks[i], &fd_read)) {
char buf[BUF_SIZE];
int n = recv(socks[i], buf, BUF_SIZE, 0);
buf[n] = 0;
printf("%s\n", buf);
number_to_recv--;
}
if (FD_ISSET(socks[i], &fd_write)) {
send_data(socks[i]);
FD_CLR(socks[i], &fd_write);
//Sleep(1);
}
}
//printf("ret and number : %d, %d\n", ret, number_to_recv);
}
for (int i = 0; i < CLIENT_SIZE; i++) {
closesocket(socks[i]);
}
WSACleanup();
}
SOCKET create_socket()
{
SOCKET cli_sock;
struct sockaddr_in addr;
if ((cli_sock = socket(AF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET)
{
printf("Create Socket Failed::%d\n", GetLastError());
return -1;
}
//inet_pton
memset(addr.sin_zero, 0x00, 8);
addr.sin_family = AF_INET;
inet_pton(AF_INET, IP_ADDRESS, (void*)(&addr.sin_addr.s_addr));
addr.sin_port = htons(PORT);
if (connect(cli_sock, (struct sockaddr*)&addr, sizeof(addr)) == SOCKET_ERROR)
{
printf("Connect Error::%d\n", GetLastError());
return -1;
}
return cli_sock;
}
void send_data(SOCKET sock)
{
const int SEND_SIZE = BUF_SIZE / 2;
char buf[SEND_SIZE] = { 0 };
memset(buf, 'a', SEND_SIZE);
shuffle_buffer(buf, SEND_SIZE);
if (send(sock, buf, SEND_SIZE, 0) == SOCKET_ERROR)
{
printf("Send Info Error::%d\n", GetLastError());
}
}
void shuffle_buffer(char* buf, size_t size)
{
for (int i = 0; i < size; i++) {
buf[i] += int(rand() % 26);
}
}
Code above is a socket client using select model run on Win10, the problem is after I send data, but I can not receive data(I am sure that server has sent back data), this code below doesn`t run, so what is the problem? Thanks
The first parameter in select is maxfdp, and I know the difference between Win and Unix, so on Windows, this parameter seems not necessary, and I can write data,
but can not receive it.
if (FD_ISSET(socks[i], &fd_read)) {
char buf[BUF_SIZE];
int n = recv(socks[i], buf, BUF_SIZE, 0);
buf[n] = 0;
printf("%s\n", buf);
number_to_recv--;
}
select removes the sockets from the fd_set if they are not readable/writable. You need to add them back in before the next time you call select.
The reason your code can write data is because sockets start out being writable, so they will still be set in fd_write and your code will write data. They don't start out being readable, if no data has been received yet, so they'll be removed from the fd_read set and then your code stops checking whether they are readable.
I have two pairs of simple TCP server/client, i.e., one client per server, running on Windows:
Servers run in a process (app).
Clients run in the other process.
Servers keep sending heartbeats (a string) to their paired client.
The first pair of server/client run their mainloops in their own threads.
Once the first server/client have shaken hands with the first heartbeat, the second pair of server/client start their mainloops in their own threads.
For this test, they run on the same machine with different ports: 2345 and 2346.
Now my problem
The first client receives its server's heartbeat.
The second client does NOT, although the second server sent out heartbeats without errors.
Here is the server code:
// hello_dualchannel_server.cpp : This file contains the 'main' function.
#include "pch.h"
#include <iostream>
#include <thread>
#include <winsock2.h>
#include <ws2tcpip.h>
#include <spdlog/spdlog.h>
#define SPDLOG_WCHAR_TO_UTF8_SUPPORT
#ifdef _DEBUG
#if !defined(SPDLOG_ACTIVE_LEVEL)
#define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_TRACE
#endif // #if !defined(SPDLOG_ACTIVE_LEVEL)
#define SPDLOG_DEBUG_ON
#define SPDLOG_TRACE_ON
#define _trace SPDLOG_TRACE
#endif // #ifdef _DEBUG
using namespace spdlog;
SOCKET g_sockFirst = 0;
SOCKET g_sockClientFirst = 0;
std::thread g_threadFirst;
uint32_t g_timeLatestHeartBeatFirst = 0;
SOCKET g_sockSecond = 0;
SOCKET g_sockClientSecond = 0;
std::thread g_threadSecond;
uint32_t g_timeLatestHeartBeatSecond = 0;
void SetupLogger() {
#ifdef _DEBUG
spdlog::set_level(spdlog::level::trace);
spdlog::set_pattern("[%H:%M:%S%z][%^%L%$][%t:%s:%#] %v");
#else
spdlog::set_level(spdlog::level::info);
spdlog::set_pattern("[%H:%M:%S][%^%L%$][%t] %v");
#endif // #ifdef _DEBUG
}
int InitWinSock() {
WORD wVersionRequested;
WSADATA wsaData;
/* Use the MAKEWORD(lowbyte, highbyte) macro declared in Windef.h */
wVersionRequested = MAKEWORD(2, 2);
int err = WSAStartup(wVersionRequested, &wsaData);
if (err != 0) {
/* Tell the user that we could not find a usable */
/* Winsock DLL. */
printf("WSAStartup failed with error: %d\n", err);
return 1;
}
/* Confirm that the WinSock DLL supports 2.2.*/
/* Note that if the DLL supports versions greater */
/* than 2.2 in addition to 2.2, it will still return */
/* 2.2 in wVersion since that is the version we */
/* requested. */
if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2) {
/* Tell the user that we could not find a usable */
/* WinSock DLL. */
printf("Could not find a usable version of Winsock.dll\n");
WSACleanup();
return 1;
}
else
printf("The Winsock 2.2 dll was found okay\n");
return 0;
}
bool Init(int host_port, SOCKET* p_sockServer, SOCKET*p_sockClient) {
int err = 0;
int* p_int = 0;
std::string host_name("127.0.0.1");
struct sockaddr_in my_addr;
int addr_size = 0;
sockaddr_in sadr_client;
if (!*p_sockServer) {
*p_sockServer = socket(AF_INET, SOCK_STREAM, 0);
if (*p_sockServer == -1) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Server Error initializing socket: {}", log);
goto FINISH;
}
p_int = (int*)malloc(sizeof(int));
*p_int = 1;
if ((setsockopt(*p_sockServer, SOL_SOCKET, SO_REUSEADDR, (char*)p_int, sizeof(int)) == -1)
|| (setsockopt(*p_sockServer, SOL_SOCKET, SO_KEEPALIVE, (char*)p_int, sizeof(int)) == -1)) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Server Error setting options: {}", log);
free(p_int);
goto FINISH;
}
free(p_int);
info("Server socket is set up.");
my_addr.sin_family = AF_INET;
my_addr.sin_port = htons(host_port);
memset(&(my_addr.sin_zero), 0, 8);
my_addr.sin_addr.s_addr = INADDR_ANY;
if (bind(*p_sockServer, (sockaddr*)&my_addr, sizeof(my_addr)) == -1) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Server Error binding to socket, make sure nothing else is listening on this port: {}", log);
goto FINISH;
}
if (listen(*p_sockServer, 10) == -1) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Server Error listening: {}", log);
goto FINISH;
}
info("SUCCESS: Server socket listening ...");
}
info("Server accepting connection ...");
addr_size = sizeof(sockaddr_in);
char sAddress[MAX_PATH];
*p_sockClient = accept(*p_sockServer, (sockaddr*)&sadr_client, &addr_size);
if (*p_sockClient == INVALID_SOCKET) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Server error accepting client connection: {}", log);
// DO NOT close sockets here.
return false;
}
inet_ntop(sadr_client.sin_family, &sadr_client.sin_addr, sAddress, MAX_PATH);
g_timeLatestHeartBeatFirst = GetCurrentTime();
info("SUCCESS: Server accepted client connection.");
return true;
FINISH:
closesocket(*p_sockServer);
return false;
}
bool IsConnected(uint32_t timeLatestHeartBeat) {
// CAUTION: denser than client for sure catch
const unsigned long ConnTimeoutMs = 300;
auto cur = GetCurrentTime();
auto latest = timeLatestHeartBeat;
return cur - latest < ConnTimeoutMs;
}
bool StayInTouch(const char* name, SOCKET* pSockClient, uint32_t* pTimeLatestHeartBeat) {
if (IsConnected(*pTimeLatestHeartBeat))
return true;
char heartBeat[] = "biku";
int nBytesSent = 0;
int flags = 0;
int res = send(*pSockClient, heartBeat, sizeof(heartBeat), flags);
if (res == SOCKET_ERROR) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("{}: Server failed to send heartbeat: {}, Windows error: {}", name, log, GetLastError());
return false;
}
else if (res == 0) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("{}: Server sent zerobyte heartbeat: {}", name, log);
return false;
}
debug("{}: Heartbeat sent: {}", name, heartBeat);
*pTimeLatestHeartBeat = GetCurrentTime();
return true;
}
void Close(SOCKET* pSock) {
closesocket(*pSock);
*pSock = 0;
}
bool Connect() {
if (g_threadFirst.joinable()) {
warn("FirstTunnel already running. Skipped.");
return true;
}
g_threadFirst = std::thread([&]() {
bool isConnected = false;
while (true) {
while (!isConnected) {
isConnected = Init(2345, &g_sockFirst, &g_sockClientFirst);
}
isConnected = StayInTouch("FirstTunnel", &g_sockClientFirst, &g_timeLatestHeartBeatFirst);
if (!isConnected) {
// We don't close as client.
// We keep connecting
error("About to reconnect ...");
Sleep(1000);
continue;
}
if (!g_threadSecond.joinable()) {
g_threadSecond = std::thread([&]() {
while (true) {
while (!isConnected) {
isConnected = Init(2346, &g_sockSecond, &g_sockClientSecond);
}
isConnected = StayInTouch("SecondTunnel", &g_sockClientSecond, &g_timeLatestHeartBeatSecond);
if (!isConnected) {
// We don't close as client.
// We keep connecting
error("About to reconnect ...");
Sleep(1000);
continue;
}
}
info("SecondTunnel quitting...");
Close(&g_sockSecond);
});
}
}
info("FirstTunnel quitting...");
Close(&g_sockFirst);
});
while (true) {
//info("main thread ...");
Sleep(3000);
}
return g_threadFirst.joinable() ? true : false;
}
int main() {
SetupLogger();
info("Hello World!\n");
if (InitWinSock()) {
critical("Failed to initialize Window socket. Aborted.");
}
Connect();
if (g_threadSecond.joinable()) {
g_threadSecond.join();
}
if (g_threadFirst.joinable()) {
g_threadFirst.join();
}
WSACleanup();
info("Bye!");
}
Here is the client code
// hello_dualchannel_client.cpp : This file contains the 'main' function.
//
#include "pch.h"
#include <iostream>
#include <thread>
#include <winsock2.h>
#include <ws2tcpip.h>
#include <spdlog/spdlog.h>
#define SPDLOG_WCHAR_TO_UTF8_SUPPORT
#ifdef _DEBUG
#if !defined(SPDLOG_ACTIVE_LEVEL)
#define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_TRACE
#endif // #if !defined(SPDLOG_ACTIVE_LEVEL)
#define SPDLOG_DEBUG_ON
#define SPDLOG_TRACE_ON
#define _trace SPDLOG_TRACE
#endif // #ifdef _DEBUG
using namespace spdlog;
SOCKET g_sockFirst = 0;
std::thread g_threadFirst;
uint32_t g_timeLatestHeartBeatFirst;
SOCKET g_sockSecond = 0;
std::thread g_threadSecond;
uint32_t g_timeLatestHeartBeatSecond;
void SetupLogger() {
#ifdef _DEBUG
spdlog::set_level(spdlog::level::trace);
spdlog::set_pattern("[%H:%M:%S%z][%^%L%$][%t:%s:%#] %v");
#else
spdlog::set_level(spdlog::level::info);
spdlog::set_pattern("[%H:%M:%S][%^%L%$][%t] %v");
#endif // #ifdef _DEBUG
}
int InitWinSock() {
WORD wVersionRequested;
WSADATA wsaData;
/* Use the MAKEWORD(lowbyte, highbyte) macro declared in Windef.h */
wVersionRequested = MAKEWORD(2, 2);
int err = WSAStartup(wVersionRequested, &wsaData);
if (err != 0) {
/* Tell the user that we could not find a usable */
/* Winsock DLL. */
printf("WSAStartup failed with error: %d\n", err);
return 1;
}
/* Confirm that the WinSock DLL supports 2.2.*/
/* Note that if the DLL supports versions greater */
/* than 2.2 in addition to 2.2, it will still return */
/* 2.2 in wVersion since that is the version we */
/* requested. */
if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2) {
/* Tell the user that we could not find a usable */
/* WinSock DLL. */
printf("Could not find a usable version of Winsock.dll\n");
WSACleanup();
return 1;
}
else
printf("The Winsock 2.2 dll was found okay\n");
return 0;
}
bool Init(int host_port, SOCKET* p_sock) {
int err = 0;
int* p_int = 0;
std::string host_name("127.0.0.1");
struct sockaddr_in my_addr;
char handshake[] = "hello";
//int nBytesSent;
if (!*p_sock) {
*p_sock = socket(AF_INET, SOCK_STREAM, 0);
if (*p_sock == -1) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Client Error initializing socket {}", log);
goto FINISH;
}
p_int = (int*)malloc(sizeof(int));
*p_int = 1;
if ((setsockopt(*p_sock, SOL_SOCKET, SO_REUSEADDR, (char*)p_int, sizeof(int)) == -1)
|| (setsockopt(*p_sock, SOL_SOCKET, SO_KEEPALIVE, (char*)p_int, sizeof(int)) == -1)) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Client Error setting options {}", log);
free(p_int);
goto FINISH;
}
free(p_int);
info("SUCCESS: Client socket is set up.");
}
my_addr.sin_family = AF_INET;
my_addr.sin_port = htons(host_port);
memset(&(my_addr.sin_zero), 0, 8);
inet_pton(my_addr.sin_family, host_name.c_str(), &my_addr.sin_addr);
if (connect(*p_sock, (struct sockaddr*)&my_addr, sizeof(my_addr)) == SOCKET_ERROR) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Client Error connecting socket {}", log);
Sleep(1000);
goto FINISH;
}
/*nBytesSent = send(g_sockFirst, handshake, sizeof(handshake), 0);
if (nBytesSent <= 0) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Client error sending handshake: {}", log);
goto FINISH;
}*/
g_timeLatestHeartBeatFirst = GetCurrentTime();
info("SUCCESS: Client connected to server.");
return true;
FINISH:
closesocket(*p_sock);
*p_sock = 0;
return false;
}
bool IsConnected(uint32_t timeLatestHeartBeat) {
const unsigned long ConnTimeoutMs = 3000;
auto cur = GetCurrentTime();
auto latest = timeLatestHeartBeat;
//if (cur - latest > ConnTimeoutMs)
//{
// debug("cur: {}, late: {}", cur, latest);
//}
return cur - latest < ConnTimeoutMs;
}
bool StayInTouch(const char* name, SOCKET* pSock, uint32_t* pTimeLatestHeartBeat) {
// Client checks inbox right away and measure timeout later.
char heartBeat[MAX_PATH] = { 0 };
// CAUTION: min 100ms required for receiving heartbeat.
//const uint32_t TimeoutMS = 100;
int flags = 0;
int nBytesRecved = recv(*pSock, heartBeat, sizeof(heartBeat), flags);
bool gotHeartbeat = nBytesRecved > 0;
if (gotHeartbeat) {
debug("{}: Heartbeat received: {}", name, heartBeat);
*pTimeLatestHeartBeat = GetCurrentTime();
}
return IsConnected(*pTimeLatestHeartBeat);
}
void Close(SOCKET* pSock) {
closesocket(*pSock);
*pSock = 0;
}
bool Connect() {
if (g_threadFirst.joinable()) {
warn("FirstTunnel already running. Skipped.");
return true;
}
g_threadFirst = std::thread([&]() {
bool isConnected = false;
while (true) {
while (!isConnected) {
isConnected = Init(2345, &g_sockFirst);
}
isConnected = StayInTouch("FirstTunnel", &g_sockFirst, &g_timeLatestHeartBeatFirst);
if (!isConnected) {
// We don't close as client.
// We keep connecting
Close(&g_sockFirst);
error("About to reconnect ...");
continue;
}
if (!g_threadSecond.joinable()) {
g_threadSecond = std::thread([&]() {
while (true) {
while (!isConnected) {
isConnected = Init(2346, &g_sockSecond);
}
isConnected = StayInTouch("SecondTunnel", &g_sockSecond, &g_timeLatestHeartBeatSecond);
if (!isConnected) {
// We don't close as client.
// We keep connecting
error("About to reconnect ...");
Sleep(1000);
continue;
}
}
info("SecondTunnel quitting...");
Close(&g_sockSecond);
});
}
}
info("FirstTunnel quitting.");
Close(&g_sockFirst);
});
while (true) {
//info("main thread ...");
Sleep(3000);
}
return g_threadFirst.joinable() ? true : false;
}
int main() {
SetupLogger();
info("Hello World!\n");
if (InitWinSock()) {
critical("Failed to initialize Window socket. Aborted.");
}
Connect();
if (g_threadSecond.joinable()) {
g_threadSecond.join();
}
if (g_threadFirst.joinable()) {
g_threadFirst.join();
}
WSACleanup();
info("Bye!");
}
The main connection logic is in the function Connect().
I'd appreciate tips on where I was wrong.
To run the code as is, you need one dependency spdlog
vcpkg install spdlog:x64-Windows
You can also replace all the spdlog-based logging code with your own.
UPDATE
Observation #1
I stepped into the code and confirmed
All the loops are running. So all the threads are spawned.
No redundant threads are spawned due to the joinable() guard.
The only failure point is the recv call of the second client.
So conclusion
No firewalls
No redundant threads
Both threads run on the client and server sides by design.
Observation #2
While running the server and client programs and having let the problem happen, I tried
Keep both programs open.
Run netcat (NMap's netcat port) like this C:\Apps\Nmap\ncat.exe 127.0.0.1 2346
This actually help fix things. It tells me that there is a connection problem, while I could definitely step into the client code and see that the connection is still there.
Observation #3
After I leave breakpoints in only the second connection code of the client program, and run the program, I could not step out into the first connection code. Note that they are in separate threads.
Leaving breakpoints in both connections, and quickly step over between them, I could keep the stepping going without getting trapped in only one thread. This is when I notice that the client starts receiving the messages it should.
So I suspect that there is a threading issue there. If that's the problem, How to fix this then?
I found the problem myself. It was a stupid bug on my part:
The two threads share a state by mistake: isConnected. So the original programs keep kicking each other after one of them gains a new connection or heartbeat. They should have used different states for that.
To give more detail: isConnected, which is initialized by g_threadFirst, falls through the closure of g_threadSecond's anonymous function.
So I'm writing a very basic TCP server which just echoes back the messages sent from the client back to the client. I have a setup where the server is running in a while loop and waiting for either a new client to connect or for a message from one of the existing clients using the select() method.
My question is:
How do i, from the server side, close the master socket and basically shutting down the server. Im not asking for exact code but more for standard practice.
For some context:
In the long run I am imagining multiple clients connected to my server and I need to shutdown the server gracefully from the server side.
Even more context: The server code.
#define TRUE 1
#define FALSE 0
#define PORT 55555
int main(int argc, char* argv[])
{
int opt = TRUE;
int masterSocket, addrlen, newSocket, maxClients = 10, clientSockets[maxClients],
activity, valread, sd, maxSd;
struct sockaddr_in address;
char buffer[1025];
fd_set readfds;
const char *message = "ECHO DAMON v1.0 \r\n";
/* Initialize array with 0 so it's not read */
for(int i = 0; i < maxClients ; i++)
{
clientSockets[i] = 0;
}
/* Create master socket */
if((masterSocket = socket(AF_INET, SOCK_STREAM,0)) == 0)
{
perror("Error when trying to create master socket.\n");
exit(EXIT_FAILURE);
}
/* Set master socket to allow multiple connections */
if( setsockopt(masterSocket, SOL_SOCKET, SO_REUSEADDR, (char*)&opt, sizeof(opt)) < 0)
{
perror("Could not set sockopt");
exit(EXIT_FAILURE);
}
/* Socket type */
address.sin_family = AF_INET;
address.sin_addr.s_addr = INADDR_ANY;
address.sin_port = htons(PORT);
if( bind(masterSocket, (struct sockaddr*)&address, sizeof(address)) < 0)
{
perror("Error, could not bind master socket. \n");
exit(EXIT_FAILURE);
}
printf("Listening on %d. \n", PORT);
if( listen(masterSocket, 3) < 0)
{
perror("Error, could not listen.\n");
exit(EXIT_FAILURE);
}
addrlen = sizeof(address);
puts("Waiting for connections...\n"); //just a printf variant
/* END INIT */
while(TRUE)
{
/* Clear socket set */
FD_ZERO(&readfds);
/* Add master socket to set */
FD_SET(masterSocket, &readfds);
/* Add child sockets to set, will be 0 first iteration */
for(int i = 0; i < maxClients ; i++)
{
sd = clientSockets[i]; // sd = socket descriptor
/* If valid socket descriptor */
if(sd > 0)
{
FD_SET(sd, &readfds);
}
/* Get highest fd number, needed for the select function (later) */
if(sd > maxSd)
{
maxSd = sd;
}
}//end for-loop
/* Wait for activity on any socket */
activity = select(maxSd +1, &readfds, NULL, NULL, NULL);
if((activity < 0) && (errno != EINTR))
{
printf("Error on select.\n"); //no need for exit.
}
/* If the bit for the file descriptor fd is set in the
file descriptor set pointed to by fdset */
/* If something happend in the master socket, its a new connection */
if(FD_ISSET(masterSocket, &readfds))
{
if((newSocket = accept(masterSocket, (struct sockaddr*)&address, (socklen_t*)&addrlen)) < 0)
{
perror("Could not accept new socket.\n");
exit(EXIT_FAILURE);
}
/* Print info about connector */
printf("New connection, socket fd is %d, ip is: %s, port: %d\n", newSocket, inet_ntoa(address.sin_addr), ntohs(address.sin_port));
if( send(newSocket, message, strlen(message), 0) != strlen(message))
{
perror("Could not sent welcome message to new socket.\n");
}
puts("Welcome message sen successfully.\n");
/* Add new socket to array of clients */
for(int i = 0; i < maxClients; i++)
{
if(clientSockets[i] == 0)
{
clientSockets[i] = newSocket;
printf("Adding socket to list of client at index %d\n", i);
break;
}
}
}//end masterSocket if
/* Else something happend at client side */
for(int i = 0; i < maxClients; i++)
{
sd = clientSockets[i];
if(FD_ISSET(sd, &readfds))
{ /* Read socket, if it was closing, else read value */
if((valread = read( sd, buffer, 1024)) == 0)
{
getpeername( sd, (struct sockaddr*)&address, (socklen_t*)&addrlen);
printf("Host disconnected, ip %s, port %d.\n", inet_ntoa(address.sin_addr), ntohs(address.sin_port));
close(sd);
clientSockets[i] = 0;
}
}
else
{
buffer[valread] = '\0';
send(sd, buffer, strlen(buffer), 0);
}
}
}
return 0;
}
If you are planning to exit your application gracefully by adding a breakstatement in your app by something similar to this:
if (exit_condition)
{
break;
}
You can place this loop at the end of your main function:
/* close connections gracefully */
closesocket(masterSocket);
masterSocket = 0; /* optional, see comment below */
for(int i = 0; i < maxClients; i++)
{
if (clientSockets[i] != 0)
{
shutdown(clientSockets[i]);
closesocket(clientSockets[i]);
clientSockets[i] = 0; /* optional, except if you also have a SIGINT handler */
}
}
If you want to do the same to handle an exit with Ctrl-C, you will find details on how to setup a SIGINT handler to handle Ctr-C here:
Catch Ctrl-C in C. Place the above loop in your handler, then. Your sockets-related variables will have to be declared at global scope, since your sockets are only visible from main() in your current code.
I'm trying to understand this Libevent c++ code I got from this page.
I'm a bit confused - am I correct to think that this code might have memory leaks?
It seems like ConnectionData pointer is created in on_connect() callback, but delete() is only called on bad read or after write is complete.
What if connection was accept()ed - but there were no reads or writes? so is that pointer just stays in daemon memory?
#include <event.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <iostream>
// Read/write buffer max length
static const size_t MAX_BUF = 512;
typedef struct {
struct event ev;
char buf[MAX_BUF];
size_t offset;
size_t size;
} ConnectionData;
void on_connect(int fd, short event, void *arg);
void client_read(int fd, short event, void *arg);
void client_write(int fd, short event, void *arg);
int main(int argc, char **argv)
{
// Check arguments
if (argc < 3) {
std::cout << "Run with options: <ip address> <port>" << std::endl;
return 1;
}
// Create server socket
int server_sock = socket(AF_INET, SOCK_STREAM, 0);
if (server_sock == -1) {
std::cerr << "Failed to create socket" << std::endl;
return 1;
}
sockaddr_in sa;
int on = 1;
char * ip_addr = argv[1];
short port = atoi(argv[2]);
sa.sin_family = AF_INET;
sa.sin_port = htons(port);
sa.sin_addr.s_addr = inet_addr(ip_addr);
// Set option SO_REUSEADDR to reuse same host:port in a short time
if (setsockopt(server_sock, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) {
std::cerr << "Failed to set option SO_REUSEADDR" << std::endl;
return 1;
}
// Bind server socket to ip:port
if (bind(server_sock, reinterpret_cast<const sockaddr*>(&sa), sizeof(sa)) == -1) {
std::cerr << "Failed to bind server socket" << std::endl;
return 1;
}
// Make server to listen
if (listen(server_sock, 10) == -1) {
std::cerr << "Failed to make server listen" << std::endl;
return 1;
}
// Init events
struct event evserver_sock;
// Initialize
event_init();
// Set connection callback (on_connect()) to read event on server socket
event_set(&evserver_sock, server_sock, EV_READ, on_connect, &evserver_sock);
// Add server event without timeout
event_add(&evserver_sock, NULL);
// Dispatch events
event_dispatch();
return 0;
}
// Handle new connection {{{
void on_connect(int fd, short event, void *arg)
{
sockaddr_in client_addr;
socklen_t len = 0;
// Accept incoming connection
int sock = accept(fd, reinterpret_cast<sockaddr*>(&client_addr), &len);
if (sock < 1) {
return;
}
// Set read callback to client socket
ConnectionData * data = new ConnectionData;
event_set(&data->ev, sock, EV_READ, client_read, data);
// Reschedule server event
event_add(reinterpret_cast<struct event*>(arg), NULL);
// Schedule client event
event_add(&data->ev, NULL);
}
//}}}
// Handle client request {{{
void client_read(int fd, short event, void *arg)
{
ConnectionData * data = reinterpret_cast<ConnectionData*>(arg);
if (!data) {
close(fd);
return;
}
int len = read(fd, data->buf, MAX_BUF - 1);
if (len < 1) {
close(fd);
delete data;
return;
}
data->buf[len] = 0;
data->size = len;
data->offset = 0;
// Set write callback to client socket
event_set(&data->ev, fd, EV_WRITE, client_write, data);
// Schedule client event
event_add(&data->ev, NULL);
}
//}}}
// Handle client responce {{{
void client_write(int fd, short event, void *arg)
{
ConnectionData * data = reinterpret_cast<ConnectionData*>(arg);
if (!data) {
close(fd);
return;
}
// Send data to client
int len = write(fd, data->buf + data->offset, data->size - data->offset);
if (len < data->size - data->offset) {
// Failed to send rest data, need to reschedule
data->offset += len;
event_set(&data->ev, fd, EV_WRITE, client_write, data);
// Schedule client event
event_add(&data->ev, NULL);
}
close(fd);
delete data;
}
//}}}
The documentation for event_set says that the only valid event types are EV_READ or EV_WRITE, but the callback will be invoked with EV_TIMEOUT, EV_SIGNAL, EV_READ, or EV_WRITE. The documentation is not clear, but I expect the read callback will be invoked when the socket is closed by the client. I expect the delete in the failure branch in client_read will handle this situation.
Note that that is only the case if the client sends a FIN or RST packet. A client could establish a connection and leave it open forever. For this reason, this code should be modified to have a timeout (perhaps via event_once) and require the client send a message within that timeout.
I can't seem to get working 2 watchers per socket.. Code below doesn't acually works at all, but if i mix up these calls(for ex. call init/set/start for 1 watcher and then for other), i get only 1 watcher working.. Is there something I'm missing badly here...?
I don't think it has anything to do with loops and setup... I have 1 accept loop(default loop) and 1 loop for accepted connections. I tried both, running code below directly after accepting connection on accept loop and via ev_async_send(...) then executing this code from other io loop. Results were same.
Also setting both events on 1 watcher works fine too.
Thank you
ev_init (pSockWatcher->_wW, &CNetServer::send_cb);
ev_init (pSockWatcher->_wR, &CNetServer::recv_cb);
ev_io_set (pSockWatcher->_wW, pSockWatcher->_sd, EV_WRITE );
ev_io_set (pSockWatcher->_wR, pSockWatcher->_sd, EV_READ );
ev_io_start (loop, pSockWatcher->_wR);
ev_io_start (loop, pSockWatcher->_wW);
Well, here is an example with two I/O watchers on one socket fd, which seems to work fine for me. I am using the ev_io_init() function, however, not the ev_init() and ev_set().
#include <ev.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
static struct ev_loop *loop;
static ev_timer timeout_watcher;
static ev_io in_watcher, out_watcher;
static ev_idle idle_watcher;
static int sock_fd;
// socket input watcher
static void in_cb(EV_P_ ev_io *watcher, int revents) {
int r, t;
char buf[1024];
for (t = 0; (r = read(sock_fd, buf, sizeof(buf))) > 0;) {
t += r;
write(STDOUT_FILENO, buf, r); // copy input to stdout
if (buf[r-1] == '\n') break; // operate line-at-a-time
}
fprintf(stderr, "in: count = %d\n", t);
if (r == 0) {
fputs("in: connection closed\n", stderr);
ev_io_stop(loop, &in_watcher); // stop the socket watcher
ev_break(loop, EVBREAK_ALL); // exit the loop
} else if (r < 0) {
perror("read");
}
}
// socket output watcher
static void out_cb(EV_P_ ev_io *watcher, int revents) {
int r, t, lim;
char buf[1024];
ev_io_stop(loop, &out_watcher);
for (t = 0; t < sizeof(buf); t++) {
buf[t] = 'a' + (rand() % 26);
}
for (t = 0, lim = rand() % 10000 + 1000;
(r = write(sock_fd, buf, (lim - t > sizeof(buf)) ? sizeof(buf) : lim - t)) > 0;) {
t += r;
if (t >= lim) break;
}
if (r < 0) {
perror("write");
}
fprintf(stderr, "out: finished sending, count = %d\n", t);
}
static void timeout_cb(EV_P_ ev_timer *watcher, int revents) {
fprintf(stderr, "timeout: now = %f\n", ev_now(loop));
// send a bunch of stuff on the socket when able
ev_io_start (loop, &out_watcher);
}
static void idle_cb(EV_P_ ev_idle *watcher, int revents) {
static long idle_count = 0;
fprintf(stderr, "idle: count = %ld\n", ++idle_count);
sleep(1); // simulate doing stuff
}
int main() {
extern int errno;
int master_fd;
int sock_opt = 1;
int conn_port = 7000;
struct sockaddr_in addr;
socklen_t addrlen;
// **** the following is needed to set up a socket to receive data ****
master_fd = socket(AF_INET, SOCK_STREAM, 0);
if (master_fd == -1) {
perror("socket");
return errno;
}
if (setsockopt(master_fd, SOL_SOCKET, SO_REUSEADDR, (char *) &sock_opt, sizeof(sock_opt)) == -1) {
perror("setsockopt");
return errno;
}
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = INADDR_ANY;
addr.sin_port = htons(conn_port);
addrlen = sizeof(addr);
if (bind(master_fd, (struct sockaddr *) &addr, addrlen) != 0) {
perror("bind");
return errno;
}
if (listen(master_fd, 3) != 0) {
perror("listen");
return errno;
}
fprintf(stderr, "awaiting a connection on port %d\n", conn_port);
sock_fd = accept(master_fd, (struct sockaddr *) &addr, &addrlen);
if (sock_fd == -1) {
perror("accept");
return errno;
}
fputs("in: connection established\n", stderr);
// **** end of socket setup code ****
// define a loop
loop = ev_default_loop(0);
// define a repeating timer
ev_timer_init (&timeout_watcher, timeout_cb, 5.0, 5.0);
ev_timer_start (loop, &timeout_watcher);
// define an idle process
ev_idle_init(&idle_watcher, idle_cb);
ev_idle_start (loop, &idle_watcher);
// define the socket data receiver
ev_io_init(&in_watcher, in_cb, sock_fd, EV_READ);
ev_io_start (loop, &in_watcher);
// define the socket data write complete watcher
ev_io_init(&out_watcher, out_cb, sock_fd, EV_WRITE);
// run the loop
ev_run(loop, 0);
// clean up
close(sock_fd);
close(master_fd);
return 0;
}