thread-safety issues managing connections objects inside a TCP server - c++

Question

I'm coding a simple client-server application using TCP. I'm using Winsock, but I'm limiting myself to the BSD socket API (since I want the code to run on Linux as well).
The application should run a server and accept incoming client connections, receive and send messages, and gracefully handle client disconnections. The approach I use is this:
(disclaimer: I know it might not be the best one, and that there are a bunch of more efficient techniques, but I'm just starting with network programming)
A connection object (that represents the actual endpoint of the connection and contains a socket object) has a queue of outgoing messages, and a reference to a queue of incoming messages (actual incoming messages queues are inside the clients and the server object, and are passed on construction):
#ifndef CONNECTION_H
#define CONNECTION_H
#include <memory>
#include <thread>
#include "ThreadsafeQueue.h"
#include "Message.h"
#include "debug.h"
template <typename T>
class Connection : public std::enable_shared_from_this<Connection<T>>
{
public:
enum class Owner { CLIENT, SERVER };
private:
using std::enable_shared_from_this<Connection>::shared_from_this;
public:
Connection(Owner owner, uint32_t id, SOCKET socket, ThreadsafeQueue<OwnedMessage<T>> &inMessageQueue);
~Connection();
void Send(const Message<T> &message);
void Close();
bool IsOpen() const { return mIsOpen; }
uint32_t GetId() const { return mId; }
private:
const Owner mOwner;
uint32_t mId;
SOCKET mSocket;
bool mIsOpen;
ThreadsafeQueue<Message<T>> mOutMessageQueue;
ThreadsafeQueue<OwnedMessage<T>> &mInMessageQueue;
std::thread mSendThread;
std::thread mReceiveThread;
void Send_();
void Receive_();
};
template <typename T>
Connection<T>::Connection(Owner owner, uint32_t id, SOCKET socket, ThreadsafeQueue<OwnedMessage<T>> &inMessageQueue) : mOwner(owner), mId(id), mSocket(socket), mInMessageQueue(inMessageQueue)
{
unsigned long socketMode = 1U;
if (ioctlsocket(socket, FIONBIO, &socketMode) != 0) // set non blocking socket
Error("error setting socket i/o mode");
mIsOpen = true;
mSendThread = std::thread(&Connection<T>::Send_, this);
mReceiveThread = std::thread(&Connection<T>::Receive_, this);
}
template <typename T>
Connection<T>::~Connection()
{
Close();
}
template <typename T>
void Connection<T>::Send(const Message<T> &message)
{
mOutMessageQueue.EnQueue(message);
}
template <typename T>
void Connection<T>::Close()
{
if (mIsOpen)
mIsOpen = false;
if (mSendThread.joinable())
mSendThread.join();
if (mReceiveThread.joinable())
mReceiveThread.join();
closesocket(mSocket);
}
template <typename T>
void Connection<T>::Send_()
{
//DbgPrint("send thread start");
while (mIsOpen)
{
// get from outgoing queue and send
if (!mOutMessageQueue.Empty())
{
Message<T> outMessage = mOutMessageQueue.Front();
mOutMessageQueue.DeQueue();
int sendFlags = 0;
int totalBytesSent = 0;
while (totalBytesSent < sizeof(Message<T>::Header))
{
int bytesSent = send(mSocket, reinterpret_cast<char*>(&outMessage.mHeader) + totalBytesSent, sizeof(Message<T>::Header) - totalBytesSent, sendFlags);
totalBytesSent += bytesSent;
if (bytesSent == SOCKET_ERROR)
Error("error sending message");
}
totalBytesSent = 0;
while (totalBytesSent < outMessage.mBody.Size())
{
int bytesSent = send(mSocket, reinterpret_cast<char*>(outMessage.mBody.Data()) + totalBytesSent, outMessage.mBody.Size() - totalBytesSent, sendFlags);
totalBytesSent += bytesSent;
if (bytesSent == SOCKET_ERROR)
Error("error sending message");
}
}
}
//DbgPrint("send thread end");
}
template <typename T>
void Connection<T>::Receive_()
{
//DbgPrint("receive thread start");
static bool skip = false; // used in non blocking mode
while (mIsOpen)
{
skip = false;
Message<T> inMessage;
int recvFlags = 0;
int totalBytesReceived = 0;
while (totalBytesReceived < sizeof(Message<T>::Header)) // receive message header
{
int bytesReceived = recv(mSocket, reinterpret_cast<char*>(&inMessage.mHeader) + totalBytesReceived, sizeof(Message<T>::Header) - totalBytesReceived, recvFlags);
if (bytesReceived == SOCKET_ERROR && WSAGetLastError() == WSAEWOULDBLOCK)
{
skip = true;
break;
}
if (bytesReceived == SOCKET_ERROR)
Error("error receiving message");
else if (bytesReceived == 0) // other side closed connection
{
mIsOpen = false;
skip = true;
break;
}
totalBytesReceived += bytesReceived;
}
if (skip)
continue;
inMessage.mBody.Resize(inMessage.mHeader.mSize);
totalBytesReceived = 0;
while (totalBytesReceived < inMessage.mBody.Size()) // receive message body
{
int bytesReceived = recv(mSocket, reinterpret_cast<char*>(inMessage.mBody.Data()) + totalBytesReceived, inMessage.mBody.Size() - totalBytesReceived, recvFlags);
if (bytesReceived == SOCKET_ERROR && WSAGetLastError() == WSAEWOULDBLOCK)
continue;
totalBytesReceived += bytesReceived;
if (bytesReceived == SOCKET_ERROR)
Error("error receiving message");
else if (bytesReceived == 0) // other side closed connection
{
mIsOpen = false;
skip = true;
break;
}
}
if (skip)
continue;
if (mOwner == Owner::SERVER)
mInMessageQueue.EnQueue(OwnedMessage<T>(shared_from_this(), inMessage)); // put received message into incoming queue
else
mInMessageQueue.EnQueue(OwnedMessage<T>(nullptr, inMessage));
}
//DbgPrint("receive thread start");
}
#endif // CONNECTION_H
A client has one Connection objects, the server has many of them in a container. Server runs and accept clients, spawns a Connection and put it into a container. Each connection runs in two threads (one to send and the other to receive, since my sockets are non-blocking it is redundant, I'll address the issue in the future). The connected client stores its own connection as well. The Send_ () and Receive_() threads take care of transferring messages in and out of the message queues (which are brutally thread-safe by synchronizing every operation on a std::mutex, not the best design I know).
The server runs two threads: one to keep listening to incoming clients, the other to "garbage-collect" the closed connections: a server-side Connection can detect if a client closed its connection by reading 0 from recv(), and signals that to the server by setting mIsOpen to false. The CloseSocket thread runs and cleans up the connections (calling the OnClientDisconnect hook):
#ifndef SERVER_H
#define SERVER_H
#include <WinSock2.h>
#include <WS2tcpip.h>
#include <thread>
#include <string>
#include "Vector.h"
#include "Connection.h"
#include "ThreadsafeQueue.h"
#include "Message.h"
#include "debug.h"
template <typename T>
class Server
{
protected:
using ConnectionPtr = std::shared_ptr<Connection<T>>;
public:
Server(uint16_t port);
~Server();
void Start();
void Stop();
void Send(ConnectionPtr connection, const Message<T> &message) const;
void SendAll(const Message<T> &message, ConnectionPtr ignore = nullptr) const;
void Disconnect(ConnectionPtr connection);
bool Available() const { return !mInMessageQueue.Empty(); }
void ProcessMessage();
protected:
virtual void OnClientConnect(ConnectionPtr connection) = 0;
virtual void OnClientDisconnect(ConnectionPtr connection) = 0;
virtual void OnMessage(ConnectionPtr sender, Message<T> &message) = 0;
private:
Vector<ConnectionPtr> mConnections;
ThreadsafeQueue<OwnedMessage<T>> mInMessageQueue;
SOCKET mListenSocket;
std::thread mListenThread;
void Listen();
std::thread mSocketRemoveThread;
void RemoveConnections();
std::string mHost;
uint16_t mPort;
static const uint8_t sMaxNumConnections = 10;
bool mIsRunning;
};
template <typename T>
Server<T>::Server(uint16_t port) :mListenSocket(INVALID_SOCKET), mIsRunning(false)
{
WSAData wsa;
WSAStartup(MAKEWORD(2, 2), &wsa);
addrinfo hints, *address;
memset(&hints, 0, sizeof hints);
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
//hints.ai_flags = AI_PASSIVE;
if (getaddrinfo(nullptr, std::to_string(port).c_str(), &hints, &address) != 0)
Error("cannot get server address");
char stringBuf[INET6_ADDRSTRLEN];
if (address->ai_family == AF_INET)
inet_ntop(AF_INET, &reinterpret_cast<sockaddr_in *>(address->ai_addr)->sin_addr, stringBuf, sizeof stringBuf);
else // address->ai_family == AF_INET6
inet_ntop(AF_INET6, &reinterpret_cast<sockaddr_in6 *>(address->ai_addr)->sin6_addr, stringBuf, sizeof stringBuf);
mHost = stringBuf;
mPort = ntohs(address->ai_family == AF_INET ? reinterpret_cast<sockaddr_in *>(address->ai_addr)->sin_port : reinterpret_cast<sockaddr_in6 *>(address->ai_addr)->sin6_port);
if ((mListenSocket = socket(address->ai_family, address->ai_socktype, address->ai_protocol)) == INVALID_SOCKET)
Error("cannot create server socket");
if (bind(mListenSocket, address->ai_addr, address->ai_addrlen) != 0)
Error("cannot bind server socket");
}
template <typename T>
Server<T>::~Server()
{
if (mIsRunning)
closesocket(mListenSocket);
WSACleanup();
}
template <typename T>
void Server<T>::Start()
{
if (mIsRunning) // TODO: error check
return;
mIsRunning = true;
PRINTLN("server running");
mListenThread = std::thread(&Server::Listen, this); // start thread that accepts new connections
mSocketRemoveThread = std::thread(&Server::RemoveConnections, this);
}
template <typename T>
void Server<T>::Stop()
{
for (std::unique_ptr<Connection<T>> &connection : mConnections)
connection->Close();
}
template <typename T>
void Server<T>::Listen()
{
if (listen(mListenSocket, sMaxNumConnections) != 0)
Error("listen error");
PRINT("server listening # ");
PRINT(mHost);
PRINT(" on port ");
PRINTLN(std::to_string(mPort));
static uint32_t connectionId = 0;
while (true)
{
sockaddr_storage clientAddress;
int clientAddressLength = sizeof(sockaddr_storage);
SOCKET clientSocket = accept(mListenSocket, reinterpret_cast<sockaddr*>(&clientAddress), &clientAddressLength); // accept connections (blocking)
if (clientSocket == INVALID_SOCKET)
Error("cannot create client socket");
mConnections.EmplaceLast(new Connection<T>(Connection<T>::Owner::SERVER, connectionId++, clientSocket, mInMessageQueue)); // create new connection to client (receive/send threads)
OnClientConnect(mConnections.Last()); // callback called on new connections
char stringBuf[INET6_ADDRSTRLEN];
if (clientAddress.ss_family == AF_INET)
inet_ntop(AF_INET, &reinterpret_cast<sockaddr_in *>(&clientAddress)->sin_addr, stringBuf, sizeof stringBuf);
else // clientAddress.ss_family == AF_INET6
inet_ntop(AF_INET6, &reinterpret_cast<sockaddr_in6 *>(&clientAddress)->sin6_addr, stringBuf, sizeof stringBuf);
PRINT("accepted connection from ");
PRINT(stringBuf);
PRINT(" on port ");
PRINTLN(clientAddress.ss_family == AF_INET ? std::to_string(ntohs(reinterpret_cast<sockaddr_in *>(&clientAddress)->sin_port)) : std::to_string(ntohs(reinterpret_cast<sockaddr_in6 *>(&clientAddress)->sin6_port)));
}
}
template <typename T>
void Server<T>::Send(ConnectionPtr connection, const Message<T> &message) const
{
if (connection->IsOpen())
connection->Send(message);
}
template <typename T>
void Server<T>::SendAll(const Message<T> &message, ConnectionPtr ignore) const
{
for (const ConnectionPtr &connection : mConnections)
{
if (connection != ignore && connection->IsOpen())
connection->Send(message);
}
}
template <typename T>
void Server<T>::ProcessMessage()
{
OwnedMessage<T> message(mInMessageQueue.Front());
mInMessageQueue.DeQueue();
OnMessage(message.GetSender(), message);
}
template <typename T>
void Server<T>::RemoveConnections()
{
while (true)
{
for (typename Vector<ConnectionPtr>::Iterator it = mConnections.Begin(), end = mConnections.End(); it != end; ++it)
if (!(*it)->IsOpen())
{
(*it)->Close();
OnClientDisconnect(*it);
it = mConnections.Remove(it);
}
}
}
#endif // SERVER_H
Everything works fine, but I have problems with synchronizing the container of Connections in the server: it is accessed by both the listening thread (which inserts new connections) and the garbage collector thread (which scans the array for dead connection to remove).
How can I address this issue? A vector is obviously the worst choice (dynamic reallocation), being inherently unsafe for concurrent access, but how can I store connections that needs to be manipulated by different threads? Please I need basic/bare-bones/not fancy solutions, since this application is for learning purposes only.
As an aside, how/where can I start learning about how to structure network code more efficiently? How can I improve the structure of my framework?
Thanks