Making a boost C++ UdpConnection class thread-safe - c++

Question

I have implemented some connection classes, using Boost ASIO, to replace some low level C code in an application, and everything is working great, except for one problem.
Basically, I have a UdpConnection class that does synchronous read and write, but it uses async methods to handle time-outs as per the boost examples. The problem is I can't figure out how to make it threadsafe.
I have tried adding strands to the event handlers to make this class threadsafe (code below), but that isn't working. I suspect it is because of the way timeout is implemented. I have included my code in 4 classes in pastebin.
Single threaded is working fine. I also have TcpConnection and UnixSocketConnection classes that don't need to be shared amongst multiple threads and they work fine. However, I can't get multi-threaded UDP code to work.
Am I missing something?
Connection.h && AsioConnection.h http://pastebin.com/Cbbw37gL
UdpConnection.h && UdpConnection.cpp http://pastebin.com/VLnHBnPs
EDIT Attaching code as suggested:
AsioConnection.h
/*
* AsioConnection.h
*
* Created on: 2011-04-08
* Author: cdunphy
*
* All classes that want to use the ASIO io_service
* and deadline timers will want to subclass this.
*/
#ifndef ASIOCONNECTION_H_
#define ASIOCONNECTION_H_
#include "Connection.h"
#include <boost/shared_ptr.hpp>
#include <boost/asio.hpp>
#include <boost/array.hpp>
#include <boost/bind.hpp>
namespace shaw_rsc
{
/*
* This exception throws if there is a timeout when connecting
* to a remote socket.
*/
struct SocketTimeoutException : public std::runtime_error
{
SocketTimeoutException(const std::string& msg) : std::runtime_error(msg)
{ }
}
;
/*
* This is the root class of every Connection
* class that wants to make use of boost asio.
*/
class AsioConnection : public Connection
{
public:
AsioConnection(
int c_timeout,
int r_timeout
) : Connection(),
conn_timeout_int(c_timeout),
read_timeout_int(r_timeout),
conn_timeout(c_timeout),
read_timeout(r_timeout),
io_service(),
strand(io_service),
deadline(strand.get_io_service()),
error()
{
reset_deadline();
}
const boost::system::error_code& getError() const
{
return error;
}
int get_read_timeout() const
{
return read_timeout_int;
}
int get_conn_timeout() const
{
return conn_timeout_int;
}
/*
* These are the callback handlers for our asynchronous
* IO operations.
*/
void handle_write(const boost::system::error_code& ec,
std::size_t len,
boost::system::error_code* out_ec,
std::size_t* out_len)
{
*out_ec = ec;
*out_len = len;
}
/*
* These are the callback handlers for our asynchronous
* IO operations.
*/
void handle_send(const boost::system::error_code& ec,
std::size_t len,
boost::system::error_code* out_ec,
std::size_t* out_len)
{
*out_ec = ec;
*out_len = len;
}
void handle_read(const boost::system::error_code& ec,
std::size_t len,
boost::system::error_code* out_ec,
std::size_t* out_len)
{
*out_ec = ec;
*out_len = len;
}
void handle_receive(const boost::system::error_code& ec,
std::size_t len,
boost::system::error_code* out_ec,
std::size_t* out_len)
{
*out_ec = ec;
*out_len = len;
}
void handle_connect(const boost::system::error_code& ec,
boost::system::error_code* out_ec)
{
*out_ec = ec;
}
protected:
int conn_timeout_int;
int read_timeout_int;
boost::posix_time::seconds conn_timeout;
boost::posix_time::seconds read_timeout;
boost::asio::io_service io_service;
boost::asio::strand strand;
boost::asio::deadline_timer deadline;
boost::system::error_code error;
void reset_deadline()
{
// No deadline is required until the first socket operation is started. We
// set the deadline to positive infinity so that the actor takes no action
// until a specific deadline is set.
deadline.expires_at(boost::posix_time::pos_infin);
}
};
}
#endif /* ASIOCONNECTION_H_ */
Connection.h
/*
* Connection.h
*
* Created on: 2011-02-25
* Author: cdunphy
*/
#ifndef CONNECTION_H_
#define CONNECTION_H_
#include <vector>
#include <string>
#include <sstream>
#include <stdexcept>
#include <boost/thread.hpp>
#include <boost/shared_ptr.hpp>
namespace shaw_rsc
{
class Connection;
const std::size_t BUF_SIZE = 128;
/*
* This is the type of reference we will
* provide to the clients.
*/
typedef boost::shared_ptr<Connection> ConnPtr;
typedef std::vector<char> DataBuffer;
typedef DataBuffer::iterator DB_Iter;
typedef DataBuffer::const_iterator DB_CIter;
// This is the mode we are using for the connection
enum Mode {
CLIENT,
SERVER
};
/*
* This is a generic class that allows data to be read or
* written to using a connection. This is quite abstract
* and it can be used both for file operations and for
* network operations.
*/
class Connection
{
public:
Connection() { }
virtual ~Connection() { }
/*
* This method writes the current contents of the data buffer
* to the connected resource. Be sure to set the right data
* in the buffer by calling the setData method first.
*
* The number of bytes written is returned.
*/
virtual std::size_t write(const DataBuffer& data) = 0;
/*
* This method reads data from the connected resource and stores
* it in our data buffer which we pass in by reference.
* Please note that it clears whatever data was in the buffer prior to
* reading.
*
* The number of bytes read is returned.
*/
virtual std::size_t read(DataBuffer& data) = 0;
virtual const std::string str() const = 0;
};
inline std::vector<unsigned char> convert_data_to_unsigned(const DataBuffer& data)
{
return std::vector<unsigned char>(data.begin(), data.end());
}
inline std::string dataBufferToStr(const DataBuffer& data)
{
return std::string(data.begin(), data.end());
}
}
#endif /* CONNECTION_H_ */
UdpConnection.h
/*
* UdpConnection.h
*
* Created on: 2011-02-25
* Author: cdunphy
*/
// Portions Copyright (c) 2003-2011 Christopher M. Kohlhoff (chris at kohlhoff dot com)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#ifndef DATAGRAMCONNECTION_H_
#define DATAGRAMCONNECTION_H_
#include "AsioConnection.h"
#include <boost/lexical_cast.hpp>
namespace shaw_rsc
{
struct UdpException: public std::runtime_error
{
UdpException(const std::string& msg) : std::runtime_error(msg) { }
};
/*
* This is the concrete class that manages UDP connections.
*/
class UdpConnection: public AsioConnection
{
public:
/*
* Use this constructor for clients (connecting to a remote socket).
*/
UdpConnection(
const std::string& _host,
const std::string& _port,
int r_timeout,
Mode mode
) : AsioConnection(0, r_timeout),
socket(strand.get_io_service()),
remote_endpoint(),
host(_host),
port(_port)
{
check_deadline();
connect(mode);
}
std::size_t write(const DataBuffer& data);
std::size_t read(DataBuffer& data);
const std::string str() const;
private:
void connect(Mode mode);
void check_deadline();
boost::asio::ip::udp::socket socket;
boost::asio::ip::udp::endpoint remote_endpoint;
std::string host;
std::string port;
};
}
#endif /* DATAGRAMCONNECTION_H_ */
UdpConnection.cpp
/*
* UdpConnection.cpp
*
* Created on: 2011-02-25
* Author: cdunphy
*/
#include "UdpConnection.h"
using std::string;
using std::endl;
using std::stringstream;
using std::exception;
using boost::asio::buffer;
using boost::asio::ip::udp;
using boost::system::error_code;
using boost::system::system_error;
using boost::asio::deadline_timer;
using boost::bind;
using boost::lexical_cast;
namespace shaw_rsc
{
size_t UdpConnection::write(const DataBuffer& data)
{
size_t bytes_written = 0;
/*
* Check to see if the socket is bad before writing
*/
if (error &&
error.value() != boost::asio::error::operation_aborted &&
error.value() != boost::asio::error::timed_out &&
error != boost::asio::error::try_again)
throw UdpException(error.message());
socket.async_send_to(buffer(data), remote_endpoint,
strand.wrap(bind(&AsioConnection::handle_send, this, _1, _2,
&error, &bytes_written)));
do
{
strand.get_io_service().run_one();
}
while (error == boost::asio::error::would_block
|| error == boost::asio::error::try_again || bytes_written == 0);
if (error)
{
if (error.value() == boost::asio::error::operation_aborted
|| error.value() == boost::asio::error::timed_out)
throw SocketTimeoutException(error.message());
else
throw UdpException(error.message());
}
reset_deadline();
return bytes_written;
}
size_t UdpConnection::read(DataBuffer& data)
{
/*
* Check to see if the socket is bad before writing
*/
if (error &&
error.value() != boost::asio::error::operation_aborted &&
error.value() != boost::asio::error::timed_out &&
error != boost::asio::error::try_again)
throw UdpException(error.message());
data.clear();
/*
* Reset the deadline timer to expire according
* to the configured read timeout.
*/
deadline.expires_from_now(read_timeout);
size_t bytes_read = 0;
boost::array<char, BUF_SIZE> buff;
error = boost::asio::error::would_block;
socket.async_receive_from(buffer(buff), remote_endpoint,
strand.wrap(boost::bind(&AsioConnection::handle_receive, this, _1, _2, &error,
&bytes_read)));
do
{
strand.get_io_service().run_one();
}
while (error == boost::asio::error::would_block ||
error == boost::asio::error::try_again || bytes_read == 0);
/*
* Check for errors after the read.
*/
if (error)
{
if (error.value() == boost::asio::error::operation_aborted
|| error.value() == boost::asio::error::timed_out)
throw SocketTimeoutException(error.message());
else
throw UdpException(error.message());
}
else
data.insert(data.end(), buff.begin(), buff.begin() + bytes_read);
// Reset the deadline timer so we can leave this socket open as long
// as we want.
reset_deadline();
return bytes_read;
}
void UdpConnection::connect(Mode mode)
{
socket.open(boost::asio::ip::udp::v4());
if (mode == SERVER)
{
socket.bind(
udp::endpoint(udp::v4(),
lexical_cast<int>(port)), error);
}
else if (mode == CLIENT)
{
udp::resolver resolver(strand.get_io_service());
udp::resolver::query query(udp::v4(), host, port);
remote_endpoint = *resolver.resolve(query, error);
}
}
void UdpConnection::check_deadline()
{
// Check whether the deadline has passed. We compare the deadline against
// the current time since a new asynchronous operation may have moved the
// deadline before this actor had a chance to run.
if (deadline.expires_at() <= deadline_timer::traits_type::now())
{
// The deadline has passed. The outstanding asynchronous operation needs
// to be cancelled so that the blocked receive() function will return.
//
// Please note that cancel() has portability issues on some versions of
// Microsoft Windows, and it may be necessary to use close() instead.
// Consult the documentation for cancel() for further information.
socket.cancel();
// There is no longer an active deadline. The expiry is set to positive
// infinity so that the actor takes no action until a new deadline is set.
reset_deadline();
}
// Put the actor back to sleep.
deadline.async_wait(strand.wrap(boost::bind(&UdpConnection::check_deadline, this)));
}
/*
* This member function is good for diagnostic purposes
*/
const string UdpConnection::str() const
{
stringstream sstr;
sstr << "Host: " << host << endl;
sstr << "Port: " << port << endl;
sstr << "Read timeout: " << read_timeout_int << endl;
sstr << "Remote Endpoint Address: " << remote_endpoint.address().to_string()
<< endl;
sstr << "Remote Endpoint Port: " << remote_endpoint.port() << endl;
try
{
sstr << "Socket Remote Endpoint Address: "
<< socket.remote_endpoint().address().to_string() << endl;
sstr << "Socket Remote Endpoint Port: "
<< socket.remote_endpoint().port() << endl;
}
catch (exception& e)
{ }
try
{
sstr << "Socket Local Endpoint Address: "
<< socket.local_endpoint().address().to_string() << endl;
sstr << "Socket Local Endpoint Port: " << socket.local_endpoint().port()
<< endl;
}
catch (exception& e)
{ }
return sstr.str();
}
}
EDIT2:
Here is the test code I am trying to get working:
Server that replies in C++. All tests are working EXCEPT the threaded Udp test:
#define BOOST_TEST_DYN_LINK
#define BOOST_TEST_MODULE TestLibRSCAsio
#include <cstdio>
#include <cstdlib>
#include <cstdio>
#include <ctime>
#include <iostream>
#include <string>
#include <exception>
#include <sstream>
#include <boost/asio.hpp>
#include <boost/array.hpp>
#include <boost/date_time.hpp>
#include <boost/thread.hpp>
#include <boost/lexical_cast.hpp>
#include "boost/date_time/posix_time/posix_time.hpp"
#include <rsc_asio/Connection.h>
#include <rsc_asio/TcpConnection.h>
#include <rsc_asio/UdpConnection.h>
#include <rsc_asio/UnixSocketConnection.h>
#include "Util.h"
#include "sha1/Sha1Calc.h"
#include "servers/TestTcpServer.h"
#include <boost/test/unit_test.hpp>
using std::vector;
using std::string;
using std::size_t;
using std::cerr;
using std::cout;
using std::endl;
using std::flush;
using std::exception;
using std::time;
using std::stringstream;
using boost::lexical_cast;
using boost::thread;
using boost::mutex;
using boost::unique_lock;
using namespace shaw_rsc;
const size_t TCP_BYTE_SZ = 1000000;
const size_t UDP_BYTE_SZ = 64;
const std::string FILE_SOCKET = "/tmp/rofl";
const std::string SERVER_HOST = "0.0.0.0";
const std::string SERVER_PORT = "10999";
const std::string EXPECTED_UDP_REQUEST = "GOT_ANSWER?";
const int TIMEOUT = 3;
const int THREAD_TIMEOUT = 10;
DataBuffer write_data(ConnPtr client, Sha1Calc& sha1calc, size_t size, size_t iter)
{
unique_lock<mutex>(global_mutex);
cout << "Iter: " << iter << endl;
DataBuffer data = getRandomData(size);
sha1calc.calc_client_digest(data);
size_t bytes_written = client->write(data);
cout << "Wrote " << bytes_written << " -> " << dataBufferToStr(data) << " to socket: " << endl << client->str() << endl;
return data;
}
void write_data_threaded(ConnPtr client, Sha1Calc& sha1calc, size_t size, size_t iter)
{
cout << "Iter: " << iter << endl;
DataBuffer data = getRandomData(size);
sha1calc.calc_client_digest(data);
size_t bytes_written = client->write(data);
cout << "Wrote " << bytes_written << " -> " << dataBufferToStr(data) << " to socket: " << endl << client->str() << endl;
}
DataBuffer read_data(ConnPtr server, Sha1Calc& sha1calc, size_t iter)
{
cout << "Iter: " << iter << endl;
DataBuffer data;
size_t bytes_read = server->read(data);
cout << "Read " << bytes_read << " -> " << dataBufferToStr(data) << " from socket: " << endl << server->str() << endl;
sha1calc.calc_server_digest(data);
return data;
}
/*
* This is a suite of tests to provide unit tests
* for the RRE.
*/
BOOST_AUTO_TEST_SUITE(TestLibRSCAsioSuite)
BOOST_AUTO_TEST_CASE(TcpTest)
{
boost::asio::io_service io_service;
cout << endl << "**** TCP Test ****" << endl;
Sha1Calc sha1calc;
cout << endl << "Generating " << TCP_BYTE_SZ << " bytes of data to serve up." << endl;
DataBuffer dataToServe = getRandomData(TCP_BYTE_SZ);
sha1calc.calc_server_digest(dataToServe);
cout << "SHA1 hash of server data: " <<
sha1_to_str(sha1calc.get_server_digest()) << endl;
SrvPtr server(new TestTcpServer(std::atoi(SERVER_PORT.c_str()), dataToServe, io_service));
server->start();
try
{
// Fire up a basic TCP client for testing
cout << "Firing up TCP client on port: " << SERVER_PORT << endl;
DataBuffer clientData;
ConnPtr client(new TcpConnection(SERVER_HOST, SERVER_PORT, TIMEOUT, TIMEOUT, io_service));
size_t bytesRead = client->read(clientData);
BOOST_REQUIRE( bytesRead == TCP_BYTE_SZ );
BOOST_REQUIRE( clientData.size() == TCP_BYTE_SZ );
sha1calc.calc_client_digest(clientData);
BOOST_REQUIRE( sha1calc.compare() );// SHA-1 hashes better matctype filter texth
}
catch (SocketTimeoutException& e)
{
cerr << "Socket timeout: " << e.what() << endl;
BOOST_FAIL("Socket Timeout");
}
catch (const TcpException& e)
{
cerr << "TCP Error: " << e.what() << endl;
BOOST_FAIL("TCP Exception");
}
catch (const exception& e)
{
cerr << "Other Error: " << e.what() << endl;
BOOST_FAIL("Unknown Exception");
}
}
BOOST_AUTO_TEST_CASE(UdpTest)
{
boost::asio::io_service io_service;
std::stringstream error;
try
{
cout << endl << "**** UDP Test ****" << endl;
ConnPtr client(new UdpConnection(SERVER_HOST, SERVER_PORT, TIMEOUT, CLIENT, io_service));
ConnPtr server(new UdpConnection(SERVER_HOST, SERVER_PORT, TIMEOUT, SERVER, io_service));
for (int i = 0; i != 10; ++i)
{
Sha1Calc sha1calc;
// Write the data to the client
DataBuffer clientData = write_data(client, sha1calc, UDP_BYTE_SZ, i);
// Read the data from the server
DataBuffer serverData = read_data(server, sha1calc, i);
// Make sure the client data matches the server data
BOOST_REQUIRE( sha1calc.compare() );
cout << endl; // new-line
}
}
catch (const SocketTimeoutException& e)
{
error << "Socket timeout: " << e.what() << endl;
BOOST_FAIL(error.str());
}
catch (const UdpException& e)
{
error << "UDP Exception: " << e.what() << endl;
BOOST_FAIL(error.str());
}
catch (const exception& e)
{
error << "Other Error: " << e.what() << endl;
BOOST_FAIL(error.str());
}
}
BOOST_AUTO_TEST_CASE(UdpThreadTest)
{
boost::asio::io_service io_service;
std::stringstream error;
try
{
cout << endl << "**** UDP Multi-thread Test ****" << endl;
ConnPtr server(new UdpConnection(SERVER_HOST, SERVER_PORT, THREAD_TIMEOUT, SERVER, io_service));
Sha1Calc sha1calc;
for (int i = 0; i != 10; ++i)
{
// Read the data from the server, make sure it matches
// the expected request?
DataBuffer serverData = read_data(server, sha1calc, i);
BOOST_REQUIRE(dataBufferToStr(serverData) == EXPECTED_UDP_REQUEST);
// Repply on the remote socket
thread t1(bind(&write_data_threaded, server, sha1calc, UDP_BYTE_SZ, i));
cout << endl; // new-line
}
}
catch (const SocketTimeoutException& e)
{
error << "Socket timeout: " << e.what() << endl;
BOOST_FAIL(error.str());
}
catch (const UdpException& e)
{
error << "UDP Exception: " << e.what() << endl;
BOOST_FAIL(error.str());
}
catch (const exception& e)
{
error << "Other Error: " << e.what() << endl;
BOOST_FAIL(error.str());
}
}
BOOST_AUTO_TEST_CASE(UnixSocketTest)
{
boost::asio::io_service io_service;
try
{
cout << endl << "**** UNIX Socket Test ****" << endl;
std::remove(FILE_SOCKET.c_str());
ConnPtr server(new UnixSocketConnection(FILE_SOCKET, TIMEOUT, SERVER, io_service));
ConnPtr client(new UnixSocketConnection(FILE_SOCKET, TIMEOUT, CLIENT, io_service));
Sha1Calc sha1calc;
DataBuffer clientData = write_data(client, sha1calc, UDP_BYTE_SZ, 0);
cout << "Wrote the data to the Unix Socket client:" << dataBufferToStr(clientData) << endl;
DataBuffer serverData = read_data(server, sha1calc, 0);
cout << "Read from UDP Server: " << dataBufferToStr(serverData) << endl;
BOOST_REQUIRE( sha1calc.compare() );
cout << sha1_to_str(sha1calc.get_server_digest()) << endl;
}
catch (const SocketTimeoutException& e)
{
cerr << "Socket timeout: " << e.what() << endl;
BOOST_FAIL("Socket Timeout");
}
catch (const UnixSocketException& e)
{
cerr << "UNIX Socket Error: " << e.what() << endl;
BOOST_FAIL("UNIXSocket Exception");
}
catch (const exception& e)
{
cerr << "Other Error: " << e.what() << endl;
BOOST_FAIL("Unknown Exception");
}
std::remove(FILE_SOCKET.c_str());
}
BOOST_AUTO_TEST_SUITE_END()
}
Client written in Java:
package com.shaw.udp.sender;
import java.net.*;
import java.nio.ByteBuffer;
import java.nio.channels.DatagramChannel;
public class Client {
public static void main(String[] args) throws Exception {
DatagramChannel channel = DatagramChannel.open();
SocketAddress address = new InetSocketAddress(0);
SocketAddress client = new InetSocketAddress(SERVER_HOST, 10999);
DatagramSocket socket = channel.socket();
// This is the local socket
socket.setSoTimeout(5000);
socket.bind(address);
for (int i = 0; i != 10; ++i) {
// Send the data to the remote server
ByteBuffer buffer = ByteBuffer.wrap("GOT_ANSWER?".getBytes());
channel.send(buffer, client);
System.out.println("Iter: " + i + " => Sent request: "
+ new String(buffer.array()));
// Listen for reply from the server
buffer = ByteBuffer.allocate(64);
channel.receive(buffer);
System.out.println("Iter: " + i + " => Got reply: "
+ new String(buffer.array()));
}
}
}