How to construct a boost::beast::http::message from std::string? - c++

Is that possible to construct a boost::beast::http::message (specifically I have to construct a boost::beast::http::response<bb_http::string_body>) from std::string, std::string_view or other raw buffer?
Maybe there is some kind of parser? From what I see in Boost.Beast samples, we can either:
receive a response from boost::beast::read* functions. In that case the first argument should be a SyncReadStream, which has to comply with contracts of SyncReadStream from boost/beast/core/type_traits.hpp:
struct is_sync_read_stream<T, detail::void_t<decltype(
std::declval<std::size_t&>() = std::declval<T>().read_some(
std::declval<detail::MutableBufferSequence>()),
std::declval<std::size_t&>() = std::declval<T>().read_some(
std::declval<detail::MutableBufferSequence>(),
std::declval<boost::system::error_code&>()),
(void)0)>> : std::true_type {};
or construct it by hand like http::request<http::string_body> req{http::verb::get, target, version};

You can manually invoke the parser, e.g. with this simple skeleton function:
http::response<http::string_body> do_parse(std::string_view input)
{
beast::error_code ec;
http::response_parser<http::string_body> p;
// read headers
auto buf = boost::asio::buffer(sample);
auto n = p.put(buf, ec);
assert(p.is_header_done());
// read body
if (!ec) {
buf += n;
n = p.put(buf, ec);
p.put_eof(ec);
}
if (ec)
throw boost::system::system_error(ec);
assert(p.is_done());
return p.release();
}
This assumes that input is a sinlge complete request.
Live Demo
Live On Coliru
#include <boost/beast.hpp>
#include <boost/beast/http.hpp>
#include <string_view>
#include <iostream>
#include <iomanip>
namespace beast = boost::beast;
namespace http = beast::http;
http::response<http::string_body> do_parse(std::string_view input)
{
beast::error_code ec;
http::response_parser<http::string_body> p;
// read headers
auto buf = boost::asio::buffer(input);
auto n = p.put(buf, ec);
assert(p.is_header_done());
// read body
if (!ec) {
buf += n;
n = p.put(buf, ec);
p.put_eof(ec);
}
if (ec)
throw boost::system::system_error(ec);
assert(p.is_done());
return p.release();
}
int main() {
auto res = do_parse(
"HTTP/1.1 200 OK\r\n"
"Date: Sun, 10 Oct 2010 23:26:07 GMT\r\n"
"Server: Apache/2.2.8 (Ubuntu) mod_ssl/2.2.8 OpenSSL/0.9.8g\r\n"
"Last-Modified: Sun, 26 Sep 2010 22:04:35 GMT\r\n"
"ETag: 45b6-834-49130cc1182c0\r\n"
"Accept-Ranges: bytes\r\n"
"Content-Length: 12\r\n"
"Connection: close\r\n"
"Content-Type: text/html\r\n"
"\r\n"
"Hello world!");
std::cout << res << '\n';
std::cout << "====== body:\n" << std::quoted(res.body()) << "\n";
}
Prints
HTTP/1.1 200 OK
Date: Sun, 10 Oct 2010 23:26:07 GMT
Server: Apache/2.2.8 (Ubuntu) mod_ssl/2.2.8 OpenSSL/0.9.8g
Last-Modified: Sun, 26 Sep 2010 22:04:35 GMT
ETag: 45b6-834-49130cc1182c0
Accept-Ranges: bytes
Content-Length: 12
Connection: close
Content-Type: text/html
Hello world!
====== body:
"Hello world!"

Turns out there already has been a sample snippet for reading a boost::beast::message from std::istream.
https://www.boost.org/doc/libs/1_66_0/libs/beast/doc/html/beast/using_http/buffer_oriented_parsing.html
/** Read a message from a `std::istream`.
This function attempts to parse a complete HTTP/1 message from the stream.
#param is The `std::istream` to read from.
#param buffer The buffer to use.
#param msg The message to store the result.
#param ec Set to the error, if any occurred.
*/
template<
class Allocator,
bool isRequest,
class Body>
void
read_istream(
std::istream& is,
basic_flat_buffer<Allocator>& buffer,
message<isRequest, Body, fields>& msg,
error_code& ec)
{
// Create the message parser
//
// Arguments passed to the parser's constructor are
// forwarded to the message constructor. Here, we use
// a move construction in case the caller has constructed
// their message in a non-default way.
//
parser<isRequest, Body> p{std::move(msg)};
do
{
// Extract whatever characters are presently available in the istream
if(is.rdbuf()->in_avail() > 0)
{
// Get a mutable buffer sequence for writing
auto const b = buffer.prepare(
static_cast<std::size_t>(is.rdbuf()->in_avail()));
// Now get everything we can from the istream
buffer.commit(static_cast<std::size_t>(is.readsome(
reinterpret_cast<char*>(b.data()), b.size())));
}
else if(buffer.size() == 0)
{
// Our buffer is empty and we need more characters,
// see if we've reached the end of file on the istream
if(! is.eof())
{
// Get a mutable buffer sequence for writing
auto const b = buffer.prepare(1024);
// Try to get more from the istream. This might block.
is.read(reinterpret_cast<char*>(b.data()), b.size());
// If an error occurs on the istream then return it to the caller.
if(is.fail() && ! is.eof())
{
// We'll just re-use io_error since std::istream has no error_code interface.
ec = make_error_code(errc::io_error);
return;
}
// Commit the characters we got to the buffer.
buffer.commit(static_cast<std::size_t>(is.gcount()));
}
else
{
// Inform the parser that we've reached the end of the istream.
p.put_eof(ec);
if(ec)
return;
break;
}
}
// Write the data to the parser
auto const bytes_used = p.put(buffer.data(), ec);
// This error means that the parser needs additional octets.
if(ec == error::need_more)
ec = {};
if(ec)
return;
// Consume the buffer octets that were actually parsed.
buffer.consume(bytes_used);
}
while(! p.is_done());
// Transfer ownership of the message container in the parser to the caller.
msg = p.release();
}

Related

Boost.beast : how to return json response only

code : https://gist.github.com/Naseefabu/173c928603e564879683ccdf10d9d0f8
When i run this and print the response to the console:
Sending GET /api/v3/time HTTP/1.1
Host: api.binance.com
content-type: application/json
User-Agent: Boost.Beast/330
HTTP/1.1 200 OK
Content-Type: application/json;charset=UTF-8
Content-Length: 28
Connection: keep-alive
Date: Fri, 13 May 2022 17:18:47 GMT
Server: nginx
x-mbx-uuid: 7e7465db-011a-4308-aa1e-6603d72c8c9a
x-mbx-used-weight: 1
x-mbx-used-weight-1m: 1
Strict-Transport-Security: max-age=31536000; includeSubdomains
X-Frame-Options: SAMEORIGIN
X-Xss-Protection: 1; mode=block
X-Content-Type-Options: nosniff
Content-Security-Policy: default-src 'self'
X-Content-Security-Policy: default-src 'self'
X-WebKit-CSP: default-src 'self'
Cache-Control: no-cache, no-store, must-revalidate
Pragma: no-cache
Expires: 0
Access-Control-Allow-Origin: *
Access-Control-Allow-Methods: GET, HEAD, OPTIONS
X-Cache: Miss from cloudfront
Via: 1.1 dfccb338f8c0489ab09835ea7dbad1a8.cloudfront.net (CloudFront)
X-Amz-Cf-Pop: MAA51-P1
X-Amz-Cf-Id: waTqtDHSDpk74QB7zkF5Ya0CdRVWuJuC-M4TZqSuMd2bfXawkq6o6g==
{"serverTime":1652462327804}
shutdown: stream truncated
what if i just want to get the json response : {"serverTime":1652462327804} ?
and store in the json variable so i could use it for my needs, other informations is not that important for me, advance thanks!
binapi::AsyncRest::httpClient* client;
That's extremely suspect, since the class is using enable_shared_from_this(). Pretty sure that should be
auto client =
std::make_shared<binapi::AsyncRest::httpClient>(ioc.get_executor(), ctx);
Next up, I assume get_server_time is a static function. I don't see why it is a member of httpClient.
So adding all the missing code back in (using lots of experience):
#include <boost/asio.hpp>
#include <boost/asio/ssl.hpp>
#include <boost/beast.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/url.hpp>
#include <boost/url/src.hpp> // for header-only
#include <fstream>
#include <iomanip>
#include <iostream>
namespace net = boost::asio;
namespace beast = boost::beast;
namespace http = boost::beast::http;
namespace ssl = boost::asio::ssl;
using net::ip::tcp;
namespace binapi { namespace AsyncRest {
// Report a failure
void fail_http(beast::error_code ec, char const* what) {
std::cerr << what << ": " << ec.message() << "\n";
}
struct httpClient : std::enable_shared_from_this<httpClient> {
using executor = net::any_io_executor;
using Stream = beast::ssl_stream<beast::tcp_stream>;
tcp::resolver resolver_;
Stream stream_;
beast::flat_buffer buffer_;
http::request<http::empty_body> req_;
http::response<http::string_body> res_;
httpClient(executor ex, ssl::context& ctx);
// Start the asynchronous operation
void run(boost::url, http::verb);
void on_resolve(beast::error_code, tcp::resolver::results_type);
void on_connect(beast::error_code, tcp::resolver::results_type::endpoint_type);
void on_handshake(beast::error_code);
void on_write(beast::error_code, size_t bytes_transferred);
void on_read(beast::error_code, size_t bytes_transferred);
void on_shutdown(beast::error_code);
};
httpClient::httpClient(executor ex, ssl::context& ctx)
: resolver_(ex)
, stream_(ex, ctx) {}
// Start the asynchronous operation
void httpClient::run(boost::url url, http::verb action) {
std::string const host(url.host());
std::string const service = url.has_port() //
? url.port()
: (url.scheme_id() == boost::urls::scheme::https) //
? "https"
: "http";
url.remove_origin(); // becomes req_.target()
// Set SNI Hostname (many hosts need this to handshake
// successfully)
if (!SSL_set_tlsext_host_name(stream_.native_handle(), host.c_str())) {
beast::error_code ec{static_cast<int>(::ERR_get_error()),
net::error::get_ssl_category()};
std::cerr << ec.message() << "\n";
return;
}
// Set up an HTTP GET/POST/DELETE/PUT request message
// req_.version(version);
req_.method(action);
req_.target(url.c_str());
req_.set(http::field::host, host);
req_.set(http::field::user_agent, BOOST_BEAST_VERSION_STRING);
req_.prepare_payload(); // make HTTP 1.1 compliant
// Look up the domain name
resolver_.async_resolve(
host, service,
beast::bind_front_handler(&httpClient::on_resolve, shared_from_this()));
}
void httpClient::on_resolve(beast::error_code ec,
tcp::resolver::results_type results) {
if (ec)
return fail_http(ec, "resolve");
// Set a timeout on the operation
beast::get_lowest_layer(stream_).expires_after(std::chrono::seconds(30));
// Make the connection on the IP address we get from a lookup
beast::get_lowest_layer(stream_).async_connect(
results,
beast::bind_front_handler(&httpClient::on_connect, shared_from_this()));
}
void httpClient::on_connect(beast::error_code ec,
tcp::resolver::results_type::endpoint_type) {
if (ec)
return fail_http(ec, "connect");
// Perform the SSL handshake
stream_.async_handshake(
ssl::stream_base::client,
beast::bind_front_handler(&httpClient::on_handshake, shared_from_this()));
}
void httpClient::on_handshake(beast::error_code ec) {
if (ec)
return fail_http(ec, "handshake");
// Set a timeout on the operation
beast::get_lowest_layer(stream_).expires_after(std::chrono::seconds(30));
// Send the HTTP request to the remote host
std::cout << "Sending " << req_ << std::endl;
http::async_write(
stream_, req_,
beast::bind_front_handler(&httpClient::on_write, shared_from_this()));
}
void httpClient::on_write(beast::error_code ec, size_t bytes_transferred) {
boost::ignore_unused(bytes_transferred);
if (ec)
return fail_http(ec, "write");
// Receive the HTTP response
http::async_read(
stream_, buffer_, res_,
beast::bind_front_handler(&httpClient::on_read, shared_from_this()));
}
void httpClient::on_read(beast::error_code ec, size_t bytes_transferred) {
boost::ignore_unused(bytes_transferred);
if (ec)
return fail_http(ec, "read");
// Write the message to standard out
std::cout << res_ << std::endl;
// Set a timeout on the operation
beast::get_lowest_layer(stream_).expires_after(std::chrono::seconds(30));
// Gracefully close the stream
stream_.async_shutdown(
beast::bind_front_handler(&httpClient::on_shutdown, shared_from_this()));
}
void httpClient::on_shutdown(beast::error_code ec) {
if (ec == net::error::eof) {
ec = {};
}
if (ec)
return fail_http(ec, "shutdown");
}
static void get_server_time(net::io_context& ioc, ssl::context& ctx) {
static boost::url_view const uri{"https://api.binance.com/api/v3/time"};
std::make_shared<httpClient>(net::make_strand(ioc), ctx)
->run(uri, http::verb::get);
}
}} // namespace binapi::AsyncRest
int main() {
net::io_context ioc;
// The SSL context is required, and holds certificates
ssl::context ctx{ssl::context::tlsv12_client};
// Verify the remote server's certificate
ctx.set_verify_mode(ssl::verify_peer);
ctx.set_default_verify_paths();
binapi::AsyncRest::get_server_time(ioc, ctx);
ioc.run();
}
Now we know that res_ is beast::http::response<beast::http::string_body>. So, if you only want to print the body, print that:
std::cout << res_.body() << std::endl;
Prints
Sending GET /api/v3/time HTTP/1.1
Host: api.binance.com
User-Agent: Boost.Beast/330
{"serverTime":1652476115413}
shutdown: stream truncated
To only print the time:
static constexpr boost::gregorian::date s_epoch{1970, 1, 1};
auto epoch_seconds = json::parse(res_.body()).at("serverTime").as_int64();
ptime serverTime(s_epoch, boost::posix_time::milliseconds(epoch_seconds));
std::cout << serverTime << std::endl;
Prints
2022-May-13 21:38:55.982000
Summary/Notes
What it really looks like you're after is a function like
ptime server_time();
Or
void async_server_time(auto completionToken);
And you'd probably want to share the client class instance instead of reconnecting for each call.

Handling large http response using boost::beast

The following code use to get http response message:
boost::beast::tcp_stream stream_;
boost::beast::flat_buffer buffer;
boost::beast::http::response<boost::beast::http::dynamic_body> res;
boost::beast::http::read(stream_, buffer, res);
However, In some cases, based on the preceding request, I can expect that the response message body will include large binary file.
Therefore, I’d like to read it directly to the filesystem and not through buffer variable to avoid excessive use of process memory. How can it be done ?
in Objective-c framework NSUrlSession there's an easy way to do it using NSURLSessionDownloadTask instead of NSURLSessionDataTask, so I wonder if it's also exist in boost.
Thanks !
In general, you can use the http::buffer_body to handle arbitrarily large request/response messages.
If you specifically want to read/write from a filesystem file, you can have the http::file_body instead.
Full Demo buffer_body
The documentation sample for buffer_body is here https://www.boost.org/doc/libs/1_77_0/libs/beast/doc/html/beast/using_http/parser_stream_operations/incremental_read.html.
Using it to write to std::cout: Live On Coliru
#include <boost/asio.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/beast.hpp>
#include <boost/beast/websocket.hpp>
#include <iostream>
namespace net = boost::asio;
namespace beast = boost::beast;
namespace http = beast::http;
using tcp = net::ip::tcp;
using socket_t = tcp::socket;
/* This function reads a message using a fixed size buffer to hold
portions of the body, and prints the body contents to a `std::ostream`.
*/
template<
bool isRequest,
class SyncReadStream,
class DynamicBuffer>
void
read_and_print_body(
std::ostream& os,
SyncReadStream& stream,
DynamicBuffer& buffer,
beast::error_code& ec)
{
http::parser<isRequest, http::buffer_body> p;
http::read_header(stream, buffer, p, ec);
if(ec)
return;
while(! p.is_done())
{
char buf[512];
p.get().body().data = buf;
p.get().body().size = sizeof(buf);
http::read(stream, buffer, p, ec);
if(ec == http::error::need_buffer)
ec = {};
if(ec)
return;
os.write(buf, sizeof(buf) - p.get().body().size);
}
}
int main() {
std::string host = "173.203.57.63"; // COLIRU 20210901
auto const port = "80";
net::io_context ioc;
tcp::resolver resolver{ioc};
socket_t s{ioc};
net::connect(s, resolver.resolve(host, port));
write(s, http::request<http::empty_body>{http::verb::get, "/", 11});
beast::error_code ec;
beast::flat_buffer buf;
read_and_print_body<false>(std::cout, s, buf, ec);
}
Full file_body example
This is much shorter, writing to body.html:
Live On Coliru
#include <boost/asio.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/beast.hpp>
#include <boost/beast/websocket.hpp>
#include <iostream>
namespace net = boost::asio;
namespace beast = boost::beast;
namespace http = beast::http;
using tcp = net::ip::tcp;
using socket_t = tcp::socket;
int main() {
std::string host = "173.203.57.63"; // COLIRU 20210901
auto const port = "80";
net::io_context ioc;
tcp::resolver resolver{ioc};
socket_t s{ioc};
net::connect(s, resolver.resolve(host, port));
write(s, http::request<http::empty_body>{http::verb::get, "/", 11});
beast::error_code ec;
beast::flat_buffer buf;
http::response<http::file_body> res;
res.body().open("body.html", beast::file_mode::write_new, ec);
if (!ec.failed())
{
read(s, buf, res, ec);
}
std::cout << "Wrote 'body.html' (" << ec.message() << ")\n";
std::cout << "Headers " << res.base() << "\n";
}
Prints
Wrote 'body.html' (Success)
Headers HTTP/1.1 200 OK
Content-Type: text/html;charset=utf-8
Content-Length: 8616
Server: WEBrick/1.4.2 (Ruby/2.5.1/2018-03-29) OpenSSL/1.0.2g
Date: Wed, 01 Sep 2021 19:52:20 GMT
Connection: Keep-Alive
With file body.html; wc body.html showing:
body.html: HTML document, ASCII text, with very long lines
185 644 8616 body.html
Beyond: streaming to child processes and streaming processing
I have an advanced example of that here: How to read data from Internet using muli-threading with connecting only once?.

How to use boost::beast, download a file no blocking and with responses

I have started with this example so won't post all the code. My objective is to download a large file without blocking my main thread. The second objective is to get notifications so I can update a progress bar. I do have the code working a couple of ways. First is to just ioc.run(); and let it go to work, I get the file downloaded. But I can not find anyway to start the session without blocking.
The second way I can make the calls down to http::async_read_some and the call works but I can not get a response that I can use. I don't know if there is a way to pass a lambda that captures.
The #if 0..#else..#endif switches the methods. I'm sure there is a simple way but I just can not see it. I'll clean up the code when I get it working, like setting the local file name. Thanks.
std::size_t on_read_some(boost::system::error_code ec, std::size_t bytes_transferred)
{
if (ec);//deal with it...
if (!bValidConnection) {
std::string_view view((const char*)buffer_.data().data(), bytes_transferred);
auto pos = view.find("Content-Length:");
if (pos == std::string_view::npos)
;//error
file_size = std::stoi(view.substr(pos+sizeof("Content-Length:")).data());
if (!file_size)
;//error
bValidConnection = true;
}
else {
file_pos += bytes_transferred;
response_call(ec, file_pos);
}
#if 0
std::cout << "in on_read_some caller\n";
http::async_read_some(stream_, buffer_, file_parser_, std::bind(
response_call,
std::placeholders::_1,
std::placeholders::_2));
#else
std::cout << "in on_read_some inner\n";
http::async_read_some(stream_, buffer_, file_parser_, std::bind(
&session::on_read_some,
shared_from_this(),
std::placeholders::_1,
std::placeholders::_2));
#endif
return buffer_.size();
}
The main, messy but.....
struct lambda_type {
bool bDone = false;
void operator ()(const boost::system::error_code ec, std::size_t bytes_transferred) {
;
}
};
int main(int argc, char** argv)
{
auto const host = "reserveanalyst.com";
auto const port = "443";
auto const target = "/downloads/demo.msi";
int version = argc == 5 && !std::strcmp("1.0", argv[4]) ? 10 : 11;
boost::asio::io_context ioc;
ssl::context ctx{ ssl::context::sslv23_client };
load_root_certificates(ctx);
//ctx.load_verify_file("ca.pem");
auto so = std::make_shared<session>(ioc, ctx);
so->run(host, port, target, version);
bool bDone = false;
auto const lambda = [](const boost::system::error_code ec, std::size_t bytes_transferred) {
std::cout << "data lambda bytes: " << bytes_transferred << " er: " << ec.message() << std::endl;
};
lambda_type lambda2;
so->set_response_call(lambda);
ioc.run();
std::cout << "not in ioc.run()!!!!!!!!" << std::endl;
so->async_read_some(lambda);
//pseudo message pump when working.........
for (;;) {
std::this_thread::sleep_for(250ms);
std::cout << "time" << std::endl;
}
return EXIT_SUCCESS;
}
And stuff I've added to the class session
class session : public std::enable_shared_from_this<session>
{
using response_call_type = void(*)(boost::system::error_code ec, std::size_t bytes_transferred);
http::response_parser<http::file_body> file_parser_;
response_call_type response_call;
//
bool bValidConnection = false;
std::size_t file_pos = 0;
std::size_t file_size = 0;
public:
auto& get_result() { return res_; }
auto& get_buffer() { return buffer_; }
void set_response_call(response_call_type the_call) { response_call = the_call; }
I've updated this as I finally put it to use and I wanted the old method where I could download to a file or a string. Link to how asio works, great talk.
CppCon 2016 Michael Caisse Asynchronous IO with BoostAsio
As for my misunderstanding of how to pass a lambda, here is Adam Nevraumont's answer
There are two ways to compile this using a type to select the method. Both are shown at the beginning of main. You can construct either a file downloader or string downloader by selecting the type of beast parser. The parsers don't have the same constructs so an if constexpr compile time conditions are used. And I checked, a release build of the downloader is about 1K so pretty light weight for what it does. In the case of a small string you don't have to handle the call backs. either pass an empty lambda or add the likes of:
if(response_call)
response_call(resp_ok, test);
This looks to be a pretty clean way to get the job done so I've updated this post as of 11/27/2202.
The code:
//
// Copyright (c) 2016-2019 Vinnie Falco (vinnie dot falco at gmail dot com)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
// Official repository: https://github.com/boostorg/beast
//------------------------------------------------------------------------------
//
// Example: HTTP SSL client, synchronous, usable in a thread with a message pump
// Added code to use from a message pump
// Also useable as body to a file download, or body to string
//
//------------------------------------------------------------------------------
#include <boost/beast/core.hpp>
#include <boost/beast/http.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/beast/version.hpp>
#include <boost/asio/connect.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/ssl/error.hpp>
#include <boost/asio/ssl/stream.hpp>
#include <cstdlib>
#include <iostream>
#include <string>
#include <fstream>
//the boost shipped certificates
#include <boost/../libs/beast/example/common/root_certificates.hpp>
//TODO add your ssl libs as you would like
#ifdef _M_IX86
#pragma comment(lib, "libcrypto.lib")
#pragma comment(lib, "libssl.lib")
#elif _M_X64
#pragma comment(lib, "libcrypto-3-x64.lib")
#pragma comment(lib, "libssl-3-x64.lib")
#endif
namespace downloader {
namespace beast = boost::beast; // from <boost/beast.hpp>
namespace http = beast::http; // from <boost/beast/http.hpp>
namespace net = boost::asio; // from <boost/asio.hpp>
namespace ssl = net::ssl; // from <boost/asio/ssl.hpp>
using tcp = net::ip::tcp; // from <boost/asio/ip/tcp.hpp>
//specialization if using < c++17; see both 'if constexpr' below.
//this is not needed otherwise
//namespace detail {
// template<typename Type>
// void open_file(http::parser < false, Type>& p, const char* name, boost::system::error_code& file_open_ec) { }
// template<>
// void open_file(http::parser<false, http::file_body>& p, const char* name, boost::system::error_code& file_open_ec) {
// p.get().body().open(name, boost::beast::file_mode::write, file_open_ec);
// }
// template<typename Type>
// std::string get_string(http::parser < false, Type>& p) { return std::string{}; }
// template<>
// std::string get_string(http::parser<false, http::string_body>& p) {
// return p.get().body();
// }
//} //namespace detail
enum responses {
resp_null,
resp_ok,
resp_done,
resp_error,
};
using response_call_type = std::function< void(responses, std::size_t)>;
template<typename ParserType>
struct download {
//as these can be set with array initialization
const char* target_ = "/";
const char* filename_ = "test.txt";
const char* host_ = "lakeweb.net";
std::string body_;
using response_call_type = std::function< void(responses, std::size_t)>;
response_call_type response_call;
boost::asio::io_context ioc_;
ssl::context ctx_{ ssl::context::sslv23_client };
ssl::stream<tcp::socket> stream_{ ioc_, ctx_ };
tcp::resolver resolver_{ ioc_ };
boost::beast::flat_buffer buffer_;
uint64_t file_size_{};
int version{ 11 };
void set_response_call(response_call_type the_call) { response_call = the_call; }
uint64_t get_file_size() { return file_size_; }
void stop() { ioc_.stop(); }
bool stopped() { return ioc_.stopped(); }
std::string get_body() { return std::move(body_); }
void run() {
try {
// TODO should have a timer in case of a hang
load_root_certificates(ctx_);
// Set SNI Hostname (many hosts need this to handshake successfully)
if (!SSL_set_tlsext_host_name(stream_.native_handle(), host_)) {
boost::system::error_code ec{ static_cast<int>(::ERR_get_error()), boost::asio::error::get_ssl_category() };
throw boost::system::system_error{ ec };
}
//TODO resolve is depreciated, use endpoint
auto const results = resolver_.resolve(host_, "443");
boost::asio::connect(stream_.next_layer(), results.begin(), results.end());
stream_.handshake(ssl::stream_base::client);
// Set up an HTTP GET request message
http::request<http::string_body> req{ http::verb::get, target_, version };
req.set(http::field::host, host_);
req.set(http::field::user_agent, "mY aGENT");
// Send the HTTP request to the remote host
http::write(stream_, req);
// Read the header
boost::system::error_code file_open_ec;
http::parser<false, ParserType> p;
p.body_limit((std::numeric_limits<std::uint32_t>::max)());
//detail::open_file(p, filename_, file_open_ec);
//or => c++17
if constexpr (std::is_same_v<ParserType, http::file_body>)
p.get().body().open(filename_, boost::beast::file_mode::write, file_open_ec);
http::read_header(stream_, buffer_, p);
file_size_ = p.content_length().has_value() ? p.content_length().value() : 0;
//Read the body
uint64_t test{};
boost::system::error_code rec;
for (;;) {
test += http::read_some(stream_, buffer_, p, rec);
if (test >= file_size_) {
response_call(resp_done, 0);
break;
}
response_call(resp_ok, test);
}
// Gracefully close the stream
boost::system::error_code ec;
stream_.shutdown(ec);
if (ec == boost::asio::error::eof)
{
// Rationale:
// http://stackoverflow.com/questions/25587403/boost-asio-ssl-async-shutdown-always-finishes-with-an-error
ec.assign(0, ec.category());
}
if (ec)
throw boost::system::system_error{ ec };
//value = detail::get_string(p);
//or => c++17
if constexpr (std::is_same_v<ParserType, http::string_body>)
body_ = p.get().body();
}
catch (std::exception const& e)
{
std::cerr << "Error: " << e.what() << std::endl;
response_call(resp_error, -1);
}
ioc_.stop();
}
};
}//namespace downloadns
//comment to test with string body
#define THE_FILE_BODY_TEST
int main(int argc, char** argv)
{
using namespace downloader;
#ifdef THE_FILE_BODY_TEST
download<http::file_body> dl{"/Nasiri%20Abarbekouh_Mahdi.pdf", "test.pdf"};
#else //string body test
download<http::string_body> dl{ "/robots.txt" };
#endif
responses dl_response{ resp_null };
size_t cur_size{};
auto static const lambda = [&dl_response, &dl, &cur_size](responses response, std::size_t bytes_transferred) {
if ((dl_response = response) == resp_ok) {
cur_size += bytes_transferred;
size_t sizes = dl.get_file_size() - cur_size;//because size is what is left
//drive your progress bar from here in a GUI app
}
};
dl.set_response_call(lambda);
std::thread thread{ [&dl]() { dl.run(); } };
//thread has started, now the pseudo message pump
bool quit = false; //true: as if a cancel button was pushed; won't finish download
for (int i = 0; ; ++i) {
switch (dl_response) { //ad hoc as if messaged
case resp_ok:
std::cout << "from sendmessage: " << cur_size << std::endl;
dl_response = resp_null;
break;
case resp_done:
std::cout << "from sendmessage: done" << std::endl;
dl_response = resp_null;
break;
case resp_error:
std::cout << "from sendmessage: error" << std::endl;
dl_response = resp_null;
}//switch
if (!(i % 5))
std::cout << "in message pump, stopped: " << std::boolalpha << dl.stopped() << std::endl;
std::this_thread::sleep_for(std::chrono::milliseconds(100));
if (quit && i == 10) //the cancel message
dl.stop();
if (!(i % 20) && dl.stopped()) {//dl job was quit or error or finished
std::cout << "dl is stopped" << std::endl;
break;
}
}
#ifdef THE_FILE_BODY_TEST
std::cout << "file written named: 'test.txt'" << std::endl;
#else
std::string res = dl.get_body();
std::cout << "body retrieved:\n" << res << std::endl;
#endif
if (thread.joinable())//in the case a thread was never started
thread.join();
std::cout << "exiting, program all done" << std::endl;
return EXIT_SUCCESS;
}
I strongly recommend against using the low-level [async_]read_some function instead of using http::[async_]read as intended with http::response_parser<http::buffer_body>
I do have an example of that - which is a little bit complicated by the fact that it also uses Boost Process to concurrently decompress the body data, but regardless it should show you how to use it:
How to read data from Internet using muli-threading with connecting only once?
I guess I could tailor it to your specific example given more complete code, but perhaps the above is good enough? Also see "Relay an HTTP message" in libs/beast/example/doc/http_examples.hpp which I used as "inspiration".
Caution: the buffer arithmetic is not intuitive. I think this is unfortunate and should not have been necessary, so pay (very) close attention to these samples for exactly how that's done.

Sequential communication failed with boost::asio::streambuf in binary way?

Using boost::asio, I'm coding network stuff.
I tried to build a simple send-and-receive-string protocol.
The sender first send the string size to the receiver. Then the sender sends the actual string to the receiver.
In particular, I designed the following two protocols.
A sender holding a string sends it to a receiver. Upon receiving it, the receiver shows the string.
Execute above protocol sequentially (two times).
I built the above protocols as shown below:
If I execute this protocol once, that works fine.
However, if i execute this protocol more than once (e.g. two times), the
string size that the receiver receives gets wrong.
First time : 1365 bytes.
Second time : 779073 bytes. (just read not 779073 but 7790)
I found that os << data_size is not done in a binary way. "779073" is just sent as 6 bytes string. But the receiver just reads 4bytes of it.
How to send a binary data and to receive a binary data using boost::asio and boost::asio::streambuf?
Receiver
// socket is already defined
// ** first step: recv data size
boost::asio::streambuf buf;
boost::asio::read(
socket,
buf,
boost::asio::transfer_exactly(sizeof(uint32_t))
);
std::istream iss(&buf);
uint32_t read_len;
iss >> read_len;
// ** second step: recv payload based on the data size
boost::asio::streambuf buf2;
read_len = boost::asio::read(socket, buf2,
boost::asio::transfer_exactly(read_len), error);
cout << " read "<< read_len << " bytes payload" << endl;
std::istream is_payload(&buf2);
std::string str;
is_payload >> str;
cout << str << endl;
Sender
// socket is already defined
string str=...; // some string to be sent
// ** first step: tell the string size to the reciever
uint32_t data_size = str.size();
boost::asio::streambuf send_buf;
std::ostream os(&send_buf);
os << data_size;
size_t sent_byte = boost::asio::write(socket, send_buf.data());
cout << sent_byte << endl; // debug purpose
// ** second step: send the actual string (payload)
sent_byte = boost::asio::write(socket, boost::asio::buffer(reinterpret_cast<const char*>(&str[0]), data_size));
cout << sent_byte << endl; // debug purpose
You can send the size binary, but that requires you to take architectural differences between devices and operating systems into account¹.
Here's my take on actually coding the protocol reusably:
//#define BOOST_ASIO_ENABLE_HANDLER_TRACKING
#include <boost/asio.hpp>
#include <boost/endian/arithmetic.hpp>
namespace ba = boost::asio;
using ba::ip::tcp;
using error_code = boost::system::error_code;
namespace Protocol { // your library
using net_size_t = boost::endian::big_int32_t; // This protocol uses Big-endian network byte order
template <typename Derived, typename Token, typename Sig = void(error_code, size_t)>
struct base_async_op : std::enable_shared_from_this<Derived> {
using base_type = base_async_op<Derived, Token, Sig>;
template <typename DeducedToken>
base_async_op(DeducedToken &&token) : _token(std::forward<DeducedToken>(token)) {}
using _Token = std::decay_t<Token>;
using _Init = ba::async_completion<_Token, Sig>;
using _Handler = typename _Init::completion_handler_type;
_Token _token;
_Init _init {_token};
auto get_allocator() const noexcept {
return (boost::asio::get_associated_allocator)(_init.completion_handler);
}
using executor_type = ba::associated_executor_t<_Handler>;
executor_type get_executor() const noexcept {
return (boost::asio::get_associated_executor)(_init.completion_handler);
}
Derived& derived() { return static_cast<Derived&>(*this); }
Derived const& derived() const { return static_cast<Derived const&>(*this); }
template <typename F>
auto wrap(F&& f) const {
//std::cout << "WRAP: " << typeid(derived().get_executor()).name() << "\n";
return ba::bind_executor(derived().get_executor(), std::forward<F>(f));
}
};
template <typename Derived, typename Stream, typename Token, typename Sig = void(error_code, size_t)>
struct stream_async_op : base_async_op<Derived, Token, Sig> {
using base_type = stream_async_op<Derived, Stream, Token, Sig>;
template <typename DeducedToken>
stream_async_op(Stream& s, DeducedToken &&token) : base_async_op<Derived, Token, Sig>(std::forward<DeducedToken>(token)), _stream(s) {}
Stream& _stream;
using executor_type = ba::associated_executor_t<typename stream_async_op::_Handler, decltype(std::declval<Stream>().get_executor())>;
executor_type get_executor() const noexcept {
return (boost::asio::get_associated_executor)(this->_init.completion_handler, _stream.get_executor());
}
};
template <typename AsyncStream, typename Buffer, typename Token>
auto async_transmit(AsyncStream& s, Buffer message_buffer, Token&& token) {
struct op : stream_async_op<op, AsyncStream, Token> {
using op::base_type::base_type;
using op::base_type::_init;
using op::base_type::_stream;
net_size_t _length[1];
auto run(Buffer buffer) {
auto self = this->shared_from_this();
_length[0] = ba::buffer_size(buffer);
ba::async_write(_stream, std::vector<ba::const_buffer> { ba::buffer(_length), buffer },
this->wrap([self,this](error_code ec, size_t transferred) { _init.completion_handler(ec, transferred); }));
return _init.result.get();
}
};
return std::make_shared<op>(s, std::forward<Token>(token))->run(message_buffer);
}
template <typename AsyncStream, typename Buffer, typename Token>
auto async_receive(AsyncStream& s, Buffer& output, Token&& token) {
struct op : stream_async_op<op, AsyncStream, Token> {
using op::base_type::base_type;
using op::base_type::_init;
using op::base_type::_stream;
net_size_t _length[1] = {0};
auto run(Buffer& output) {
auto self = this->shared_from_this();
ba::async_read(_stream, ba::buffer(_length), this->wrap([self, this, &output](error_code ec, size_t transferred) {
if (ec)
_init.completion_handler(ec, transferred);
else
ba::async_read(_stream, ba::dynamic_buffer(output), ba::transfer_exactly(_length[0]),
this->wrap([self, this](error_code ec, size_t transferred) {
_init.completion_handler(ec, transferred);
}));
}));
return _init.result.get();
}
};
return std::make_shared<op>(s, std::forward<Token>(token))->run(output);
}
template <typename Output = std::string, typename AsyncStream, typename Token>
auto async_receive(AsyncStream& s, Token&& token) {
struct op : stream_async_op<op, AsyncStream, Token, void(error_code, Output)> {
using op::base_type::base_type;
using op::base_type::_init;
using op::base_type::_stream;
Output _output;
net_size_t _length[1] = {0};
auto run() {
auto self = this->shared_from_this();
ba::async_read(_stream, ba::buffer(_length), [self,this](error_code ec, size_t) {
if (ec)
_init.completion_handler(ec, std::move(_output));
else
ba::async_read(_stream, ba::dynamic_buffer(_output), ba::transfer_exactly(_length[0]),
[self,this](error_code ec, size_t) { _init.completion_handler(ec, std::move(_output)); });
});
return _init.result.get();
}
};
return std::make_shared<op>(s, std::forward<Token>(token))->run();
}
} // Protocol
#include <iostream>
#include <iomanip>
int main() {
ba::io_context io;
tcp::socket sock(io);
sock.connect({tcp::v4(), 6767});
auto cont = [](auto name, auto continuation = []{}) { return [=](error_code ec, size_t transferred) {
std::cout << name << " completed (" << transferred << ", " << ec.message() << ")\n";
if (!ec) continuation();
}; };
auto report = [=](auto name) { return cont(name, []{}); };
// send chain
std::string hello = "Hello", world = "World";
Protocol::async_transmit(sock, ba::buffer(hello),
cont("Send hello", [&] { Protocol::async_transmit(sock, ba::buffer(world), report("Send world")); }
));
#ifndef DEMO_USE_FUTURE
// receive chain
std::string object1, object2;
Protocol::async_receive(sock, object1,
cont("Read object 1", [&] { Protocol::async_receive(sock, object2, report("Read object 2")); }));
io.run();
std::cout << "Response object 1: " << std::quoted(object1) << "\n";
std::cout << "Response object 2: " << std::quoted(object2) << "\n";
#else
// also possible, alternative completion mechanisms:
std::future<std::string> fut = Protocol::async_receive(sock, ba::use_future);
io.run();
std::cout << "Response object: " << std::quoted(fut.get()) << "\n";
#endif
}
When talking to a test server like:
xxd -p -r <<< '0000 0006 4e6f 2077 6179 0000 0005 4a6f 73c3 a90a' | netcat -l -p 6767 | xxd
The program prints
Send hello completed (9, Success)
Send world completed (9, Success)
Read object 1 completed (6, Success)
Read object 2 completed (5, Success)
Response object 1: "No way"
Response object 2: "José"
And the netcat side prints:
00000000: 0000 0005 4865 6c6c 6f00 0000 0557 6f72 ....Hello....Wor
00000010: 6c64 ld
Enabling handler tracking allows you to use handlerviz.pl to visualize the call chains:
Note You can change big_int32_t to little_int32_t without any further change. Of course, you should change the payload on the server side to match:
xxd -p -r <<< '0600 0000 4e6f 2077 6179 0500 0000 4a6f 73c3 a90a' | netcat -l -p 6767 | xxd
¹ Endianness, e.g. using Boost Endian or ::ntohs, ::ntohl, ::htons and ::htonl

Converting asio read_some to async version

I have the following code that reads from a TCP socket using boost asio read_some function. Currently the code is synchronous and I need to convert it to the async version. The issue is initially that some bytes are read to identify the packettype and to get the length of the packet. Then we have a loop that reads the data. Would I need to use two callbacks to do this asynchronously or can it be done with one ( which would be preferable).
void Transport::OnReadFromTcp()
{
int read = 0;
// read 7 bytes from TCP into mTcpBuffer
m_sslsock->read_some(asio::buffer(mTcpBuffer, 7));
bool tag = true;
for (unsigned char i = 0; i < 5; i++)
{
tag = tag && (mTcpBuffer[i] == g_TcpPacketTag[i]);
}
// get the length from the last two bytes
unsigned short dataLen = (mTcpBuffer[5] ) | (mTcpBuffer[6] << 8);
mBuff = new char[dataLen];
int readTotal = 0;
while (readTotal < dataLen)
{
// read lengths worth of data from tcp pipe into buffer
int readlen = dataLen;
size_t read = m_sslsock->read_some(asio::buffer(&mBuff[readTotal], readlen));
readlen = dataLen - read;
readTotal += read;
}
// Process data .....
}
The first step is to realize that you can remove the read_some loop entireyl using the free function read:
void Transport::OnReadFromTcp() {
int read = 0;
// read 7 bytes from TCP into mTcpBuffer
size_t bytes = asio::read(*m_sslsock, asio::buffer(mTcpBuffer, 7), asio::transfer_all());
assert(bytes == 7);
bool tag = g_TcpPacketTag.end() == std::mismatch(
g_TcpPacketTag.begin(), g_TcpPacketTag.end(),
mTcpBuffer.begin(), mTcpBuffer.end())
.first;
// get the length from the last two bytes
uint16_t const dataLen = mTcpBuffer[5] | (mTcpBuffer[6] << 8);
mBuff.resize(dataLen);
size_t readTotal = asio::read(*m_sslsock, asio::buffer(mBuff), asio::transfer_exactly(dataLen));
assert(mBuff.size() == readTotal);
assert(dataLen == readTotal);
}
That's even regardless of whether execution is asynchronous.
Making it asynchronous is slightly involved, because it requires assumptions about lifetime of the buffers/Transport instance as well as potential multi-threading. I'll provide a sample of that after my morning coffee :)
Demo without threading/lifetime complications:
Live On Coliru
#include <boost/asio.hpp>
#include <boost/asio/ssl.hpp>
#include <boost/bind.hpp>
#include <iostream>
#include <array>
#include <cassert>
namespace asio = boost::asio;
namespace ssl = asio::ssl;
namespace {
static std::array<char, 5> g_TcpPacketTag {{'A','B','C','D','E'}};
}
struct Transport {
using tcp = asio::ip::tcp;
using SslSocket = std::shared_ptr<asio::ssl::stream<tcp::socket> >;
Transport(SslSocket s) : m_sslsock(s) { }
void OnReadFromTcp();
void OnHeaderReceived(boost::system::error_code ec, size_t transferred);
void OnContentReceived(boost::system::error_code ec, size_t transferred);
private:
uint16_t datalen() const {
return mTcpBuffer[5] | (mTcpBuffer[6] << 8);
}
SslSocket m_sslsock;
std::array<char, 7> mTcpBuffer;
std::vector<char> mBuff;
};
void Transport::OnReadFromTcp() {
// read 7 bytes from TCP into mTcpBuffer
asio::async_read(*m_sslsock, asio::buffer(mTcpBuffer, 7), asio::transfer_all(),
boost::bind(&Transport::OnHeaderReceived, this, asio::placeholders::error, asio::placeholders::bytes_transferred)
);
}
#include <boost/range/algorithm/mismatch.hpp> // I love sugar
void Transport::OnHeaderReceived(boost::system::error_code ec, size_t bytes) {
if (ec) {
std::cout << "Error: " << ec.message() << "\n";
}
assert(bytes == 7);
bool tag = (g_TcpPacketTag.end() == boost::mismatch(g_TcpPacketTag, mTcpBuffer).first);
if (tag) {
// get the length from the last two bytes
mBuff.resize(datalen());
asio::async_read(*m_sslsock, asio::buffer(mBuff), asio::transfer_exactly(datalen()),
boost::bind(&Transport::OnContentReceived, this, asio::placeholders::error, asio::placeholders::bytes_transferred)
);
} else {
std::cout << "TAG MISMATCH\n"; // TODO handle error
}
}
void Transport::OnContentReceived(boost::system::error_code ec, size_t readTotal) {
assert(mBuff.size() == readTotal);
assert(datalen() == readTotal);
std::cout << "Successfully completed receive of " << datalen() << " bytes\n";
}
int main() {
asio::io_service svc;
using Socket = Transport::SslSocket::element_type;
// connect to localhost:6767 with SSL
ssl::context ctx(ssl::context::sslv23);
auto s = std::make_shared<Socket>(svc, ctx);
s->lowest_layer().connect({ {}, 6767 });
s->handshake(Socket::handshake_type::client);
// do transport
Transport tx(s);
tx.OnReadFromTcp();
svc.run();
// all done
std::cout << "All done\n";
}
When using against a sample server that accepts SSL connections on port 6767:
(printf "ABCDE\x01\x01F"; cat main.cpp) |
openssl s_server -accept 6767 -cert so.crt -pass pass:test
Prints:
Successfully completed receive of 257 bytes
All done