Streaming file from aws s3 through boost socket - c++

Question

I'm trying to connect via boost ssl socket to aws s3.
It works but when I read, I had several problem.
Corrupted files on the same file but not others.
File not corrupted (md5filter got all the data) but the data sent to the buffer are not good. Meaning there is a problem between the different layers of read somewhere but can't figure out where.
Sometimes the program get stuck in the S3_client::read function and loop thousands of times in the do-while loop calling read. But it never reaches md5filter read.
It get stuck between filterStream.read() and md5filter.read() which is not called. I don't know if it gzip or filterStream. But it only happens if there is no call to the lower layers of read for a while.
Can you help spot the problem in my code ?
#ifndef BTLOOP_AWSCLIENT_H
#define BTLOOP_AWSCLIENT_H
#include "boost/iostreams/filter/gzip.hpp"
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filtering_streambuf.hpp>
#include <boost/iostreams/categories.hpp>
#include <boost/iostreams/stream.hpp>
#include <string>
#include <set>
#include <map>
#include <openssl/md5.h>
#include <sstream>
#include <fstream>
#include <iostream>
#include <boost/asio.hpp>
#include <boost/asio/ssl.hpp>
#include "Logger.h"
namespace io = boost::iostreams;
namespace asio = boost::asio;
namespace ssl = boost::asio::ssl;
typedef ssl::stream<asio::ip::tcp::socket> ssl_socket;
namespace S3Reader
{
class MD5Filter
{
public:
typedef char char_type;
struct category :
io::multichar_input_filter_tag{};
MD5Filter( std::streamsize n );
~MD5Filter();
template<typename Source>
std::streamsize read( Source& src, char* s, std::streamsize n );
void setBigFileMode() { _bigFileMode = true; }
std::string close();
void setFileName( std::string fileName ) { _fileName = fileName; };
inline std::streamsize writtenBytes() {std::streamsize res = _writtenBytes; _writtenBytes= 0; return res;};
inline bool eof(){return _eof;};
private:
void computeMd5( char* buffer, size_t size, bool force = false );
private:
bool _bigFileMode;
int _blockCount;
std::vector<unsigned char> _bufferMD5;
MD5_CTX _mdContext;
unsigned char _hashMd5[MD5_DIGEST_LENGTH];
std::string _fileName;
std::streamsize _writtenBytes;
int _totalSize;
bool _eof;
};
class Ssl_wrapper : public io::device<io::bidirectional>
{
public:
Ssl_wrapper( ssl_socket* sock, std::streamsize n ) :
_sock( sock ),_totalSize(0) { };
std::streamsize read( char_type* s, std::streamsize n )
{
boost::system::error_code ec;
size_t rval = _sock->read_some( asio::buffer( s, n ), ec );
_totalSize +=rval;
LOG_AUDIT( " wrapperR: " << rval << " " << _totalSize << " "<<ec.message());
if ( !ec )
{
return rval;
}
else if ( ec == asio::error::eof )
return -1;
else
throw boost::system::system_error( ec, "Wrapper read_some" );
}
std::streamsize write( const char* s, std::streamsize n )
{
boost::system::error_code ec;
size_t rval = _sock->write_some( asio::buffer( s, n ), ec );
if ( !ec )
{
return rval;
}
else if ( ec == asio::error::eof )
return -1;
else
throw boost::system::system_error( ec, " Wrapper read_some" );
}
private:
ssl_socket* _sock;
int _totalSize;
};
class S3_client
{
public:
S3_client( const std::string& key_id, const std::string& key_secret, const std::string& bucket );
virtual ~S3_client();
bool open( const std::string& fileName );
int read( char* buffer, size_t size );
int readLine( char* buffer, size_t size );
void close();
bool eof() { return _filterStream.eof(); }
std::string authorize( const std::string request );
bool connectSocket( std::string url, std::string port, std::string auth );
private :
std::string _key_id;
std::string _key_secret;
std::string _bucket;
std::string _fileName;
io::gzip_decompressor _gzip;
MD5Filter _md5Filter;
boost::posix_time::seconds _timeout;
ssl_socket* _sock;
Ssl_wrapper* _wrapper;
io::stream<Ssl_wrapper>* _sockstream;
std::map<std::string, std::string> _headerMap;
io::filtering_istream _filterStream;
int _totalSize;
};
}
#endif //BTLOOP_AWSCLIENT_H
S3Client.cpp
#include "S3_client.h"
#include <boost/algorithm/string.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/iostreams/copy.hpp>
#include <boost/iostreams/filter/counter.hpp>
#include <boost/exception/diagnostic_information.hpp>
#include <system/ArmError.h>
namespace io = boost::iostreams;
namespace asio = boost::asio;
namespace ssl = boost::asio::ssl;
namespace S3Reader
{
static const size_t s3_block_size = 8 * 1024 * 1024;
static const std::string base64_chars =
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz"
"0123456789+/";
static inline bool is_base64( unsigned char c )
{
return (isalnum( c ) || (c == '+') || (c == '/'));
}
std::string url_encode( const std::string& value )
{
std::ostringstream escaped;
escaped.fill( '0' );
escaped << std::hex;
for ( auto i = value.begin(), n = value.end(); i != n; ++i )
{
auto c = *i;
if ( isalnum( c ) || c == '-' || c == '_' || c == '.' || c == '~' )
{
escaped << c;
continue;
}
escaped << std::uppercase;
escaped << '%' << std::setw( 2 ) << int((unsigned char) c );
escaped << std::nouppercase;
}
return escaped.str();
}
std::string base64_encode( unsigned char const* bytes_to_encode, unsigned int in_len )
{
std::string ret;
int i = 0;
int j = 0;
unsigned char char_array_3[3];
unsigned char char_array_4[4];
while ( in_len-- )
{
char_array_3[i++] = *(bytes_to_encode++);
if ( i == 3 )
{
char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;
char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);
char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);
char_array_4[3] = char_array_3[2] & 0x3f;
for ( i = 0; (i < 4); i++ )
ret += base64_chars[char_array_4[i]];
i = 0;
}
}
if ( i )
{
for ( j = i; j < 3; j++ )
char_array_3[j] = '\0';
char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;
char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);
char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);
char_array_4[3] = char_array_3[2] & 0x3f;
for ( j = 0; (j < i + 1); j++ )
ret += base64_chars[char_array_4[j]];
while ((i++ < 3))
ret += '=';
}
return ret;
}
std::string to_hex( const uint8_t* buffer, size_t buffer_size )
{
std::stringstream sst;
for ( uint i = 0; i < buffer_size; i++ )
{
sst << std::setw( 2 ) << std::setfill( '0' ) << std::hex << int( buffer[i] );
}
return sst.str();
}
std::string getDateForHeader( bool amzFormat )
{
time_t lt;
time( &lt );
struct tm* tmTmp;
tmTmp = gmtime( &lt );
char buf[50];
if ( amzFormat )
{
strftime( buf, 50, "Date: %a, %d %b %Y %X +0000", tmTmp );
return std::string( buf );
}
else
{
tmTmp->tm_hour++;
//strftime( buf, 50, "%a, %d %b %Y %X +0000", tmTmp );
std::stringstream ss;
ss << mktime( tmTmp );
return ss.str();
}
}
MD5Filter::MD5Filter( std::streamsize n ) :
_bigFileMode( false ), _blockCount( 0 ), _writtenBytes(0), _totalSize( 0 )
{
MD5_Init( &_mdContext );
memset( _hashMd5, 0, MD5_DIGEST_LENGTH );
}
MD5Filter::~MD5Filter()
{
close();
}
template<typename Source>
std::streamsize MD5Filter::read( Source& src, char* s, std::streamsize n )
{
int result =0;
try
{
if ((result = io::read( src, s, n )) == -1 )
{
_eof=true;
LOG_AUDIT( _fileName << " md5R: " << result << " " << _totalSize );
return -1;
}
}
catch ( boost::exception& ex)
{
LOG_ERROR( _fileName <<" "<< boost::diagnostic_information(ex)<< " " << result );
}
computeMd5( s, (size_t) result );
_totalSize += result;
_writtenBytes = result;
LOG_AUDIT( _fileName << " md5R: " << result << " " << _totalSize );
return result;
}
void MD5Filter::computeMd5( char* buffer, size_t size, bool force )
{
size_t realSize = s3_block_size;
uint8_t blockMd5[MD5_DIGEST_LENGTH];
if ( !_bigFileMode )
{
MD5_Update( &_mdContext, buffer, size );
return;
}
if ( size > 0 )
{
_bufferMD5.insert( _bufferMD5.end(), &buffer[0], &buffer[size] );
}
if ((_bufferMD5.size() < s3_block_size) && !force )
return;
if ( force )
realSize = _bufferMD5.size();
MD5( &_bufferMD5[0], realSize, blockMd5 );
MD5_Update( &_mdContext, blockMd5, MD5_DIGEST_LENGTH );
_blockCount++;
if ( _bufferMD5.size() == s3_block_size )
{
_bufferMD5.clear();
return;
}
if ( force )
return;
memcpy( &_bufferMD5[0], &_bufferMD5[s3_block_size], _bufferMD5.size() - s3_block_size );
_bufferMD5.erase( _bufferMD5.begin() + s3_block_size, _bufferMD5.end());
}
std::string MD5Filter::close()
{
std::string mdOutput;
computeMd5( NULL, 0, true );
MD5_Final( _hashMd5, &_mdContext );
mdOutput = to_hex( _hashMd5, MD5_DIGEST_LENGTH );
if ( _bigFileMode )
{
mdOutput += "-" + boost::lexical_cast<std::string>( _blockCount );
}
return mdOutput;
}
std::string S3_client::authorize( const std::string request )
{
unsigned char* digest;
digest = HMAC( EVP_sha1(), _key_secret.c_str(), (int) _key_secret.size(), (unsigned char*) request.c_str(), (int) request.size(), NULL, NULL );
std::string signature( url_encode( base64_encode( digest, 20 )));
return "?AWSAccessKeyId=" + _key_id + "&Expires=" + getDateForHeader( false ) + "&Signature=" + signature;
}
S3_client::S3_client( const std::string& key_id, const std::string& key_secret, const std::string& bucket ) :
_key_id( key_id ), _key_secret( key_secret ), _bucket( bucket ), _gzip( io::gzip::default_window_bits, 1024 * 1024 )
, _md5Filter( s3_block_size ), _timeout( boost::posix_time::seconds( 1 )), _totalSize( 0 ) { }
S3_client::~S3_client()
{
close();
}
bool S3_client::connectSocket( std::string url, std::string port, std::string auth )
{
std::string amzDate = getDateForHeader( true );
std::string host = "url";
boost::asio::io_service io_service;
boost::asio::ip::tcp::resolver resolver( io_service );
boost::asio::ip::tcp::resolver::query query( url, "https" );
auto endpoint = resolver.resolve( query );
// Context with default path
ssl::context ctx( ssl::context::sslv23 );
ctx.set_default_verify_paths();
_sock = new ssl_socket( io_service, ctx );
boost::asio::socket_base::keep_alive option( true );
_wrapper = new Ssl_wrapper( _sock, s3_block_size );
_sockstream = new io::stream<Ssl_wrapper>( boost::ref( *_wrapper ));
asio::connect( _sock->lowest_layer(), endpoint );
_sock->set_verify_mode( ssl::verify_peer );
_sock->set_verify_callback( ssl::rfc2818_verification( url ));
_sock->handshake( ssl_socket::client );
_sock->lowest_layer().set_option( option );
std::stringstream ss;
ss << "GET " << _fileName << auth << " HTTP/1.1\r\n" << "Host: " << host << "\r\nAccept: */*\r\n\r\n";
_sockstream->write( ss.str().c_str(), ss.str().size());
_sockstream->flush();
std::string http_version;
int status_code = 0;
(*_sockstream) >> http_version;
(*_sockstream) >> status_code;
if ( !_sockstream || http_version.substr( 0, 5 ) != "HTTP/" )
{
std::cout << "Invalid response: " << http_version << " " << status_code << std::endl;
return false;
}
if ( status_code != 200 )
{
std::cout << "Response returned with status code " << http_version << " " << status_code << std::endl;
return false;
}
return true;
}
bool S3_client::open( const std::string& fileName )
{
std::string port = "443";
std::string url = "bucket";
std::stringstream authRequest;
std::string date = getDateForHeader( false );
_fileName = fileName;
authRequest << "GET\n\n\n" << date << "\n/" << _bucket << "" << fileName;
std::string auth = authorize( authRequest.str());
if ( !connectSocket( url, port, auth ))
THROW( "Failed to open socket" );
std::string header;
while ( std::getline( *_sockstream, header ) && header != "\r" )
{
std::vector<std::string> vectLine;
boost::split( vectLine, header, boost::is_any_of( ":" ));
if ( vectLine.size() < 2 )
continue;
boost::erase_all( vectLine[1], "\"" );
boost::erase_all( vectLine[1], "\r" );
boost::erase_all( vectLine[1], " " );
_headerMap[vectLine[0]] = vectLine[1];
}
if ( _headerMap.find( "Content-Length" ) == _headerMap.end())
return false;
if ( _headerMap.find( "Content-Type" ) == _headerMap.end())
return false;
if ((uint) std::atoi( _headerMap.at( "Content-Length" ).c_str()) > s3_block_size )
_md5Filter.setBigFileMode();
_md5Filter.setFileName( _fileName );
if ( _headerMap["Content-Type"] == "binary/octet-stream" )
_filterStream.push( _gzip, s3_block_size );
_filterStream.push( boost::ref( _md5Filter ), s3_block_size );
_filterStream.push( boost::ref( *_sockstream ), s3_block_size );
return true;
}
void S3_client::close()
{
std::string localMD5 = _md5Filter.close();
std::string headerMD5 = _headerMap["ETag"];
if ( localMD5 != headerMD5 )
THROW ( "Corrupted file " << _fileName << " " << localMD5 << " " << headerMD5 << "." );
else
LOG_AUDIT( "Close S3: " << _fileName << " " << localMD5 << " " << headerMD5 << "." );
}
int S3_client::readLine( char* buffer, size_t size )
{
_filterStream.getline( buffer, size );
return _filterStream.gcount();
}
int S3_client::read( char* buffer, size_t size )
{
std::streamsize sizeRead = 0;
do
{
_filterStream.read( buffer, size );
sizeRead = _md5Filter.writtenBytes();
_totalSize += sizeRead;
LOG_AUDIT( _fileName << " s3R: " << sizeRead << " " << _totalSize );
}
while( sizeRead ==0 && !_md5Filter.eof() && !_sockstream->eof() && _filterStream.good() && _sock->next_layer().is_open());
return sizeRead;
}
}
int main( int argc, char** argv )
{
S3Reader::S3_client client( key_id, key_secret, s3_bucket );
client.open("MyFile");
while (client.read(buffer, bufferSize) >0 ) {}
}