What I am trying to do is use the Detours library to hook into an applications WinSock2 send() and recv() functions (a packet logger).
While it does work for the send() function, it does not, however, work for the recv() function.
Here is my relevant code:
#include <cstdio>
#include <ctime>
#include <fstream>
#include <iomanip>
#include <string>
#include <windows.h>
#include <detours.h>
#pragma comment( lib, "Ws2_32.lib" )
#pragma comment( lib, "detours.lib" )
#pragma comment( lib, "detoured.lib" )
#pragma comment( lib, "Mswsock.lib" )
std::ofstream Logger;
std::string NowToString() {
time_t rawtime;
tm *timeinfo = new tm();
char buffer[32];
time( &rawtime );
localtime_s( timeinfo, &rawtime );
strftime( buffer, 32, "%m/%d/%Y %I:%M:%S %p", timeinfo );
delete timeinfo;
return std::string( buffer );
}
std::string TimeToString() {
time_t rawtime;
tm *timeinfo = new tm();
char buffer[32];
time( &rawtime );
localtime_s( timeinfo, &rawtime );
strftime( buffer, 32, "%I:%M:%S %p", timeinfo );
delete timeinfo;
return std::string( buffer );
}
void LogPacket( const char *buf, int len ) {
Logger << " 0 1 2 3 4 5 6 7 8 9 A B C D E F\n";
Logger << " -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --\n";
Logger << "0000 ";
for ( int i = 0; i < len; ++i ) {
if ( i != 0 && i % 16 == 0 ) {
Logger << " ";
int line = ( i / 16 ) - 1;
for ( int j = 0; j < 16; ++j ) {
char c = buf[line * 16 + j];
if ( c >= 32 && c <= 126 ) {
Logger << c;
} else {
Logger << '.';
}
}
Logger << "\n" << std::hex << std::setw( 4 ) << std::setfill( '0' ) << i << std::dec << std::setw( 0 ) << " ";
} else if ( i % 16 == 8 ) {
Logger << ' ';
}
Logger << std::hex << std::setw( 2 ) << std::setfill( '0' ) << ( int( buf[i] ) & 0xFF ) << ' ';
Logger << std::dec << std::setw( 0 );
if ( i == len - 1 ) {
int remaining = 16 - ( len % 16 );
int fill = ( remaining * 3 ) + 2;
if ( remaining >= 8 ) {
++fill;
}
for ( int j = 0; j < fill; ++j ) {
Logger << ' ';
}
int line = ( i - ( ( len % 16 ) - 1 ) ) / 16 ;
for ( int k = 0; k < ( len % 16 ); ++k ) {
char c = buf[line * 16 + k];
if ( c >= 32 && c <= 126 ) {
Logger << c;
} else {
Logger << '.';
}
}
}
}
Logger << "\n\n";
}
int ( WINAPI *Real_Send )( SOCKET s, const char *buf, int len, int flags ) = send;
int ( WINAPI *Real_Recv )( SOCKET s, char *buf, int len, int flags ) = recv;
int ( WINAPI *Real_RecvFrom )( SOCKET s, char *buf, int len, int flags, sockaddr *from, int *fromlen ) = recvfrom;
int ( WINAPI *Real_WSARecvEx )( SOCKET s, char *buf, int len, int *flags ) = WSARecvEx;
int WINAPI Mine_Send( SOCKET s, const char* buf, int len, int flags );
int WINAPI Mine_Recv( SOCKET s, char *buf, int len, int flags );
int WINAPI Mine_RecvFrom( SOCKET s, char *buf, int len, int flags, sockaddr *from, int *fromlen );
int WINAPI Mine_WSARecvEx( SOCKET s, char *buf, int len, int *flags );
int WINAPI Mine_Send( SOCKET s, const char *buf, int len, int flags ) {
Logger << TimeToString() << ": Client -> Server (Length: " << len << " bytes)\n\n";
LogPacket( buf, len );
Logger << std::endl;
return Real_Send( s, buf, len, flags );
}
int WINAPI Mine_Recv( SOCKET s, char *buf, int len, int flags ) {
Logger << TimeToString() << ": Server -> Client (Length: " << len << " bytes)\n\n";
LogPacket( buf, len );
Logger << std::endl;
return Real_Recv( s, buf, len, flags );
}
int WINAPI Mine_RecvFrom( SOCKET s, char *buf, int len, int flags, sockaddr *from, int *fromlen ) {
Logger << TimeToString() << ": Server -> Client (Length: " << len << " bytes)*\n\n";
LogPacket( buf, len );
Logger << std::endl;
return Real_RecvFrom( s, buf, len, flags, from, fromlen );
}
int WINAPI Mine_WSARecvEx( SOCKET s, char *buf, int len, int *flags ) {
Logger << TimeToString() << ": Server -> Client (Length: " << len << " bytes)**\n\n";
LogPacket( buf, len );
Logger << std::endl;
return Real_WSARecvEx( s, buf, len, flags );
}
BOOL WINAPI DllMain( HINSTANCE, DWORD dwReason, LPVOID ) {
switch ( dwReason ) {
case DLL_PROCESS_ATTACH:
Logger.open( "C:\\Packets.txt", std::ios::out | std::ios::app | std::ios::ate );
if ( Logger.tellp() > 0 ) {
Logger << "\n\n\n";
}
Logger << "##\n## Logging Started (" << NowToString() << ")\n##\n\n\n";
DetourTransactionBegin();
DetourUpdateThread( GetCurrentThread() );
DetourAttach( &(PVOID &)Real_Send, Mine_Send );
DetourAttach( &(PVOID &)Real_Recv, Mine_Recv );
DetourAttach( &(PVOID &)Real_RecvFrom, Mine_RecvFrom );
DetourAttach( &(PVOID &)Real_WSARecvEx, Mine_WSARecvEx );
DetourTransactionCommit();
break;
case DLL_PROCESS_DETACH:
Logger << "##\n## Logging Stopped (" << NowToString() << ")\n##";
Logger.close();
DetourTransactionBegin();
DetourUpdateThread( GetCurrentThread() );
DetourDetach( &(PVOID &)Real_Send, Mine_Send );
DetourDetach( &(PVOID &)Real_Recv, Mine_Recv );
DetourDetach( &(PVOID &)Real_RecvFrom, Mine_RecvFrom );
DetourDetach( &(PVOID &)Real_WSARecvEx, Mine_WSARecvEx );
DetourTransactionCommit();
break;
}
return TRUE;
}
Any ideas?
EDIT: So I've hooked recvfrom(), and WSARecvEx() as well, and it still doesn't log the outgoing packets! I've updated my code with my exact code.
To use hooking effectively you need to make sure you really hook all of the relevant endpoints or atleast a common denominator that is ensured to be called eventually from all of them.
In the case of recv() I think its possible that the application actually calls WSARecv() instead.
You can use depends.exe to find out exactly what functions does the application/library import from Ws2_32.dll to know which ones you need to hook.
Well, a few months later I figured it out: I was hooking WinSock 2 functions when I should have been hooking WinSock 1.1's send()/recv()!
Related
On Python, there is this option errors='ignore' for the open Python function:
open( '/filepath.txt', 'r', encoding='UTF-8', errors='ignore' )
With this, reading a file with invalid UTF8 characters will replace them with nothing, i.e., they are ignored. For example, a file with the characthers Føö»BÃ¥r is going to be read as FøöBår.
If a line as Føö»BÃ¥r is read with getline() from stdio.h, it will be read as Føö�Bår:
FILE* cfilestream = fopen( "/filepath.txt", "r" );
int linebuffersize = 131072;
char* readline = (char*) malloc( linebuffersize );
while( true )
{
if( getline( &readline, &linebuffersize, cfilestream ) != -1 ) {
std::cerr << "readline=" readline << std::endl;
}
else {
break;
}
}
How can I make stdio.h getline() read it as FøöBår instead of Føö�Bår, i..e, ignoring invalid UTF8 characters?
One overwhelming solution I can think of it do iterate throughout all characters on each line read and build a new readline without any of these characters. For example:
FILE* cfilestream = fopen( "/filepath.txt", "r" );
int linebuffersize = 131072;
char* readline = (char*) malloc( linebuffersize );
char* fixedreadline = (char*) malloc( linebuffersize );
int index;
int charsread;
int invalidcharsoffset;
while( true )
{
if( ( charsread = getline( &readline, &linebuffersize, cfilestream ) ) != -1 )
{
invalidcharsoffset = 0;
for( index = 0; index < charsread; ++index )
{
if( readline[index] != '�' ) {
fixedreadline[index-invalidcharsoffset] = readline[index];
}
else {
++invalidcharsoffset;
}
}
std::cerr << "fixedreadline=" << fixedreadline << std::endl;
}
else {
break;
}
}
Related questions:
Fixing invalid UTF8 characters
Replacing non UTF8 characters
python replace unicode characters
Python unicode: how to replace character that cannot be decoded using utf8 with whitespace?
You are confusing what you see with what is really going on. The getline function does not do any replacement of characters. [Note 1]
You are seeing a replacement character (U+FFFD) because your console outputs that character when it is asked to render an invalid UTF-8 code. Most consoles will do that if they are in UTF-8 mode; that is, the current locale is UTF-8.
Also, saying that a file contains the "characters Føö»BÃ¥r" is at best imprecise. A file does not really contain characters. It contains byte sequences which may be interpreted as characters -- for example, by a console or other user presentation software which renders them into glyphs -- according to some encoding. Different encodings produce different results; in this particular case, you have a file which was created by software using the Windows-1252 encoding (or, roughly equivalently, ISO 8859-15), and you are rendering it on a console using UTF-8.
What that means is that the data read by getline contains an invalid UTF-8 sequence, but it (probably) does not contain the replacement character code. Based on the character string you present, it contains the hex character \xbb, which is a guillemot (») in Windows code page 1252.
Finding all the invalid UTF-8 sequences in a string read by getline (or any other C library function which reads files) requires scanning the string, but not for a particular code sequence. Rather, you need to decode UTF-8 sequences one at a time, looking for the ones which are not valid. That's not a simple task, but the mbtowc function can help (if you have enabled a UTF-8 locale). As you'll see in the linked manpage, mbtowc returns the number of bytes contained in a valid "multibyte sequence" (which is UTF-8 in a UTF-8 locale), or -1 to indicate an invalid or incomplete sequence. In the scan, you should pass through the bytes in a valid sequence, or remove/ignore the single byte starting an invalid sequence, and then continue the scan until you reach the end of the string.
Here's some lightly-tested example code (in C):
#include <stdlib.h>
#include <string.h>
/* Removes in place any invalid UTF-8 sequences from at most 'len' characters of the
* string pointed to by 's'. (If a NUL byte is encountered, conversion stops.)
* If the length of the converted string is less than 'len', a NUL byte is
* inserted.
* Returns the length of the possibly modified string (with a maximum of 'len'),
* not including the NUL terminator (if any).
* Requires that a UTF-8 locale be active; since there is no way to test for
* this condition, no attempt is made to do so. If the current locale is not UTF-8,
* behaviour is undefined.
*/
size_t remove_bad_utf8(char* s, size_t len) {
char* in = s;
/* Skip over the initial correct sequence. Avoid relying on mbtowc returning
* zero if n is 0, since Posix is not clear whether mbtowc returns 0 or -1.
*/
int seqlen;
while (len && (seqlen = mbtowc(NULL, in, len)) > 0) { len -= seqlen; in += seqlen; }
char* out = in;
if (len && seqlen < 0) {
++in;
--len;
/* If we find an invalid sequence, we need to start shifting correct sequences. */
for (; len; in += seqlen, len -= seqlen) {
seqlen = mbtowc(NULL, in, len);
if (seqlen > 0) {
/* Shift the valid sequence (if one was found) */
memmove(out, in, seqlen);
out += seqlen;
}
else if (seqlen < 0) seqlen = 1;
else /* (seqlen == 0) */ break;
}
*out++ = 0;
}
return out - s;
}
Notes
Aside from the possible line-end transformation of the underlying I/O library, which will replace CR-LF with a single \n on systems like Windows where the two character CR-LF sequence is used as a line-end indication.
As #rici well explains in his answer, there can be several invalid UTF-8 sequences in a byte sequence.
Possibly iconv(3) could be worth a look, e.g. see https://linux.die.net/man/3/iconv_open.
When the string "//IGNORE" is appended to tocode, characters that cannot be represented in the target character set will be silently discarded.
Example
This byte sequence, if interpreted as UTF-8, contains some invalid UTF-8:
"some invalid\xFE\xFE\xFF\xFF stuff"
If you display this you would see something like
some invalid���� stuff
When this string passes through the remove_invalid_utf8 function in the following C program, the invalid UTF-8 bytes are removed using the iconv function mentioned above.
So the result is then:
some invalid stuff
C Program
#include <stdio.h>
#include <iconv.h>
#include <string.h>
#include <stdlib.h>
#include <stdbool.h>
#include <errno.h>
char *remove_invalid_utf8(char *utf8, size_t len) {
size_t inbytes_len = len;
char *inbuf = utf8;
size_t outbytes_len = len;
char *result = calloc(outbytes_len + 1, sizeof(char));
char *outbuf = result;
iconv_t cd = iconv_open("UTF-8//IGNORE", "UTF-8");
if(cd == (iconv_t)-1) {
perror("iconv_open");
}
if(iconv(cd, &inbuf, &inbytes_len, &outbuf, &outbytes_len)) {
perror("iconv");
}
iconv_close(cd);
return result;
}
int main() {
char *utf8 = "some invalid\xFE\xFE\xFF\xFF stuff";
char *converted = remove_invalid_utf8(utf8, strlen(utf8));
printf("converted: %s to %s\n", utf8, converted);
free(converted);
return 0;
}
I also managed to fix it by trailing/cutting down all Non-ASCII characters.
This one takes about 2.6 seconds to parse 319MB:
#include <stdlib.h>
#include <iostream>
int main(int argc, char const *argv[])
{
FILE* cfilestream = fopen( "./test.txt", "r" );
size_t linebuffersize = 131072;
if( cfilestream == NULL ) {
perror( "fopen cfilestream" );
return -1;
}
char* readline = (char*) malloc( linebuffersize );
char* fixedreadline = (char*) malloc( linebuffersize );
if( readline == NULL ) {
perror( "malloc readline" );
return -1;
}
if( fixedreadline == NULL ) {
perror( "malloc fixedreadline" );
return -1;
}
char* source;
if( ( source = std::setlocale( LC_ALL, "en_US.utf8" ) ) == NULL ) {
perror( "setlocale" );
}
else {
std::cerr << "locale='" << source << "'" << std::endl;
}
int index;
int charsread;
int invalidcharsoffset;
unsigned int fixedchar;
while( true )
{
if( ( charsread = getline( &readline, &linebuffersize, cfilestream ) ) != -1 )
{
invalidcharsoffset = 0;
for( index = 0; index < charsread; ++index )
{
fixedchar = static_cast<unsigned int>( readline[index] );
// std::cerr << "index " << std::setw(3) << index
// << " readline " << std::setw(10) << fixedchar
// << " -> '" << readline[index] << "'" << std::endl;
if( 31 < fixedchar && fixedchar < 128 ) {
fixedreadline[index-invalidcharsoffset] = readline[index];
}
else {
++invalidcharsoffset;
}
}
fixedreadline[index-invalidcharsoffset] = '\0';
// std::cerr << "fixedreadline=" << fixedreadline << std::endl;
}
else {
break;
}
}
std::cerr << "fixedreadline=" << fixedreadline << std::endl;
free( readline );
free( fixedreadline );
fclose( cfilestream );
return 0;
}
Alternative and slower version using memcpy
Using menmove does not improve much speed, so you could either one.
This one takes about 3.1 seconds to parse 319MB:
#include <stdlib.h>
#include <iostream>
#include <cstring>
#include <iomanip>
int main(int argc, char const *argv[])
{
FILE* cfilestream = fopen( "./test.txt", "r" );
size_t linebuffersize = 131072;
if( cfilestream == NULL ) {
perror( "fopen cfilestream" );
return -1;
}
char* readline = (char*) malloc( linebuffersize );
char* fixedreadline = (char*) malloc( linebuffersize );
if( readline == NULL ) {
perror( "malloc readline" );
return -1;
}
if( fixedreadline == NULL ) {
perror( "malloc fixedreadline" );
return -1;
}
char* source;
char* destination;
char* finalresult;
int index;
int lastcopy;
int charsread;
int charstocopy;
int invalidcharsoffset;
bool hasignoredbytes;
unsigned int fixedchar;
if( ( source = std::setlocale( LC_ALL, "en_US.utf8" ) ) == NULL ) {
perror( "setlocale" );
}
else {
std::cerr << "locale='" << source << "'" << std::endl;
}
while( true )
{
if( ( charsread = getline( &readline, &linebuffersize, cfilestream ) ) != -1 )
{
hasignoredbytes = false;
source = readline;
destination = fixedreadline;
lastcopy = 0;
invalidcharsoffset = 0;
for( index = 0; index < charsread; ++index )
{
fixedchar = static_cast<unsigned int>( readline[index] );
// std::cerr << "fixedchar " << std::setw(10)
// << fixedchar << " -> '"
// << readline[index] << "'" << std::endl;
if( 31 < fixedchar && fixedchar < 128 ) {
if( hasignoredbytes ) {
charstocopy = index - lastcopy - invalidcharsoffset;
memcpy( destination, source, charstocopy );
source += index - lastcopy;
lastcopy = index;
destination += charstocopy;
invalidcharsoffset = 0;
hasignoredbytes = false;
}
}
else {
++invalidcharsoffset;
hasignoredbytes = true;
}
}
if( destination != fixedreadline ) {
charstocopy = charsread - static_cast<int>( source - readline )
- invalidcharsoffset;
memcpy( destination, source, charstocopy );
destination += charstocopy - 1;
if( *destination == '\n' ) {
*destination = '\0';
}
else {
*++destination = '\0';
}
finalresult = fixedreadline;
}
else {
finalresult = readline;
}
// std::cerr << "finalresult=" << finalresult << std::endl;
}
else {
break;
}
}
std::cerr << "finalresult=" << finalresult << std::endl;
free( readline );
free( fixedreadline );
fclose( cfilestream );
return 0;
}
Optimized solution using iconv
This takes about 4.6 seconds to parse 319MB of text.
#include <iconv.h>
#include <string.h>
#include <stdlib.h>
#include <iostream>
// Compile it with:
// g++ -o main test.cpp -O3 -liconv
int main(int argc, char const *argv[])
{
FILE* cfilestream = fopen( "./test.txt", "r" );
size_t linebuffersize = 131072;
if( cfilestream == NULL ) {
perror( "fopen cfilestream" );
return -1;
}
char* readline = (char*) malloc( linebuffersize );
char* fixedreadline = (char*) malloc( linebuffersize );
if( readline == NULL ) {
perror( "malloc readline" );
return -1;
}
if( fixedreadline == NULL ) {
perror( "malloc fixedreadline" );
return -1;
}
char* source;
char* destination;
int charsread;
size_t inchars;
size_t outchars;
if( ( source = std::setlocale( LC_ALL, "en_US.utf8" ) ) == NULL ) {
perror( "setlocale" );
}
else {
std::cerr << "locale='" << source << "'" << std::endl;
}
iconv_t conversiondescriptor = iconv_open("UTF-8//IGNORE", "UTF-8");
if( conversiondescriptor == (iconv_t)-1 ) {
perror( "iconv_open conversiondescriptor" );
}
while( true )
{
if( ( charsread = getline( &readline, &linebuffersize, cfilestream ) ) != -1 )
{
source = readline;
inchars = charsread;
destination = fixedreadline;
outchars = charsread;
if( iconv( conversiondescriptor, &source, &inchars, &destination, &outchars ) )
{
perror( "iconv" );
}
// Trim out the new line character
if( *--destination == '\n' ) {
*--destination = '\0';
}
else {
*destination = '\0';
}
// std::cerr << "fixedreadline='" << fixedreadline << "'" << std::endl;
}
else {
break;
}
}
std::cerr << "fixedreadline='" << fixedreadline << "'" << std::endl;
free( readline );
free( fixedreadline );
if( fclose( cfilestream ) ) {
perror( "fclose cfilestream" );
}
if( iconv_close( conversiondescriptor ) ) {
perror( "iconv_close conversiondescriptor" );
}
return 0;
}
Slowest solution ever using mbtowc
This takes about 24.2 seconds to parse 319MB of text.
If you comment out the line fixedchar = mbtowc(NULL, source, charsread); and uncomment the line charsread -= fixedchar; (breaking the invalid characters removal) this will take 1.9 seconds instead of 24.2 seconds (also compiled with -O3 optimization level).
#include <stdlib.h>
#include <string.h>
#include <iostream>
#include <cstring>
#include <iomanip>
int main(int argc, char const *argv[])
{
FILE* cfilestream = fopen( "./test.txt", "r" );
size_t linebuffersize = 131072;
if( cfilestream == NULL ) {
perror( "fopen cfilestream" );
return -1;
}
char* readline = (char*) malloc( linebuffersize );
if( readline == NULL ) {
perror( "malloc readline" );
return -1;
}
char* source;
char* lineend;
char* destination;
int charsread;
int fixedchar;
if( ( source = std::setlocale( LC_ALL, "en_US.utf8" ) ) == NULL ) {
perror( "setlocale" );
}
else {
std::cerr << "locale='" << source << "'" << std::endl;
}
while( true )
{
if( ( charsread = getline( &readline, &linebuffersize, cfilestream ) ) != -1 )
{
lineend = readline + charsread;
destination = readline;
for( source = readline; source != lineend; )
{
// fixedchar = 1;
fixedchar = mbtowc(NULL, source, charsread);
charsread -= fixedchar;
// std::ostringstream contents;
// for( int index = 0; index < fixedchar; ++index )
// contents << source[index];
// std::cerr << "fixedchar=" << std::setw(10)
// << fixedchar << " -> '"
// << contents.str().c_str() << "'" << std::endl;
if( fixedchar > 0 ) {
memmove( destination, source, fixedchar );
source += fixedchar;
destination += fixedchar;
}
else if( fixedchar < 0 ) {
source += 1;
// std::cerr << "errno=" << strerror( errno ) << std::endl;
}
else {
break;
}
}
// Trim out the new line character
if( *--destination == '\n' ) {
*--destination = '\0';
}
else {
*destination = '\0';
}
// std::cerr << "readline='" << readline << "'" << std::endl;
}
else {
break;
}
}
std::cerr << "readline='" << readline << "'" << std::endl;
if( fclose( cfilestream ) ) {
perror( "fclose cfilestream" );
}
free( readline );
return 0;
}
Fastest version from all my others above using memmove
You cannot use memcpy here because the memory regions overlap!
This takes about 2.4 seconds to parse 319MB.
If you comment out the lines *destination = *source and memmove( destination, source, 1 ) (breaking the invalid characters removal) the performance still almost the same as when memmove is being called. Here in, calling memmove( destination, source, 1 ) is a little slower than directly doing *destination = *source;
#include <stdlib.h>
#include <iostream>
#include <cstring>
#include <iomanip>
int main(int argc, char const *argv[])
{
FILE* cfilestream = fopen( "./test.txt", "r" );
size_t linebuffersize = 131072;
if( cfilestream == NULL ) {
perror( "fopen cfilestream" );
return -1;
}
char* readline = (char*) malloc( linebuffersize );
if( readline == NULL ) {
perror( "malloc readline" );
return -1;
}
char* source;
char* lineend;
char* destination;
int charsread;
unsigned int fixedchar;
if( ( source = std::setlocale( LC_ALL, "en_US.utf8" ) ) == NULL ) {
perror( "setlocale" );
}
else {
std::cerr << "locale='" << source << "'" << std::endl;
}
while( true )
{
if( ( charsread = getline( &readline, &linebuffersize, cfilestream ) ) != -1 )
{
lineend = readline + charsread;
destination = readline;
for( source = readline; source != lineend; ++source )
{
fixedchar = static_cast<unsigned int>( *source );
// std::cerr << "fixedchar=" << std::setw(10)
// << fixedchar << " -> '" << *source << "'" << std::endl;
if( 31 < fixedchar && fixedchar < 128 ) {
*destination = *source;
++destination;
}
}
// Trim out the new line character
if( *source == '\n' ) {
*--destination = '\0';
}
else {
*destination = '\0';
}
// std::cerr << "readline='" << readline << "'" << std::endl;
}
else {
break;
}
}
std::cerr << "readline='" << readline << "'" << std::endl;
if( fclose( cfilestream ) ) {
perror( "fclose cfilestream" );
}
free( readline );
return 0;
}
Bonus
You can also use Python C Extensions (API).
It takes about 2.3 seconds to parse 319MB without converting them to cached version UTF-8 char*
And takes about 3.2 seconds to parse 319MB converting them to UTF-8 char*.
And also takes about 3.2 seconds to parse 319MB converting them to cached ASCII char*.
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <iostream>
typedef struct
{
PyObject_HEAD
}
PyFastFile;
static PyModuleDef fastfilepackagemodule =
{
// https://docs.python.org/3/c-api/module.html#c.PyModuleDef
PyModuleDef_HEAD_INIT,
"fastfilepackage", /* name of module */
"Example module that wrapped a C++ object", /* module documentation, may be NULL */
-1, /* size of per-interpreter state of the module, or
-1 if the module keeps state in global variables. */
NULL, /* PyMethodDef* m_methods */
NULL, /* inquiry m_reload */
NULL, /* traverseproc m_traverse */
NULL, /* inquiry m_clear */
NULL, /* freefunc m_free */
};
// initialize PyFastFile Object
static int PyFastFile_init(PyFastFile* self, PyObject* args, PyObject* kwargs) {
char* filepath;
if( !PyArg_ParseTuple( args, "s", &filepath ) ) {
return -1;
}
int linecount = 0;
PyObject* iomodule;
PyObject* openfile;
PyObject* fileiterator;
iomodule = PyImport_ImportModule( "builtins" );
if( iomodule == NULL ) {
std::cerr << "ERROR: FastFile failed to import the io module '"
"(and open the file " << filepath << "')!" << std::endl;
PyErr_PrintEx(100);
return -1;
}
PyObject* openfunction = PyObject_GetAttrString( iomodule, "open" );
if( openfunction == NULL ) {
std::cerr << "ERROR: FastFile failed get the io module open "
<< "function (and open the file '" << filepath << "')!" << std::endl;
PyErr_PrintEx(100);
return -1;
}
openfile = PyObject_CallFunction(
openfunction, "ssiss", filepath, "r", -1, "ASCII", "ignore" );
if( openfile == NULL ) {
std::cerr << "ERROR: FastFile failed to open the file'"
<< filepath << "'!" << std::endl;
PyErr_PrintEx(100);
return -1;
}
PyObject* iterfunction = PyObject_GetAttrString( openfile, "__iter__" );
Py_DECREF( openfunction );
if( iterfunction == NULL ) {
std::cerr << "ERROR: FastFile failed get the io module iterator"
<< "function (and open the file '" << filepath << "')!" << std::endl;
PyErr_PrintEx(100);
return -1;
}
PyObject* openiteratorobject = PyObject_CallObject( iterfunction, NULL );
Py_DECREF( iterfunction );
if( openiteratorobject == NULL ) {
std::cerr << "ERROR: FastFile failed get the io module iterator object"
<< " (and open the file '" << filepath << "')!" << std::endl;
PyErr_PrintEx(100);
return -1;
}
fileiterator = PyObject_GetAttrString( openfile, "__next__" );
Py_DECREF( openiteratorobject );
if( fileiterator == NULL ) {
std::cerr << "ERROR: FastFile failed get the io module iterator "
<< "object (and open the file '" << filepath << "')!" << std::endl;
PyErr_PrintEx(100);
return -1;
}
PyObject* readline;
while( ( readline = PyObject_CallObject( fileiterator, NULL ) ) != NULL ) {
linecount += 1;
PyUnicode_AsUTF8( readline );
Py_DECREF( readline );
// std::cerr << "linecount " << linecount << " readline '" << readline
// << "' '" << PyUnicode_AsUTF8( readline ) << "'" << std::endl;
}
std::cerr << "linecount " << linecount << std::endl;
// PyErr_PrintEx(100);
PyErr_Clear();
PyObject* closefunction = PyObject_GetAttrString( openfile, "close" );
if( closefunction == NULL ) {
std::cerr << "ERROR: FastFile failed get the close file function for '"
<< filepath << "')!" << std::endl;
PyErr_PrintEx(100);
return -1;
}
PyObject* closefileresult = PyObject_CallObject( closefunction, NULL );
Py_DECREF( closefunction );
if( closefileresult == NULL ) {
std::cerr << "ERROR: FastFile failed close open file '"
<< filepath << "')!" << std::endl;
PyErr_PrintEx(100);
return -1;
}
Py_DECREF( closefileresult );
Py_XDECREF( iomodule );
Py_XDECREF( openfile );
Py_XDECREF( fileiterator );
return 0;
}
// destruct the object
static void PyFastFile_dealloc(PyFastFile* self) {
Py_TYPE(self)->tp_free( (PyObject*) self );
}
static PyTypeObject PyFastFileType =
{
PyVarObject_HEAD_INIT( NULL, 0 )
"fastfilepackage.FastFile" /* tp_name */
};
// create the module
PyMODINIT_FUNC PyInit_fastfilepackage(void)
{
PyObject* thismodule;
// https://docs.python.org/3/c-api/typeobj.html
PyFastFileType.tp_new = PyType_GenericNew;
PyFastFileType.tp_basicsize = sizeof(PyFastFile);
PyFastFileType.tp_dealloc = (destructor) PyFastFile_dealloc;
PyFastFileType.tp_flags = Py_TPFLAGS_DEFAULT;
PyFastFileType.tp_doc = "FastFile objects";
PyFastFileType.tp_init = (initproc) PyFastFile_init;
if( PyType_Ready( &PyFastFileType) < 0 ) {
return NULL;
}
thismodule = PyModule_Create(&fastfilepackagemodule);
if( thismodule == NULL ) {
return NULL;
}
// Add FastFile class to thismodule allowing the use to create objects
Py_INCREF( &PyFastFileType );
PyModule_AddObject( thismodule, "FastFile", (PyObject*) &PyFastFileType );
return thismodule;
}
To built it, create the file source/fastfilewrappar.cpp with the contents of the above file and the setup.py with the following contents:
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from setuptools import setup, Extension
myextension = Extension(
language = "c++",
extra_link_args = ["-std=c++11"],
extra_compile_args = ["-std=c++11"],
name = 'fastfilepackage',
sources = [
'source/fastfilewrapper.cpp'
],
include_dirs = [ 'source' ],
)
setup(
name = 'fastfilepackage',
ext_modules= [ myextension ],
)
To run example, use following Python script:
import time
import datetime
import fastfilepackage
testfile = './test.txt'
timenow = time.time()
iterable = fastfilepackage.FastFile( testfile )
fastfile_time = time.time() - timenow
timedifference = datetime.timedelta( seconds=fastfile_time )
print( 'FastFile timedifference', timedifference, flush=True )
Example:
user#user-pc$ /usr/bin/pip3.6 install .
Processing /fastfilepackage
Building wheels for collected packages: fastfilepackage
Building wheel for fastfilepackage (setup.py) ... done
Stored in directory: /pip-ephem-wheel-cache-j313cpzc/wheels/e5/5f/bc/52c820
Successfully built fastfilepackage
Installing collected packages: fastfilepackage
Found existing installation: fastfilepackage 0.0.0
Uninstalling fastfilepackage-0.0.0:
Successfully uninstalled fastfilepackage-0.0.0
Successfully installed fastfilepackage-0.0.0
user#user-pc$ /usr/bin/python3.6 fastfileperformance.py
linecount 820800
FastFile timedifference 0:00:03.204614
Using std::getline
This takes about 4.7 seconds to parse 319MB.
If you remove the UTF-8 removal algorithm borrowed from the fastest benchmark using stdlib.h getline(), it takes 1.7 seconds to run.
#include <stdlib.h>
#include <iostream>
#include <locale>
#include <fstream>
#include <iomanip>
int main(int argc, char const *argv[])
{
unsigned int fixedchar;
int linecount = -1;
char* source;
char* lineend;
char* destination;
if( ( source = setlocale( LC_ALL, "en_US.ascii" ) ) == NULL ) {
perror( "setlocale" );
return -1;
}
else {
std::cerr << "locale='" << source << "'" << std::endl;
}
std::ifstream fileifstream{ "./test.txt" };
if( fileifstream.fail() ) {
std::cerr << "ERROR: FastFile failed to open the file!" << std::endl;
return -1;
}
size_t linebuffersize = 131072;
char* readline = (char*) malloc( linebuffersize );
if( readline == NULL ) {
perror( "malloc readline" );
return -1;
}
while( true )
{
if( !fileifstream.eof() )
{
linecount += 1;
fileifstream.getline( readline, linebuffersize );
lineend = readline + fileifstream.gcount();
destination = readline;
for( source = readline; source != lineend; ++source )
{
fixedchar = static_cast<unsigned int>( *source );
// std::cerr << "fixedchar=" << std::setw(10)
// << fixedchar << " -> '" << *source << "'" << std::endl;
if( 31 < fixedchar && fixedchar < 128 ) {
*destination = *source;
++destination;
}
}
// Trim out the new line character
if( *source == '\n' ) {
*--destination = '\0';
}
else {
*destination = '\0';
}
// std::cerr << "readline='" << readline << "'" << std::endl;
}
else {
break;
}
}
std::cerr << "linecount='" << linecount << "'" << std::endl;
if( fileifstream.is_open() ) {
fileifstream.close();
}
free( readline );
return 0;
}
Resume
2.6 seconds trimming UTF-8 using two buffers with indexing
3.1 seconds trimming UTF-8 using two buffers with memcpy
4.6 seconds removing invalid UTF-8 with iconv
24.2 seconds removing invalid UTF-8 with mbtowc
2.4 seconds trimming UTF-8 using one buffer with pointer direct assigning
Bonus
2.3 seconds removing invalid UTF-8 without converting them to a cached UTF-8 char*
3.2 seconds removing invalid UTF-8 converting them to a cached UTF-8 char*
3.2 seconds trimming UTF-8 and caching as ASCII char*
4.7 seconds trimming UTF-8 with std::getline() using one buffer with pointer direct assigning
The used file ./text.txt had 820.800 lines where each line was equal to:
id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char&id-é-char\r\n
And all versions where compiled with
g++ (GCC) 7.4.0
iconv (GNU libiconv 1.14)
g++ -o main test.cpp -O3 -liconv && time ./main
I am currently working with portaudio with an application that records, and I seem to have some issues collecting the samples. From what I can see is only one sample being stored, and the callback is only being called once, and that is it, even though the variable NUM_OF_SECONDS is set to 30 seconds.
I currently running out of ideas of what I can test, and how I can debug this, so I've come here, any suggestions to how I can debug my problem?
Here is the code:
main.cpp:
#include <record.h>
int main()
{
record somethis;
somethis.start_record();
return 0;
}
record.h
#pragma once
#include <iostream> // Functionality: COUT
#include "portaudio.h"
#include <stdio.h>
#include <stdlib.h>
#include <chrono> //Functionality: Sleep
#include <thread> //Functionality: Sleep
#include <algorithm> //Functionality: fill_n
#define SAMPLE_RATE (44100)
typedef float SAMPLE;
#define NUM_SECONDS 30
#define NUM_CHANNELS 2
#define SAMPLE_SILENCE 0.0f
#define PA_SAMPLE_TYPE paFloat32
#define FRAMES_PER_BUFFER (512)
#define TRUE (1==1)
#define FALSE (!TRUE)
#define WRITE_TO_FILE TRUE
typedef struct
{
int frameIndex;
int maxFrameindex;
SAMPLE *recordedSamples;
}
paTestData;
class record {
public:
record();
void start_record();
private:
PaStreamParameters inputParameters,
outputParameters;
PaStream* stream;
PaError err = paNoError;
paTestData data;
int totalFrames;
int numSamples;
int numBytes;
SAMPLE max, val;
double average;
int recordCallback(const void *inputBuffer, void *outputBuffer,
unsigned long framesPerBuffer,
const PaStreamCallbackTimeInfo* timeInfo,
PaStreamCallbackFlags statusFlags, void *userData);
static int recordCallbackSub(const void *inputBuffer, void *outputBuffer,
unsigned long framesPerBuffer, const PaStreamCallbackTimeInfo *timeInfo,
PaStreamCallbackFlags statusFlags, void *userData)
{
auto pThis = reinterpret_cast<record*>(userData); // get back the this pointer.
return pThis->recordCallback( inputBuffer, outputBuffer,framesPerBuffer, timeInfo,statusFlags, nullptr);
}
};
record.cpp
#include "record.h"
record::record()
{
std::cout << "Record object made" << std::endl;
std::cout << "Portaudio Version: " << Pa_GetVersion() << std::endl;
this->data.maxFrameindex = this->totalFrames = NUM_SECONDS * SAMPLE_RATE;
this->data.frameIndex = 0;
this->numSamples = this->totalFrames * NUM_CHANNELS;
numBytes = numSamples * sizeof(SAMPLE);
this->data.recordedSamples = new SAMPLE[numSamples]; /* From now on, recordedSamples is initialised. */
if( this->data.recordedSamples == NULL )
{
std::cout << "Could not allocate record array" << std::endl;
exit(1);
}
for(int i=0; i<numSamples; i++ )
{
this->data.recordedSamples[i] = 0;
}
int err = Pa_Initialize();
if( err == paNoError )
{
std::cout << "No error in init" << std::endl;
std::cout << "PortAudio init: "<< Pa_GetErrorText( err ) << std::endl;
}
else
{
printf( "PortAudio error: %s\n", Pa_GetErrorText( err ) );
exit(1);
}
this->inputParameters.device = Pa_GetDefaultInputDevice(); /* default input device */
if (this->inputParameters.device == paNoDevice) {
std::cout << "Error: No default input device" << std::endl;
exit(1);
}
this->inputParameters.channelCount = 1; /* stereo input */
this->inputParameters.sampleFormat = PA_SAMPLE_TYPE;
this->inputParameters.suggestedLatency = Pa_GetDeviceInfo( this->inputParameters.device )->defaultLowInputLatency;
this->inputParameters.hostApiSpecificStreamInfo = NULL;
std::cout << "Device name: " <<Pa_GetDeviceInfo(this->inputParameters.device)->name << std::endl;
std::cout << "Max inputChannels: " <<Pa_GetDeviceInfo(this->inputParameters.device)->maxInputChannels << std::endl;
}
int record::recordCallback(const void *inputBuffer, void *outputBuffer,
unsigned long framesPerBuffer,
const PaStreamCallbackTimeInfo* timeInfo,
PaStreamCallbackFlags statusFlags, void *userData)
{
std::cout << "Callback called" << std::endl;
this->data = (paTestData&) userData;
const SAMPLE *rptr = (const SAMPLE*)inputBuffer;
SAMPLE *wptr = &this->data.recordedSamples[this->data.frameIndex * NUM_CHANNELS];
long framesToCalc;
long i;
int finished;
unsigned long framesLeft = this->data.maxFrameindex - this->data.frameIndex;
(void) outputBuffer; /* Prevent unused variable warnings. */
(void) timeInfo;
(void) statusFlags;
//(void) userData;
if( framesLeft < framesPerBuffer )
{
framesToCalc = framesLeft;
finished = paComplete;
}
else
{
framesToCalc = framesPerBuffer;
finished = paContinue;
}
if( inputBuffer == NULL )
{
for(int i=0; i<framesToCalc; i++ )
{
*wptr++ = SAMPLE_SILENCE; /* left */
if( NUM_CHANNELS == 2 ) *wptr++ = SAMPLE_SILENCE; /* right */
}
}
else
{
for(int i=0; i<framesToCalc; i++ )
{
*wptr++ = *rptr++; /* left */
if( NUM_CHANNELS == 2 ) *wptr++ = *rptr++; /* right */
}
}
this->data.frameIndex += framesToCalc;
return finished;
}
void record::start_record()
{
err = Pa_OpenStream(
&this->stream,
&this->inputParameters,
NULL, /* &outputParameters, */
SAMPLE_RATE,
FRAMES_PER_BUFFER,
paClipOff, /* we won't output out of range samples so don't bother clipping them */
&record::recordCallbackSub,
this );
if( err != paNoError )
{
std::cout << "Something wrong - open_stream check" << std::endl;
std::cout << "PortAudio error: "<< Pa_GetErrorText( err ) << std::endl;
exit(1);
}
this->err = Pa_StartStream( this->stream );
if( err != paNoError )
{
std::cout << "Something wrong in stream check" << std::endl;
std::cout << "PortAudio error: "<< Pa_GetErrorText( err ) << std::endl;
exit(1);
}
std::cout << "Waiting for playback to finish" << std::endl;
while( ( err = Pa_IsStreamActive( stream ) ) == 1 )
{
Pa_Sleep(1000);
printf("index = %d\n", this->data.frameIndex ); fflush(stdout);
}
if( err < 0 )
{
std::cout << "error check with isStreamActive - something wrong" << std::endl;
std::cout << "PortAudio error: "<< Pa_GetErrorText( err ) << std::endl;
exit(1);
}
err = Pa_CloseStream( stream );
if( err != paNoError )
{
std::cout << "error check with close_stream- something wrong" << std::endl;
std::cout << "PortAudio error: "<< Pa_GetErrorText( err ) << std::endl;
exit(1);
}
std::cout << "Number of entries: " << sizeof(this->data.recordedSamples)/sizeof(this->data.recordedSamples[0]) << std::endl;
/* Measure maximum peak amplitude. */
max = 0;
average = 0.0;
for(int i=0; i<numSamples; i++ )
{
val = this->data.recordedSamples[i];
std::cout << "i: " << i << " : "<< val << std::endl;
std::this_thread::sleep_for(std::chrono::milliseconds(100));
if( val < 0 ) val = -val; /* ABS */
if( val > max )
{
max = val;
}
average += val;
}
average = average / (double)numSamples;
std::cout<<"sample max amplitude = " << max << std::endl;
std::cout<<"sample average = " << average << std::endl;
if (WRITE_TO_FILE)
{
FILE *fid;
fid = fopen("recorded.wav", "wb");
if( fid == NULL )
{
printf("Could not open file.");
}
else
{
fwrite( data.recordedSamples, NUM_CHANNELS * sizeof(SAMPLE), totalFrames, fid );
fclose( fid );
printf("Wrote data to 'recorded.raw'\n");
}
}
std::cout << "Everythin done!" << std::endl;
}
Update:
I've from some debugging notices that the callback is only being called once, and after it returns, becomes the stream inactive, hence making the it impossible to make calls to the callback function. Why does the stream become inactive?
Your first callback
static int recordCallbackSub(const void *inputBuffer, void *outputBuffer,
unsigned long framesPerBuffer, const PaStreamCallbackTimeInfo *timeInfo,
PaStreamCallbackFlags statusFlags, void *userData)
{
auto pThis = reinterpret_cast<record*>(userData); // get back the this pointer.
return pThis->recordCallback( inputBuffer, outputBuffer,framesPerBuffer, timeInfo,statusFlags, nullptr);
}
calls your second callback
int record::recordCallback(const void *inputBuffer, void *outputBuffer,
unsigned long framesPerBuffer,
const PaStreamCallbackTimeInfo* timeInfo,
PaStreamCallbackFlags statusFlags, void *userData)
{
std::cout << "Callback called" << std::endl;
this->data = (paTestData&) userData;
....
}
with the userData parameter set to nullptr. You then cast the nullptr into a paTestData& and set your data member variable to the result, which I suspect you didn't intend to do.
Delete the this->data = (paTestData&) userData; line.
I'm trying to connect via boost ssl socket to aws s3.
It works but when I read, I had several problem.
Corrupted files on the same file but not others.
File not corrupted (md5filter got all the data) but the data sent to the buffer are not good. Meaning there is a problem between the different layers of read somewhere but can't figure out where.
Sometimes the program get stuck in the S3_client::read function and loop thousands of times in the do-while loop calling read. But it never reaches md5filter read.
It get stuck between filterStream.read() and md5filter.read() which is not called. I don't know if it gzip or filterStream. But it only happens if there is no call to the lower layers of read for a while.
Can you help spot the problem in my code ?
#ifndef BTLOOP_AWSCLIENT_H
#define BTLOOP_AWSCLIENT_H
#include "boost/iostreams/filter/gzip.hpp"
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filtering_streambuf.hpp>
#include <boost/iostreams/categories.hpp>
#include <boost/iostreams/stream.hpp>
#include <string>
#include <set>
#include <map>
#include <openssl/md5.h>
#include <sstream>
#include <fstream>
#include <iostream>
#include <boost/asio.hpp>
#include <boost/asio/ssl.hpp>
#include "Logger.h"
namespace io = boost::iostreams;
namespace asio = boost::asio;
namespace ssl = boost::asio::ssl;
typedef ssl::stream<asio::ip::tcp::socket> ssl_socket;
namespace S3Reader
{
class MD5Filter
{
public:
typedef char char_type;
struct category :
io::multichar_input_filter_tag{};
MD5Filter( std::streamsize n );
~MD5Filter();
template<typename Source>
std::streamsize read( Source& src, char* s, std::streamsize n );
void setBigFileMode() { _bigFileMode = true; }
std::string close();
void setFileName( std::string fileName ) { _fileName = fileName; };
inline std::streamsize writtenBytes() {std::streamsize res = _writtenBytes; _writtenBytes= 0; return res;};
inline bool eof(){return _eof;};
private:
void computeMd5( char* buffer, size_t size, bool force = false );
private:
bool _bigFileMode;
int _blockCount;
std::vector<unsigned char> _bufferMD5;
MD5_CTX _mdContext;
unsigned char _hashMd5[MD5_DIGEST_LENGTH];
std::string _fileName;
std::streamsize _writtenBytes;
int _totalSize;
bool _eof;
};
class Ssl_wrapper : public io::device<io::bidirectional>
{
public:
Ssl_wrapper( ssl_socket* sock, std::streamsize n ) :
_sock( sock ),_totalSize(0) { };
std::streamsize read( char_type* s, std::streamsize n )
{
boost::system::error_code ec;
size_t rval = _sock->read_some( asio::buffer( s, n ), ec );
_totalSize +=rval;
LOG_AUDIT( " wrapperR: " << rval << " " << _totalSize << " "<<ec.message());
if ( !ec )
{
return rval;
}
else if ( ec == asio::error::eof )
return -1;
else
throw boost::system::system_error( ec, "Wrapper read_some" );
}
std::streamsize write( const char* s, std::streamsize n )
{
boost::system::error_code ec;
size_t rval = _sock->write_some( asio::buffer( s, n ), ec );
if ( !ec )
{
return rval;
}
else if ( ec == asio::error::eof )
return -1;
else
throw boost::system::system_error( ec, " Wrapper read_some" );
}
private:
ssl_socket* _sock;
int _totalSize;
};
class S3_client
{
public:
S3_client( const std::string& key_id, const std::string& key_secret, const std::string& bucket );
virtual ~S3_client();
bool open( const std::string& fileName );
int read( char* buffer, size_t size );
int readLine( char* buffer, size_t size );
void close();
bool eof() { return _filterStream.eof(); }
std::string authorize( const std::string request );
bool connectSocket( std::string url, std::string port, std::string auth );
private :
std::string _key_id;
std::string _key_secret;
std::string _bucket;
std::string _fileName;
io::gzip_decompressor _gzip;
MD5Filter _md5Filter;
boost::posix_time::seconds _timeout;
ssl_socket* _sock;
Ssl_wrapper* _wrapper;
io::stream<Ssl_wrapper>* _sockstream;
std::map<std::string, std::string> _headerMap;
io::filtering_istream _filterStream;
int _totalSize;
};
}
#endif //BTLOOP_AWSCLIENT_H
S3Client.cpp
#include "S3_client.h"
#include <boost/algorithm/string.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/iostreams/copy.hpp>
#include <boost/iostreams/filter/counter.hpp>
#include <boost/exception/diagnostic_information.hpp>
#include <system/ArmError.h>
namespace io = boost::iostreams;
namespace asio = boost::asio;
namespace ssl = boost::asio::ssl;
namespace S3Reader
{
static const size_t s3_block_size = 8 * 1024 * 1024;
static const std::string base64_chars =
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz"
"0123456789+/";
static inline bool is_base64( unsigned char c )
{
return (isalnum( c ) || (c == '+') || (c == '/'));
}
std::string url_encode( const std::string& value )
{
std::ostringstream escaped;
escaped.fill( '0' );
escaped << std::hex;
for ( auto i = value.begin(), n = value.end(); i != n; ++i )
{
auto c = *i;
if ( isalnum( c ) || c == '-' || c == '_' || c == '.' || c == '~' )
{
escaped << c;
continue;
}
escaped << std::uppercase;
escaped << '%' << std::setw( 2 ) << int((unsigned char) c );
escaped << std::nouppercase;
}
return escaped.str();
}
std::string base64_encode( unsigned char const* bytes_to_encode, unsigned int in_len )
{
std::string ret;
int i = 0;
int j = 0;
unsigned char char_array_3[3];
unsigned char char_array_4[4];
while ( in_len-- )
{
char_array_3[i++] = *(bytes_to_encode++);
if ( i == 3 )
{
char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;
char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);
char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);
char_array_4[3] = char_array_3[2] & 0x3f;
for ( i = 0; (i < 4); i++ )
ret += base64_chars[char_array_4[i]];
i = 0;
}
}
if ( i )
{
for ( j = i; j < 3; j++ )
char_array_3[j] = '\0';
char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;
char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);
char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);
char_array_4[3] = char_array_3[2] & 0x3f;
for ( j = 0; (j < i + 1); j++ )
ret += base64_chars[char_array_4[j]];
while ((i++ < 3))
ret += '=';
}
return ret;
}
std::string to_hex( const uint8_t* buffer, size_t buffer_size )
{
std::stringstream sst;
for ( uint i = 0; i < buffer_size; i++ )
{
sst << std::setw( 2 ) << std::setfill( '0' ) << std::hex << int( buffer[i] );
}
return sst.str();
}
std::string getDateForHeader( bool amzFormat )
{
time_t lt;
time( < );
struct tm* tmTmp;
tmTmp = gmtime( < );
char buf[50];
if ( amzFormat )
{
strftime( buf, 50, "Date: %a, %d %b %Y %X +0000", tmTmp );
return std::string( buf );
}
else
{
tmTmp->tm_hour++;
//strftime( buf, 50, "%a, %d %b %Y %X +0000", tmTmp );
std::stringstream ss;
ss << mktime( tmTmp );
return ss.str();
}
}
MD5Filter::MD5Filter( std::streamsize n ) :
_bigFileMode( false ), _blockCount( 0 ), _writtenBytes(0), _totalSize( 0 )
{
MD5_Init( &_mdContext );
memset( _hashMd5, 0, MD5_DIGEST_LENGTH );
}
MD5Filter::~MD5Filter()
{
close();
}
template<typename Source>
std::streamsize MD5Filter::read( Source& src, char* s, std::streamsize n )
{
int result =0;
try
{
if ((result = io::read( src, s, n )) == -1 )
{
_eof=true;
LOG_AUDIT( _fileName << " md5R: " << result << " " << _totalSize );
return -1;
}
}
catch ( boost::exception& ex)
{
LOG_ERROR( _fileName <<" "<< boost::diagnostic_information(ex)<< " " << result );
}
computeMd5( s, (size_t) result );
_totalSize += result;
_writtenBytes = result;
LOG_AUDIT( _fileName << " md5R: " << result << " " << _totalSize );
return result;
}
void MD5Filter::computeMd5( char* buffer, size_t size, bool force )
{
size_t realSize = s3_block_size;
uint8_t blockMd5[MD5_DIGEST_LENGTH];
if ( !_bigFileMode )
{
MD5_Update( &_mdContext, buffer, size );
return;
}
if ( size > 0 )
{
_bufferMD5.insert( _bufferMD5.end(), &buffer[0], &buffer[size] );
}
if ((_bufferMD5.size() < s3_block_size) && !force )
return;
if ( force )
realSize = _bufferMD5.size();
MD5( &_bufferMD5[0], realSize, blockMd5 );
MD5_Update( &_mdContext, blockMd5, MD5_DIGEST_LENGTH );
_blockCount++;
if ( _bufferMD5.size() == s3_block_size )
{
_bufferMD5.clear();
return;
}
if ( force )
return;
memcpy( &_bufferMD5[0], &_bufferMD5[s3_block_size], _bufferMD5.size() - s3_block_size );
_bufferMD5.erase( _bufferMD5.begin() + s3_block_size, _bufferMD5.end());
}
std::string MD5Filter::close()
{
std::string mdOutput;
computeMd5( NULL, 0, true );
MD5_Final( _hashMd5, &_mdContext );
mdOutput = to_hex( _hashMd5, MD5_DIGEST_LENGTH );
if ( _bigFileMode )
{
mdOutput += "-" + boost::lexical_cast<std::string>( _blockCount );
}
return mdOutput;
}
std::string S3_client::authorize( const std::string request )
{
unsigned char* digest;
digest = HMAC( EVP_sha1(), _key_secret.c_str(), (int) _key_secret.size(), (unsigned char*) request.c_str(), (int) request.size(), NULL, NULL );
std::string signature( url_encode( base64_encode( digest, 20 )));
return "?AWSAccessKeyId=" + _key_id + "&Expires=" + getDateForHeader( false ) + "&Signature=" + signature;
}
S3_client::S3_client( const std::string& key_id, const std::string& key_secret, const std::string& bucket ) :
_key_id( key_id ), _key_secret( key_secret ), _bucket( bucket ), _gzip( io::gzip::default_window_bits, 1024 * 1024 )
, _md5Filter( s3_block_size ), _timeout( boost::posix_time::seconds( 1 )), _totalSize( 0 ) { }
S3_client::~S3_client()
{
close();
}
bool S3_client::connectSocket( std::string url, std::string port, std::string auth )
{
std::string amzDate = getDateForHeader( true );
std::string host = "url";
boost::asio::io_service io_service;
boost::asio::ip::tcp::resolver resolver( io_service );
boost::asio::ip::tcp::resolver::query query( url, "https" );
auto endpoint = resolver.resolve( query );
// Context with default path
ssl::context ctx( ssl::context::sslv23 );
ctx.set_default_verify_paths();
_sock = new ssl_socket( io_service, ctx );
boost::asio::socket_base::keep_alive option( true );
_wrapper = new Ssl_wrapper( _sock, s3_block_size );
_sockstream = new io::stream<Ssl_wrapper>( boost::ref( *_wrapper ));
asio::connect( _sock->lowest_layer(), endpoint );
_sock->set_verify_mode( ssl::verify_peer );
_sock->set_verify_callback( ssl::rfc2818_verification( url ));
_sock->handshake( ssl_socket::client );
_sock->lowest_layer().set_option( option );
std::stringstream ss;
ss << "GET " << _fileName << auth << " HTTP/1.1\r\n" << "Host: " << host << "\r\nAccept: */*\r\n\r\n";
_sockstream->write( ss.str().c_str(), ss.str().size());
_sockstream->flush();
std::string http_version;
int status_code = 0;
(*_sockstream) >> http_version;
(*_sockstream) >> status_code;
if ( !_sockstream || http_version.substr( 0, 5 ) != "HTTP/" )
{
std::cout << "Invalid response: " << http_version << " " << status_code << std::endl;
return false;
}
if ( status_code != 200 )
{
std::cout << "Response returned with status code " << http_version << " " << status_code << std::endl;
return false;
}
return true;
}
bool S3_client::open( const std::string& fileName )
{
std::string port = "443";
std::string url = "bucket";
std::stringstream authRequest;
std::string date = getDateForHeader( false );
_fileName = fileName;
authRequest << "GET\n\n\n" << date << "\n/" << _bucket << "" << fileName;
std::string auth = authorize( authRequest.str());
if ( !connectSocket( url, port, auth ))
THROW( "Failed to open socket" );
std::string header;
while ( std::getline( *_sockstream, header ) && header != "\r" )
{
std::vector<std::string> vectLine;
boost::split( vectLine, header, boost::is_any_of( ":" ));
if ( vectLine.size() < 2 )
continue;
boost::erase_all( vectLine[1], "\"" );
boost::erase_all( vectLine[1], "\r" );
boost::erase_all( vectLine[1], " " );
_headerMap[vectLine[0]] = vectLine[1];
}
if ( _headerMap.find( "Content-Length" ) == _headerMap.end())
return false;
if ( _headerMap.find( "Content-Type" ) == _headerMap.end())
return false;
if ((uint) std::atoi( _headerMap.at( "Content-Length" ).c_str()) > s3_block_size )
_md5Filter.setBigFileMode();
_md5Filter.setFileName( _fileName );
if ( _headerMap["Content-Type"] == "binary/octet-stream" )
_filterStream.push( _gzip, s3_block_size );
_filterStream.push( boost::ref( _md5Filter ), s3_block_size );
_filterStream.push( boost::ref( *_sockstream ), s3_block_size );
return true;
}
void S3_client::close()
{
std::string localMD5 = _md5Filter.close();
std::string headerMD5 = _headerMap["ETag"];
if ( localMD5 != headerMD5 )
THROW ( "Corrupted file " << _fileName << " " << localMD5 << " " << headerMD5 << "." );
else
LOG_AUDIT( "Close S3: " << _fileName << " " << localMD5 << " " << headerMD5 << "." );
}
int S3_client::readLine( char* buffer, size_t size )
{
_filterStream.getline( buffer, size );
return _filterStream.gcount();
}
int S3_client::read( char* buffer, size_t size )
{
std::streamsize sizeRead = 0;
do
{
_filterStream.read( buffer, size );
sizeRead = _md5Filter.writtenBytes();
_totalSize += sizeRead;
LOG_AUDIT( _fileName << " s3R: " << sizeRead << " " << _totalSize );
}
while( sizeRead ==0 && !_md5Filter.eof() && !_sockstream->eof() && _filterStream.good() && _sock->next_layer().is_open());
return sizeRead;
}
}
int main( int argc, char** argv )
{
S3Reader::S3_client client( key_id, key_secret, s3_bucket );
client.open("MyFile");
while (client.read(buffer, bufferSize) >0 ) {}
}
I'm compiling ZLIB directly into the executable with is being compiled on VS2010 - 64-bit and this simple program always hangs, any ideas? More info below:
Call Stack:
zlibtest.exe!inflate(z_stream_s * strm, int flush) Line 607
zlibtest.exe!uncompress(unsigned char * dest, unsigned long * destLen, const unsigned char * source, unsigned long sourceLen) Line 44
zlibtest.exe!main(int argc, char * * argv) Line 137
It hangs here:
for (;;)
switch (state->mode) {
case HEAD:
if (state->wrap == 0) {
state->mode = TYPEDO;
break;
}
**NEEDBITS(16);**
Program:
#include "zlib.h"
int main( int argc, char * argv[] )
{
char * buffer = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
ULONG buffer_size = strlen( buffer ) + 1;
ULONG compressed_size = compressBound( buffer_size );
BYTE * compressed_buffer = new BYTE[ compressed_size ];
if ( Z_OK != compress2( compressed_buffer, &compressed_size, ( Bytef * ) buffer, buffer_size, Z_DEFAULT_COMPRESSION ) )
printf( "Failed to compress." );
ULONG uncompressed_size = buffer_size;
BYTE * uncompressed_buffer = new BYTE[ uncompressed_size ];
if ( Z_OK != uncompress( uncompressed_buffer, &uncompressed_size, compressed_buffer, compressed_size ) )
printf( "Failed to uncompress" );
printf( "Press <ENTER> to exit..." );
std::cin.ignore();
}
EDIT
I've made changes to what I saw below and this is what I have
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/un.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <string>
#include <vector>
#include <iostream>
#include <cstring>
#include <cstdlib>
#include <errno.h>
using namespace std;
string buffer;
vector<string> ex;
int s;
void recvline ( int s, string* buf ) {
char in, t;
while ( 1 ) {
recv ( s, &in, 1, 0 );
*buf += in;
if ( in == 10 ) {
t = 1; }
if ( t && in == 13 ) {
break; }
}
}
void push ( int s, string msg ) {
string o = msg + "\r\n";
cout << "SENT:", o;
send ( s, o.c_str(), o.size(), 0 );
}
int main ( int argc, char *argv[] ) {
if ( argc < 3 ) {
cout << "Insufficient Arguments" << endl;
exit ( 7 ); }
s = socket ( AF_INET, SOCK_STREAM, IPPROTO_TCP );
if ( s < 0 )
exit ( 1 );
struct hostent h = *gethostbyname ( argv[1] );
struct sockaddr_in c;
c.sin_family = AF_INET;
c.sin_port = htons(atoi(argv[2]));
c.sin_addr.s_addr = inet_addr ( h.h_addr_list[0] );
if ( connect ( s, (struct sockaddr*)&c, sizeof c ) != 0 ) {
cout << "Unable to connect to network" << endl;
cout << strerror(errno) << endl;
exit ( 2 );
}
push ( s, "USER LOLwat Lw lol.wat :LOLwat" );
push ( s, "NICK LOLwat" );
while ( true ) {
recvline ( s, &buffer );
cout << buffer;
if ( buffer.substr(0,4).c_str() == "PING" )
push ( s, "PONG " + buffer.substr(6,-2) );
}
}
And this is the result:
[dbdii407#xpcd Desktop]$ g++ ?.cpp -o 4096 -
[dbdii407#xpcd Desktop]$ ./4096 irc.scrapirc.com 6667 - Unable to connect to network - Network is unreachable
I think the problem is that this line:
c.sin_port = htons(*argv[2]);
Is not doing what you think it's doing. argv[2] is a string, *argv[2] is the first character of the string. So if you passed "4567" as the second command-line argument, then *argv[2] will be '4' which has ASCII value 52. That means you'll be attempting to connect to port 52, not "4567" as you would expect.
Change the line to:
c.sin_port = htons(atoi(argv[2]));
The atoi function takes a string and converts it to an integer. So "4567" would become 4567.
Also, in general, you should check the value of errno when a function call like that fails (it'll usually tell you in the documentation whether errno is set and the possible values it can be set to). That should help to give you some clue in the future.
Edit
As others have noted, make sure you pay attention to your braces. It's usually easier if you just always use braces around if, while, and so on. That is, this:
if ( connect ( s, (struct sockaddr*)&c, sizeof c ) != 0 )
cout << "Unable to connect to network" << endl;
exit ( 2 );
Is completely different to this:
if ( connect ( s, (struct sockaddr*)&c, sizeof c ) != 0 ) {
cout << "Unable to connect to network" << endl;
exit ( 2 );
}
I decided to completely redo my answer, in part due to the following comment in the gethostbyname manpage:
The gethostbyname*() and
gethostbyaddr*() functions are
obsolete. Applications should use
getaddrinfo(3) and getnameinfo(3)
instead.
Here is the reworked program ( cleaned up a bit with bcpp ) based on using getaddrinfo. I would strongly suggest always compiling with the following options:
g++ -Wall -Wextra irc.cpp -o irc
This showed up the following bugs in your code:
irc.cpp: In function ‘void push(int, std::string)’:
irc.cpp:40: warning: right-hand operand of comma has no effect
irc.cpp: In function ‘int main(int, char**)’:
irc.cpp:87: warning: comparison with string literal results in unspecified behaviour
I went ahead and fixed the errors. Also, try and eliminate global variables as much as possible.
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/un.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <string>
#include <vector>
#include <iostream>
#include <cstring>
#include <cstdlib>
#include <errno.h>
using namespace std;
string buffer;
vector<string> ex;
void recvline ( int s, string* buf )
{
char in, t;
while ( 1 )
{
recv ( s, &in, 1, 0 );
*buf += in;
if ( in == 10 )
{
t = 1;
}
if ( t && in == 13 )
{
break;
}
}
}
void push ( int s, string msg )
{
string o = msg + "\r\n";
cout << "SENT:" << o;
send ( s, o.c_str(), o.size(), 0 );
}
int main ( int argc, char *argv[] )
{
if ( argc < 3 )
{
cout << "Insufficient Arguments" << endl;
exit ( 7 );
}
int s, sfd;
struct addrinfo *result, *rp;
s = getaddrinfo(argv[1], argv[2], NULL, &result);
if (s != 0) {
fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(s));
exit(EXIT_FAILURE);
}
for (rp = result; rp != NULL; rp = rp->ai_next) {
sfd = socket(rp->ai_family, rp->ai_socktype,
rp->ai_protocol);
if (sfd == -1)
continue;
if (connect(sfd, rp->ai_addr, rp->ai_addrlen) != -1)
break; /* Success */
close(sfd);
}
if (rp == NULL) { /* No address succeeded */
fprintf(stderr, "Could not connect\n");
exit(EXIT_FAILURE);
}
freeaddrinfo(result); /* No longer needed */
push ( sfd, "USER LOLwat Lw lol.wat :LOLwat" );
push ( sfd, "NICK LOLwat" );
while ( true )
{
recvline ( sfd, &buffer );
cout << buffer;
if ( buffer.substr(0,4) == "PING" )
push ( sfd, "PONG " + buffer.substr(6,-2) );
}
}