Winpcap saving raw packets not from an adapter - c++

I am trying to build an application that converts my old custom Ethernet logs (bin files) to standard winpcap style logs.
The problem is that I can't seem to find an example of how to opening a pcap_t* without using an adapter (network card). The temp.pkt has not been created.
I have looked thou the examples provided with Winpcap and all of them use a live adapter when dumping packets. This example is the closest \WpdPack\Examples-pcap\savedump\savedump.c is the closest, see example below slightly modified.
#ifdef _MSC_VER
/*
* we do not want the warnings about the old deprecated and unsecure CRT functions
* since these examples can be compiled under *nix as well
*/
#define _CRT_SECURE_NO_WARNINGS
#endif
#include "pcap.h"
int main(int argc, char **argv)
{
pcap_if_t *alldevs;
pcap_if_t *d;
int inum;
int i=0;
pcap_t *adhandle;
char errbuf[PCAP_ERRBUF_SIZE];
pcap_dumper_t *dumpfile;
/* Open the adapter */
if ((adhandle= pcap_open(??????, // name of the device
65536, // portion of the packet to capture.
// 65536 grants that the whole packet will be captured on all the MACs.
1, // promiscuous mode (nonzero means promiscuous)
1000, // read timeout
errbuf // error buffer
)) == NULL)
{
fprintf(stderr,"\nUnable to open the adapter. %s is not supported by WinPcap\n", d->name);
/* Free the device list */
pcap_freealldevs(alldevs);
return -1;
}
/* Open the dump file */
dumpfile = pcap_dump_open(adhandle, argv[1]);
if(dumpfile==NULL) {
fprintf(stderr,"\nError opening output file\n");
return -1;
}
// ---------------------------
struct pcap_pkthdr header;
header.ts.tv_sec = 1 ; /* seconds */
header.ts.tv_usec = 1; /* and microseconds */
header.caplen = 100; /* length of portion present */
header.len = 100 ; /* length this packet (off wire) */
u_char pkt_data[100];
for( int i = 0 ; i < 100 ; i++ ) {
pkt_data[i] = i ;
}
pcap_dump( (u_char *) dumpfile, &header, (u_char *) &pkt_data);
// ---------------------------
/* start the capture */
// pcap_loop(adhandle, 0, packet_handler, (unsigned char *)dumpfile);
pcap_close(adhandle);
return 0;
}

I suggest doing that using pcap_t since using WinPcap is better than writing it yourself.
The following steps is how to do it:
Use pcap_open_dead() function to create a pcap_t. Read the function description here. The linktype for Ethernet is 1.
Use pcap_dump_open() function to create a pcap_dumper_t.
Use pcap_dump() function to write the packet to the dump file.
I hope this would help you.

If all you're doing is converting your own file format to .pcap, you don't need a pcap_t*, you can just use something like:
FILE* create_pcap_file(const char *filename, int linktype)
{
struct pcap_file_header fh;
fh.magic = TCPDUMP_MAGIC;
fh.sigfigs = 0;
fh.version_major = 2;
fh.version_minor = 4;
fh.snaplen = 2<<15;
fh.thiszone = 0;
fh.linktype = linktype;
FILE *file = fopen(filename, "wb");
if(file != NULL) {
if(fwrite(&fh, sizeof(fh), 1, file) != 1) {
fclose(file);
file = NULL;
}
}
return file;
}
int write_pcap_packet(FILE* file,size_t length,const unsigned char *data,const struct timeval *tval)
{
struct pcap_pkthdr pkhdr;
pkhdr.caplen = length;
pkhdr.len = length;
pkhdr.ts = *tval;
if(fwrite(&pkhdr, sizeof(pkhdr), 1, file) != 1) {
return 1;
}
if(fwrite(data, 1, length, file) != length) {
return 2;
}
return 0;
}

Related

parsing complete messages from serial port

I am trying to read complete messages from my GPS via serial port.
The message I am looking for starts with:
0xB5 0x62 0x02 0x13
So I read from the serial port like so
while (running !=0)
{
int n = read (fd, input_buffer, sizeof input_buffer);
for (int i=0; i<BUFFER_SIZE; i++)
{
if (input_buffer[i]==0xB5 && input_buffer[i+1]== 0x62 && input_buffer[i+2]== 0x02 && input_buffer[i+3]== 0x13 && i<(BUFFER_SIZE-1) )
{
// process the message.
}
}
The problem I am having is that I need to get a complete message. Half of a message could be in the buffer one iteration. And the other half could come into the message the next iteration.
Somebody suggested that free the buffer up from the complete message. And then I move the rest of data in the buffer to the beginning of the buffer.
How do I do that or any other way that make sure I get every complete selected message that comes in?
edit//
I want a particular class and ID. But I can also read in the length
To minimize the overhead of making many read() syscalls of small byte counts, use an intermediate buffer in your code.
The read()s should be in blocking mode to avoid a return code of zero bytes.
#define BLEN 1024
unsigned char rbuf[BLEN];
unsigned char *rp = &rbuf[BLEN];
int bufcnt = 0;
static unsigned char getbyte(void)
{
if ((rp - rbuf) >= bufcnt) {
/* buffer needs refill */
bufcnt = read(fd, rbuf, BLEN);
if (bufcnt <= 0) {
/* report error, then abort */
}
rp = rbuf;
}
return *rp++;
}
For proper termios initialization code for the serial terminal, see this answer. You should increase the VMIN parameter to something closer to the BLEN value.
Now you can conveniently access the received data a byte at a time with minimal performance penalty.
#define MLEN 1024 /* choose appropriate value for message protocol */
unsigned char mesg[MLEN];
while (1) {
while (getbyte() != 0xB5)
/* hunt for 1st sync */ ;
retry_sync:
if ((sync = getbyte()) != 0x62) {
if (sync == 0xB5)
goto retry_sync;
else
continue; /* restart sync hunt */
}
class = getbyte();
id = getbyte();
length = getbyte();
length += getbyte() << 8;
if (length > MLEN) {
/* report error, then restart sync hunt */
continue;
}
for (i = 0; i < length; i++) {
mesg[i] = getbyte();
/* accumulate checksum */
}
chka = getbyte();
chkb = getbyte();
if ( /* valid checksum */ )
break; /* verified message */
/* report error, and restart sync hunt */
}
/* process the message */
switch (class) {
case 0x02:
if (id == 0x13) {
...
...
You can break the read into three parts. Find the start of a message. Then get the LENGTH. Then read the rest of the message.
// Should probably clear these in case data left over from a previous read
input_buffer[0] = input_buffer[1] = 0;
// First make sure first char is 0xB5
do {
n = read(fd, input_buffer, 1);
} while (0xB5 != input_buffer[0]);
// Check for 2nd sync char
n = read(fd, &input_buffer[1], 1);
if (input_buffer[1] != 0x62) {
// Error
return;
}
// Read up to LENGTH
n = read(fd, &input_buffer[2], 4);
// Parse length
//int length = *((int *)&input_buffer[4]);
// Since I don't know what size an int is on your system, this way is better
int length = input_buffer[4] | (input_buffer[5] << 8);
// Read rest of message
n = read(fd, &input_buffer[6], length);
// input_buffer should now have a complete message
You should add error checking...

Extract raw audio data from WAV files using libsndfile in C++

I am working on an application that will provide audio input to some device. The device expects the audio input to be provided in the form of raw audio data stream (16 bit, 48kHz). So, irrespective of the format of audio data in the wave file (8-bit, 16-bit, 24-bit, 32-bit etc.), I want to extract raw audio data from the WAV file. I planned to use libsndFile library for this purpose. I modified the C++ sample code of libsndfile as shown below:
#include "stdafx.h"
#include <sndfile.hh>
static void create_file (const char * fname, int format, const short* buffer,const unsigned int& len)
{
// file ;
int channels = 1 ; //A Mono wave file.
int srate = 48000 ;
printf ("Creating file named '%s'\n", fname) ;
SndfileHandle file = SndfileHandle (fname, SFM_WRITE, format, channels, srate) ;
int x = file.write (buffer, len) ;
}
static void read_file (const char * fname)
{
SndfileHandle file ;
file = SndfileHandle (fname) ;
const unsigned int uiBuffLen = file.channels() * file.frames();
short* data = new short [uiBuffLen] ;
memset(data,0x00,uiBuffLen);
int x = file.command(SFC_SET_SCALE_FLOAT_INT_READ, (void*)data, uiBuffLen);
file.read (data, uiBuffLen) ; //Read the audio data in the form of 16 bit short integer
//Now create a new wave file with audio data in the form of 16 bit short integers
create_file ("ConvertedFile.wav", SF_FORMAT_WAV | SF_FORMAT_PCM_16,data, (const unsigned int&)uiBuffLen) ;
//Now fill a buffer containing audio data and dump it into a file so that the same can be fed to a device expecting the raw audio data
unsigned char* bytBuffer = new unsigned char[uiBuffLen*2];
memset(bytBuffer, 0x00, uiBuffLen*2);
file.readRaw(bytBuffer, uiBuffLen*2);
FILE * pFile;
pFile = fopen ("RawAudio.dat","w");
if (pFile!=NULL)
{
fwrite(bytBuffer, 1, uiBuffLen*2, pFile);
fclose (pFile);
}
delete [] data;
delete [] bytBuffer;
}
int _tmain(int argc, _TCHAR* argv[])
{
//The sample file is a Mono file containing audio data in float format.
const char * fname = "MonoWavFile.wav" ;
read_file (fname) ;
return 0;
}
Well, the above code might look horrible, but I am just looking for the idea at the moment. I use a file "MonoWaveFile.wav" which is a mono wave file and has audio data in the form of 32 bit float values.
I create a new file "ConvertedFile.wav" using the libsndfile library. This file has audio data in 16-bit PCM format. I play this file in a media player and I see that the conversion has been done properly.
Then I create another file "RawAudio.dat" to save only the audio data, which I can use to feed the audio input to the device. The file is created and when I send it to the device, the audio is not proper at all. This indicates that I am doing somthing horribly wrong. Can any one let me know what wrong I am doing? I have never worked on anything like this before, so I will appreciate if I get any sort of help.
I used sf_open_virtual.
I made another buffer which I have filled using sf_open_virtual. Then I used this "another buffer" to feed RTP packets.
So I guess this is what you need.
I have problems making it the other directions from RTP packet to file.
//============================================================================
// Name : libsndfile_demo.cpp
// Author :
// Version :
// Copyright : Your copyright notice
// Description : Hello World in C++, Ansi-style
//============================================================================
#include <iostream>
#include <string.h>
#include "sndfile.h"
#include <assert.h>
#include <unistd.h>
using namespace std;
typedef struct
{ sf_count_t offset, length ;
unsigned char data [160] ;
} VIO_DATA ;
FILE *checker;
const void* old_ptr = NULL;
static sf_count_t vfget_filelen (void *user_data)
{
VIO_DATA *vf = (VIO_DATA *) user_data ;
return vf->length ;
} /* vfget_filelen */
static sf_count_t vfseek (sf_count_t offset, int whence, void *user_data)
{
VIO_DATA *vf = (VIO_DATA *) user_data ;
switch (whence)
{ case SEEK_SET :
vf->offset = offset ;
break ;
case SEEK_CUR :
vf->offset = vf->offset + offset ;
break ;
case SEEK_END :
vf->offset = vf->length + offset ;
break ;
default :
break ;
} ;
return vf->offset ;
} /* vfseek */
static sf_count_t vfread (void *ptr, sf_count_t count, void *user_data)
{
VIO_DATA *vf = (VIO_DATA *) user_data ;
/*
** This will brack badly for files over 2Gig in length, but
** is sufficient for testing.
*/
if (vf->offset + count > vf->length)
count = vf->length - vf->offset ;
memcpy (ptr, vf->data + vf->offset, count) ;
vf->offset += count ;
return count ;
} /* vfread */
static sf_count_t vfwrite (const void *ptr, sf_count_t count, void *user_data)
{
static int skip = 0;
//TODO: Why this is working ?!?!?!
if (skip < 1)
{
skip++;
return count;
}
//SendTo RTP packetizer instead of writing to file
fwrite(ptr, count, 1, checker);
return count ;
} /* vfwrite */
static sf_count_t vftell (void *user_data)
{
VIO_DATA *vf = (VIO_DATA *) user_data ;
return vf->offset ;
} /* vftell */
int main()
{
SF_INFO writing_sfinfo;
writing_sfinfo.channels = 1;
writing_sfinfo.format = SF_FORMAT_WAV | SF_FORMAT_GSM610;
writing_sfinfo.samplerate = 8000;
assert(true == sf_format_check(&writing_sfinfo));
SF_INFO reading_sfinfo;
memset(&reading_sfinfo, 0, sizeof(reading_sfinfo));
SNDFILE *input = sf_open("/home/georgi/Downloads/thank_you_60.PCMA", SFM_READ, &reading_sfinfo);
checker = fopen("/home/georgi/Downloads/checker.wav", "w+");
short file_data[reading_sfinfo.channels * 160];
int read_frames = 0;
SF_VIRTUAL_IO vio ;
// Set up pointers to the locally defined functions.
vio.get_filelen = vfget_filelen ;
vio.seek = vfseek ;
vio.read = vfread ;
vio.write = vfwrite ;
vio.tell = vftell ;
VIO_DATA vio_data ;
// Set virtual file offset and length to zero.
vio_data.offset = 0 ;
vio_data.length = 0 ;
SNDFILE *virt_file = sf_open_virtual (&vio, SFM_WRITE, &writing_sfinfo, &vio_data);
int old_length = 0;
while ((read_frames = sf_readf_short(input, file_data, 160)))
{
sf_writef_short(virt_file, file_data, read_frames);
}
sf_close(virt_file);
sf_close(input);
fclose(checker);
return 0;
}

Convert raw PCM to FLAC?

EDIT: I've updated the code below to resemble the progress I have made. I'm trying to write the .wav header myself. The code does not work properly as of now, the audio is not being written to the file properly. The code does not contain any attempts to convert it to a .flac file yet.
I am using a Raspberry Pi (Debian Linux) to record audio with the ALSA library. The recording works fine, but I need to encode the input audio into the FLAC codec.
This is where I get lost. I have spent a considerable amount of time trying to figure out how to convert this raw data into FLAC, but I keep coming up with examples of how to convert .wav files into .flac files.
Here is the current (updated) code I have for recording audio with ALSA (it may be a bit rough, I'm still picking up C++):
// Use the newer ALSA API
#define ALSA_PCM_NEW_HW_PARAMS_API
#include <alsa/asoundlib.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
struct Riff
{
char chunkId[4]; // "RIFF" (assuming char is 8 bits)
int chunkSize; // (assuming int is 32 bits)
char format[4]; // "WAVE"
};
struct Format
{
char chunkId[4]; // "fmt "
int chunkSize;
short format; // assuming short is 16 bits
short numChannels;
int sampleRate;
int byteRate;
short align;
short bitsPerSample;
};
struct Data
{
char chunkId[4]; // "data"
int chunkSize; // length of data
char* data;
};
struct Wave // Actual structure of a PCM WAVE file
{
Riff riffHeader;
Format formatHeader;
Data dataHeader;
};
int main(int argc, char *argv[])
{
void saveWaveFile(struct Wave *waveFile);
long loops;
int rc;
int size;
snd_pcm_t *handle;
snd_pcm_hw_params_t *params;
unsigned int sampleRate = 44100;
int dir;
snd_pcm_uframes_t frames;
char *buffer;
char *device = (char*) "plughw:1,0";
//char *device = (char*) "default";
printf("Capture device is %s\n", device);
/* Open PCM device for recording (capture). */
rc = snd_pcm_open(&handle, device, SND_PCM_STREAM_CAPTURE, 0);
if (rc < 0)
{
fprintf(stderr, "Unable to open PCM device: %s\n", snd_strerror(rc));
exit(1);
}
/* Allocate a hardware parameters object. */
snd_pcm_hw_params_alloca(&params);
/* Fill it in with default values. */
snd_pcm_hw_params_any(handle, params);
/* Set the desired hardware parameters. */
/* Interleaved mode */
snd_pcm_hw_params_set_access(handle, params, SND_PCM_ACCESS_RW_INTERLEAVED);
/* Signed 16-bit little-endian format */
snd_pcm_hw_params_set_format(handle, params, SND_PCM_FORMAT_S16_LE);
/* Two channels (stereo) */
snd_pcm_hw_params_set_channels(handle, params, 2);
/* 44100 bits/second sampling rate (CD quality) */
snd_pcm_hw_params_set_rate_near(handle, params, &sampleRate, &dir);
/* Set period size to 32 frames. */
frames = 32;
snd_pcm_hw_params_set_period_size_near(handle, params, &frames, &dir);
/* Write the parameters to the driver */
rc = snd_pcm_hw_params(handle, params);
if (rc < 0)
{
fprintf(stderr, "Unable to set HW parameters: %s\n", snd_strerror(rc));
exit(1);
}
/* Use a buffer large enough to hold one period */
snd_pcm_hw_params_get_period_size(params, &frames, &dir);
size = frames * 4; /* 2 bytes/sample, 2 channels */
buffer = (char *) malloc(size);
/* We want to loop for 5 seconds */
snd_pcm_hw_params_get_period_time(params, &sampleRate, &dir);
loops = 5000000 / sampleRate;
while (loops > 0)
{
loops--;
rc = snd_pcm_readi(handle, buffer, frames);
if (rc == -EPIPE)
{
/* EPIPE means overrun */
fprintf(stderr, "Overrun occurred.\n");
snd_pcm_prepare(handle);
} else if (rc < 0)
{
fprintf(stderr, "Error from read: %s\n", snd_strerror(rc));
} else if (rc != (int)frames)
{
fprintf(stderr, "Short read, read %d frames.\n", rc);
}
if (rc != size) fprintf(stderr, "Short write: wrote %d bytes.\n", rc);
}
Wave wave;
strcpy(wave.riffHeader.chunkId, "RIFF");
wave.riffHeader.chunkSize = 36 + size;
strcpy(wave.riffHeader.format, "WAVE");
strcpy(wave.formatHeader.chunkId, "fmt");
wave.formatHeader.chunkSize = 16;
wave.formatHeader.format = 1; // PCM, other value indicates compression
wave.formatHeader.numChannels = 2; // Stereo
wave.formatHeader.sampleRate = sampleRate;
wave.formatHeader.byteRate = sampleRate * 2 * 2;
wave.formatHeader.align = 2 * 2;
wave.formatHeader.bitsPerSample = 16;
strcpy(wave.dataHeader.chunkId, "data");
wave.dataHeader.chunkSize = size;
wave.dataHeader.data = buffer;
saveWaveFile(&wave);
snd_pcm_drain(handle);
snd_pcm_close(handle);
free(buffer);
return 0;
}
void saveWaveFile(struct Wave *waveFile)
{
FILE *file = fopen("test.wav", "wb");
size_t written;
if (file == NULL)
{
fprintf(stderr, "Cannot open file for writing.\n");
exit(1);
}
written = fwrite(waveFile, sizeof waveFile[0], 1, file);
fclose(file);
if (written < 1);
{
fprintf(stderr, "Writing to file failed, error %d.\n", written);
exit(1);
}
}
How would I go about converting the PCM data into the FLAC and save it to disk for later use? I have downloaded libflac-dev already and just need an example to go off of.
The way I am doing it right now:
./capture > test.raw // or ./capture > test.flac
The way it should be (program does everything for me):
./capture
If I understand the FLAC::Encoder::File documentation, you can do something like
#include <FLAC++/encoder.h>
FLAC::Encoder::File encoder;
encoder.init("outfile.flac");
encoder.process(buffer, samples);
encoder.finish();
where buffer is an array (of size samples) of 32-bit integer pointers.
Unfortunately, I know next to nothing about audio encoding so I can't speak for any other options. Good luck!
Please refer to the below code :
FLAC Encoder Test Code
This example is using a wav file as an input and then encodes it into FLAC.
As I understand, there is no major difference b/w WAV file and your RAW data, I think you can modify this code to directly read the "buffer" and convert it. You already have all the related information (Channel/Bitrate etc) so it should not be much of a problem to remove the WAV header reading code.
Please note: this is a modified version of the Flac Encoder sample from their git repo.
It includes some comments and hints on how to change it to OP's requirements, entire source for this will be a little bit long.
And do note that this is the C API, which tends to be a bit more complex than the C++ one. But it is fairly easy to convert between the two once you get the idea.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "share/compat.h"
#include "FLAC/metadata.h"
#include "FLAC/stream_encoder.h"
/* this call back is what tells your program the progress that the encoder has made */
static void progress_callback(const FLAC__StreamEncoder *encoder, FLAC__uint64 bytes_written, FLAC__uint64 samples_written, unsigned frames_written, unsigned total_frames_estimate, void *client_data);
#define READSIZE 1024
static unsigned total_samples = 0; /* can use a 32-bit number due to WAVE size limitations */
/* buffer is where we record to, in your case what ALSA writes to */
/* Note the calculation here to take the total bytes that the buffer takes */
static FLAC__byte buffer[READSIZE/*samples*/ * 2/*bytes_per_sample*/ * 2/*channels*/];
/* pcm is input to FLAC encoder */
/* the PCM data should be here, bps is 4 here...but we are allocating ints! */
static FLAC__int32 pcm[READSIZE/*samples*/ * 2/*channels*/];
int main(int argc, char *argv[])
{
FLAC__bool ok = true;
FLAC__StreamEncoder *encoder = 0;
FLAC__StreamEncoderInitStatus init_status;
FLAC__StreamMetadata *metadata[2];
FLAC__StreamMetadata_VorbisComment_Entry entry;
FILE *fin;
unsigned sample_rate = 0;
unsigned channels = 0;
unsigned bps = 0;
if((fin = fopen(argv[1], "rb")) == NULL) {
fprintf(stderr, "ERROR: opening %s for output\n", argv[1]);
return 1;
}
/* set sample rate, bps, total samples to encode here, these are dummy values */
sample_rate = 44100;
channels = 2;
bps = 16;
total_samples = 5000;
/* allocate the encoder */
if((encoder = FLAC__stream_encoder_new()) == NULL) {
fprintf(stderr, "ERROR: allocating encoder\n");
fclose(fin);
return 1;
}
ok &= FLAC__stream_encoder_set_verify(encoder, true);
ok &= FLAC__stream_encoder_set_compression_level(encoder, 5);
ok &= FLAC__stream_encoder_set_channels(encoder, channels);
ok &= FLAC__stream_encoder_set_bits_per_sample(encoder, bps);
ok &= FLAC__stream_encoder_set_sample_rate(encoder, sample_rate);
ok &= FLAC__stream_encoder_set_total_samples_estimate(encoder, total_samples);
/* sample adds meta data here I've removed it for clarity */
/* initialize encoder */
if(ok) {
/* client data is whats the progress_callback is called with, any objects you need to update on callback can be passed thru this pointer */
init_status = FLAC__stream_encoder_init_file(encoder, argv[2], progress_callback, /*client_data=*/NULL);
if(init_status != FLAC__STREAM_ENCODER_INIT_STATUS_OK) {
fprintf(stderr, "ERROR: initializing encoder: %s\n", FLAC__StreamEncoderInitStatusString[init_status]);
ok = false;
}
}
/* read blocks of samples from WAVE file and feed to encoder */
if(ok) {
size_t left = (size_t)total_samples;
while(ok && left) {
/* record using ALSA and set SAMPLES_IN_BUFFER */
/* convert the packed little-endian 16-bit PCM samples from WAVE into an interleaved FLAC__int32 buffer for libFLAC */
/* why? because bps=2 means that we are dealing with short int(16 bit) samples these are usually signed if you do not explicitly say that they are unsigned */
size_t i;
for(i = 0; i < SAMPLES_IN_BUFFER*channels; i++) {
/* THIS. this isn't the only way to convert between formats, I do not condone this because at first the glance the code seems like it's processing two channels here, but it's not it's just copying 16bit data to an int array, I prefer to use proper type casting, none the less this works so... */
pcm[i] = (FLAC__int32)(((FLAC__int16)(FLAC__int8)buffer[2*i+1] << 8) | (FLAC__int16)buffer[2*i]);
}
/* feed samples to encoder */
ok = FLAC__stream_encoder_process_interleaved(encoder, pcm, SAMPLES_IN_BUFFER);
left-=SAMPLES_IN_BUFFER;
}
}
ok &= FLAC__stream_encoder_finish(encoder);
fprintf(stderr, "encoding: %s\n", ok? "succeeded" : "FAILED");
fprintf(stderr, " state: %s\n", FLAC__StreamEncoderStateString[FLAC__stream_encoder_get_state(encoder)]);
FLAC__stream_encoder_delete(encoder);
fclose(fin);
return 0;
}
/* the updates from FLAC's encoder system comes here */
void progress_callback(const FLAC__StreamEncoder *encoder, FLAC__uint64 bytes_written, FLAC__uint64 samples_written, unsigned frames_written, unsigned total_frames_estimate, void *client_data)
{
(void)encoder, (void)client_data;
fprintf(stderr, "wrote %" PRIu64 " bytes, %" PRIu64 "/%u samples, %u/%u frames\n", bytes_written, samples_written, total_samples, frames_written, total_frames_estimate);
}

How do I extract a user stream from a WinDbg extension?

I have embedded a custom stream in a dump (i.e. passed the UserStreamParam argument to MiniDumpWriteDump function). Now, I'm trying to extract the stream from a WinDbg extension. (Note that I have verified that I can retrieve the stream using the MiniDumpReadDumpStream function).
I'm using the IDebugAdvanced2::Request method with DEBUG_REQUEST_READ_USER_MINIDUMP_STREAM request. I am able to retrieve data from standard streams. For example, the following snippet will correctly retrieve the contents of the misc info stream.
DEBUG_READ_USER_MINIDUMP_STREAM rums = {};
rums.StreamType = MiscInfoStream;
rums.Buffer = &buf;
rums.BufferSize = sizeof buf;
hr = p->Request(DEBUG_REQUEST_READ_USER_MINIDUMP_STREAM,
&rums, sizeof rums, 0, 0, 0);
However, trying to retrieve my own stream will result in an error (0x80070570, ERROR_FILE_CORRUPT) and WinDbg outputs
Dir entry 11, ??? stream has unknown stream type 6381921
Note that the same message appears as a part of the .dumpdebug output.
Stream 11: type ??? (6381921), size 00000038, RVA 00033FA9
Dir entry 11, ??? stream has unknown stream type 6381921
What is the problem? How do I retrieve contents of my user stream?
very late answer
StreamType cannot be UserDefined StreamTypes
jen-lung chiu of ms posted so in osronline windbg lists long back
do not know if the latest dbgeng has this limitation eliminated
you either retrieve it with a dbghelp function independently
(using dbghelp functions inside windbg extensions are not recommended )
or parse the stream yourself with fopen() fread() like below
userstream:\>type ..\usrstr.cpp
#include <stdio.h>
#include <engextcpp.hpp>
#include <dbghelp.h>
const ULONG MBUFFSIZE = 0x1000;
PVOID Buff = 0;
int __cdecl ReadUserStream (char *dmpfile)
{
PMINIDUMP_HEADER MiniHeader = 0;
PMINIDUMP_DIRECTORY MiniDir = 0;
PMINIDUMP_USER_STREAM userstream = 0;
size_t result = 0;
ULONG Streams =0;
ULONG i = 0;
FILE * fp = fopen(dmpfile,"rb");
if (fp)
{
result = fread(Buff, 1, sizeof(MINIDUMP_HEADER), fp );
if ( result == sizeof(MINIDUMP_HEADER) )
{
MiniHeader = (PMINIDUMP_HEADER) Buff;
Streams = MiniHeader->NumberOfStreams;
for (i = 0; i < Streams; i++ )
{
result = fread( Buff, 1, sizeof(MINIDUMP_DIRECTORY), fp );
if ( result == sizeof(MINIDUMP_DIRECTORY) )
{
MiniDir = (PMINIDUMP_DIRECTORY) Buff;
if ( MiniDir->StreamType > LastReservedStream )
{
userstream = (PMINIDUMP_USER_STREAM)Buff;
ULONG savedbuffsize = userstream->BufferSize;
ULONG savedtype = userstream->Type;
PCHAR savedbufferptr = (PCHAR)userstream->Buffer;
long pos = ftell(fp);
fseek(fp, (long)savedbufferptr,SEEK_SET);
result = fread( Buff, 1, savedbuffsize, fp );
if ( result == savedbuffsize )
{
printf(
"\n"
"Datastream Type = %.8x\n"
"Buffer Size = %.8x\n"
"Buffer = %p\n"
"Buffer content = %s\n"
"\n",
savedtype,
savedbuffsize,
savedbufferptr,
Buff
);
fseek(fp,pos,SEEK_SET);
continue;
}
else
{
printf(
"failed to read buffer contents at offset %p of
user stream %x\n",
savedbufferptr,
savedtype);
fseek(fp,pos,SEEK_SET);
continue;
}
}
}
else
{
printf("failed to fread Minidump directory exiting \n");
goto getout;
}
}
}
else
{
printf("failed to fread Minidump header exiting \n");
goto getout;
}
}
else
{
printf("failed to open dmp file exiting \n");
goto getout;
}
getout:
if (fp)
fclose(fp);
return 0;
}
int __cdecl main (int argc, char * argv[])
{
if (argc !=2)
{
printf("Usage %s %s\n",argv[0],"somedump.dmp");
return 0;
}
Buff = malloc( MBUFFSIZE );
if (Buff)
{
ReadUserStream(argv[1]);
free(Buff);
return 0;
}
else
{
printf("malloc failed exiting\n");
return 0;
}
}
output from an userdump that has userStreams in it
(oleg staradumov debuginfo.com writeuserstream.cpp )
userstream:\>usrstr.exe
Usage usrstr.exe somedump.dmp
userstream:\>usrstr.exe test.dmp
Datastream Type = 00010000
Buffer Size = 00000021
Buffer = 000010B6
Buffer content = This is the first data stream...
Datastream Type = 00010001
Buffer Size = 00000023
Buffer = 000010D7
Buffer content = and this is the second data stream
I found this topic while looking for a method to read out user stream from the dbg file.
#blabb 's answer is correct in basics and helped me a lot, but it has a two flaws:
You should use the MINIDUMP_HEADER.StreamDirectoryRva to locate the MINIDUMP_DIRECTORY list.
You should not convert the MINIDUMP_DIRECTORY entry to MINIDUMP_USER_STREAM, because that's an error (MINIDUMP_USER_STREAM is a bigger struct than MINIDUMP_DIRECTORY, so you are reading uninitialized memory there) Use the MINIDUMP_DIRECTORY to locate the needed part of the stream.
Even if not tested, it should work if you fill StreamType with a custom value (greater than LastReservedStream = 0xFFFF) instead of MiscInfoStream.

Compressing IplImage to JPEG using libjpeg in OpenCV

So I have this problem.
I have an IplImage that i want to compress to JPEG and do something with it. I use libjpeg.
I found a lot of answers "read through examples and docs" and such and did that. And successfully written a function for that.
FILE* convert2jpeg(IplImage* frame)
{
FILE* outstream = NULL;
outstream=malloc(frame->imageSize*frame->nChannels*sizeof(char))
unsigned char *outdata = (uchar *) frame->imageData;
struct jpeg_error_mgr jerr;
struct jpeg_compress_struct cinfo;
int row_stride;
JSAMPROW row_ptr[1];
jpeg_create_compress(&cinfo);
jpeg_stdio_dest(&cinfo, outstream);
cinfo.image_width = frame->width;
cinfo.image_height = frame->height;
cinfo.input_components = frame->nChannels;
cinfo.in_color_space = JCS_RGB;
jpeg_set_defaults(&cinfo);
jpeg_start_compress(&cinfo, TRUE);
row_stride = frame->width * frame->nChannels;
while (cinfo.next_scanline < cinfo.image_height) {
row_ptr[0] = &outdata[cinfo.next_scanline * row_stride];
jpeg_write_scanlines(&cinfo, row_ptr, 1);
}
jpeg_finish_compress(&cinfo);
jpeg_destroy_compress(&cinfo);
return outstream;
}
Now this function is straight from the examples (except the part of allocating memory, but i need that since i'm not writnig to a file), but it still doesn't work.
It dies on jpeg_start_compress(&cinfo, TRUE); part?
Can anybody help?
I've been able to found a solution using the latest jpeglib available on their website.
New methods in : jpeg_mem_dest(&cinfo, outbuffer, outlen);
bool ipl2jpeg(IplImage *frame, unsigned char **outbuffer, long unsigned int *outlen) {
unsigned char *outdata = (uchar *) frame->imageData;
struct jpeg_compress_struct cinfo = {0};
struct jpeg_error_mgr jerr;
JSAMPROW row_ptr[1];
int row_stride;
*outbuffer = NULL;
*outlen = 0;
cinfo.err = jpeg_std_error(&jerr);
jpeg_create_compress(&cinfo);
jpeg_mem_dest(&cinfo, outbuffer, outlen);
cinfo.image_width = frame->width;
cinfo.image_height = frame->height;
cinfo.input_components = frame->nChannels;
cinfo.in_color_space = JCS_RGB;
jpeg_set_defaults(&cinfo);
jpeg_start_compress(&cinfo, TRUE);
row_stride = frame->width * frame->nChannels;
while (cinfo.next_scanline < cinfo.image_height) {
row_ptr[0] = &outdata[cinfo.next_scanline * row_stride];
jpeg_write_scanlines(&cinfo, row_ptr, 1);
}
jpeg_finish_compress(&cinfo);
jpeg_destroy_compress(&cinfo);
return true;
}
I got it to work using the following code for memory to memory compression.
#include <stdio.h>
#include "jpeg/jpeglib.h"
/*
This a custom destination manager for jpeglib that
enables the use of memory to memory compression.
See IJG documentation for details.
*/
typedef struct {
struct jpeg_destination_mgr pub; /* base class */
JOCTET* buffer; /* buffer start address */
int bufsize; /* size of buffer */
size_t datasize; /* final size of compressed data */
int* outsize; /* user pointer to datasize */
int errcount; /* counts up write errors due to
buffer overruns */
} memory_destination_mgr;
typedef memory_destination_mgr* mem_dest_ptr;
/* ------------------------------------------------------------- */
/* MEMORY DESTINATION INTERFACE METHODS */
/* ------------------------------------------------------------- */
/* This function is called by the library before any data gets written */
METHODDEF(void)
init_destination (j_compress_ptr cinfo)
{
mem_dest_ptr dest = (mem_dest_ptr)cinfo->dest;
dest->pub.next_output_byte = dest->buffer; /* set destination buffer */
dest->pub.free_in_buffer = dest->bufsize; /* input buffer size */
dest->datasize = 0; /* reset output size */
dest->errcount = 0; /* reset error count */
}
/* This function is called by the library if the buffer fills up
I just reset destination pointer and buffer size here.
Note that this behavior, while preventing seg faults
will lead to invalid output streams as data is over-
written.
*/
METHODDEF(boolean)
empty_output_buffer (j_compress_ptr cinfo)
{
mem_dest_ptr dest = (mem_dest_ptr)cinfo->dest;
dest->pub.next_output_byte = dest->buffer;
dest->pub.free_in_buffer = dest->bufsize;
++dest->errcount; /* need to increase error count */
return TRUE;
}
/* Usually the library wants to flush output here.
I will calculate output buffer size here.
Note that results become incorrect, once
empty_output_buffer was called.
This situation is notified by errcount.
*/
METHODDEF(void)
term_destination (j_compress_ptr cinfo)
{
mem_dest_ptr dest = (mem_dest_ptr)cinfo->dest;
dest->datasize = dest->bufsize - dest->pub.free_in_buffer;
if (dest->outsize) *dest->outsize += (int)dest->datasize;
}
/* Override the default destination manager initialization
provided by jpeglib. Since we want to use memory-to-memory
compression, we need to use our own destination manager.
*/
GLOBAL(void)
jpeg_memory_dest (j_compress_ptr cinfo, JOCTET* buffer, int bufsize, int* outsize)
{
mem_dest_ptr dest;
/* first call for this instance - need to setup */
if (cinfo->dest == 0) {
cinfo->dest = (struct jpeg_destination_mgr *)
(*cinfo->mem->alloc_small) ((j_common_ptr) cinfo, JPOOL_PERMANENT,
sizeof (memory_destination_mgr));
}
dest = (mem_dest_ptr) cinfo->dest;
dest->bufsize = bufsize;
dest->buffer = buffer;
dest->outsize = outsize;
/* set method callbacks */
dest->pub.init_destination = init_destination;
dest->pub.empty_output_buffer = empty_output_buffer;
dest->pub.term_destination = term_destination;
}
/* ------------------------------------------------------------- */
/* MEMORY SOURCE INTERFACE METHODS */
/* ------------------------------------------------------------- */
/* Called before data is read */
METHODDEF(void)
init_source (j_decompress_ptr dinfo)
{
/* nothing to do here, really. I mean. I'm not lazy or something, but...
we're actually through here. */
}
/* Called if the decoder wants some bytes that we cannot provide... */
METHODDEF(boolean)
fill_input_buffer (j_decompress_ptr dinfo)
{
/* we can't do anything about this. This might happen if the provided
buffer is either invalid with regards to its content or just a to
small bufsize has been given. */
/* fail. */
return FALSE;
}
/* From IJG docs: "it's not clear that being smart is worth much trouble"
So I save myself some trouble by ignoring this bit.
*/
METHODDEF(void)
skip_input_data (j_decompress_ptr dinfo, INT32 num_bytes)
{
/* There might be more data to skip than available in buffer.
This clearly is an error, so screw this mess. */
if ((size_t)num_bytes > dinfo->src->bytes_in_buffer) {
dinfo->src->next_input_byte = 0; /* no buffer byte */
dinfo->src->bytes_in_buffer = 0; /* no input left */
} else {
dinfo->src->next_input_byte += num_bytes;
dinfo->src->bytes_in_buffer -= num_bytes;
}
}
/* Finished with decompression */
METHODDEF(void)
term_source (j_decompress_ptr dinfo)
{
/* Again. Absolute laziness. Nothing to do here. Boring. */
}
GLOBAL(void)
jpeg_memory_src (j_decompress_ptr dinfo, unsigned char* buffer, size_t size)
{
struct jpeg_source_mgr* src;
/* first call for this instance - need to setup */
if (dinfo->src == 0) {
dinfo->src = (struct jpeg_source_mgr *)
(*dinfo->mem->alloc_small) ((j_common_ptr) dinfo, JPOOL_PERMANENT,
sizeof (struct jpeg_source_mgr));
}
src = dinfo->src;
src->next_input_byte = buffer;
src->bytes_in_buffer = size;
src->init_source = init_source;
src->fill_input_buffer = fill_input_buffer;
src->skip_input_data = skip_input_data;
src->term_source = term_source;
/* IJG recommend to use their function - as I don't know ****
about how to do better, I follow this recommendation */
src->resync_to_restart = jpeg_resync_to_restart;
}
And in your main compression function replace the jpeg_stdio_dest with
int numBytes = 0; //size of jpeg after compression
char * storage = new char[150000]; //storage buffer
JOCTET *jpgbuff = (JOCTET*)storage; //JOCTET pointer to buffer
jpeg_memory_dest(&cinfo,jpgbuff,150000,&numBytes);
The 150000 is a static size buffer, you probably will have images that will exceed it, so allocate accordingly.
I got in-memory compression to work. See the following
#define OUTPUT_BUF_SIZE 4096 /* choose an efficiently fwrite’able size */
/* Expanded data destination object for memory output */
typedef struct {
struct jpeg_destination_mgr pub; /* public fields */
unsigned char ** outbuffer; /* target buffer */
unsigned long * outsize;
unsigned char * newbuffer; /* newly allocated buffer */
JOCTET * buffer; /* start of buffer */
size_t bufsize;
} my_mem_destination_mgr;
typedef my_mem_destination_mgr * my_mem_dest_ptr;
void
init_mem_destination (j_compress_ptr cinfo)
{
/* no work necessary here */
}
boolean
empty_mem_output_buffer (j_compress_ptr cinfo)
{
size_t nextsize;
JOCTET * nextbuffer;
my_mem_dest_ptr dest = (my_mem_dest_ptr) cinfo->dest;
/* Try to allocate new buffer with double size */
nextsize = dest->bufsize * 2;
nextbuffer = (JOCTET *)malloc(nextsize);
if (nextbuffer == NULL)
ERREXIT1(cinfo, JERR_OUT_OF_MEMORY, 10);
memcpy(nextbuffer, dest->buffer, dest->bufsize);
if (dest->newbuffer != NULL)
free(dest->newbuffer);
dest->newbuffer = nextbuffer;
dest->pub.next_output_byte = nextbuffer + dest->bufsize;
dest->pub.free_in_buffer = dest->bufsize;
dest->buffer = nextbuffer;
dest->bufsize = nextsize;
return TRUE;
}
void
term_mem_destination (j_compress_ptr cinfo)
{
my_mem_dest_ptr dest = (my_mem_dest_ptr) cinfo->dest;
*dest->outbuffer = dest->buffer;
*dest->outsize = dest->bufsize – dest->pub.free_in_buffer;
}
void
jpeg_mem_dest (j_compress_ptr cinfo,
unsigned char ** outbuffer, unsigned long * outsize)
{
my_mem_dest_ptr dest;
if (outbuffer == NULL || outsize == NULL) /* sanity check */
ERREXIT(cinfo, JERR_BUFFER_SIZE);
/* The destination object is made permanent so that multiple JPEG images
* can be written to the same buffer without re-executing jpeg_mem_dest.
*/
if (cinfo->dest == NULL) { /* first time for this JPEG object? */
cinfo->dest = (struct jpeg_destination_mgr *)
(*cinfo->mem->alloc_small) ((j_common_ptr) cinfo, JPOOL_PERMANENT,
sizeof(my_mem_destination_mgr));
}
dest = (my_mem_dest_ptr) cinfo->dest;
dest->pub.init_destination = init_mem_destination;
dest->pub.empty_output_buffer = empty_mem_output_buffer;
dest->pub.term_destination = term_mem_destination;
dest->outbuffer = outbuffer;
dest->outsize = outsize;
dest->newbuffer = NULL;
if (*outbuffer == NULL || *outsize == 0) {
/* Allocate initial buffer */
dest->newbuffer = *outbuffer = (unsigned char*)malloc(OUTPUT_BUF_SIZE);
if (dest->newbuffer == NULL)
ERREXIT1(cinfo, JERR_OUT_OF_MEMORY, 10);
*outsize = OUTPUT_BUF_SIZE;
}
dest->pub.next_output_byte = dest->buffer = *outbuffer;
dest->pub.free_in_buffer = dest->bufsize = *outsize;
}
//*******************************************************************************************
To use this do something like this in the main
/************/
unsigned long outlen;
unsigned char *outbuffer;
jpeg_mem_dest (&cinfo,&outbuffer,&outlen );
printf(“outlen is %lu\n”,(long unsigned int)outlen);
After fighting with libJpeg for 2 days (pointers, memory stepping, and pulling hair) I gave up and used the all favourite save-to-disk-load-to-memory approach, so if anybody is interested here is the method:
char* convert2jpeg(IplImage* frame, int* frame_size) {
FILE* infile = NULL;
struct stat fileinfo_buf;
if (cvSaveImage(name_buf, frame) < 0) {
printf("\nCan't save image %s", name_buf);
return NULL;
}
if (stat(name_buf, &fileinfo_buf) < 0) {
printf("\nPLAYER [convert2jpeg] stat");
return NULL;
}
*frame_size = fileinfo_buf.st_size;
char* buffer = (char *) malloc(fileinfo_buf.st_size + 1);
if ((infile = fopen(name_buf, "rb")) == NULL) {
printf("\nPLAYER [convert2jpeg] fopen %s", name_buf);
free(buffer);
return NULL;
}
fread(buffer, fileinfo_buf.st_size, 1, infile);
fclose(infile);
return buffer;
}
I hope somebody finds this usefull. I wish somebody from OpenCV developers sees this thread and implements direct to buffer JPEG conversion in OpenCV and spares us the misery and 1 save/load to disk operation.