how to fill the "data field" of wavfile - c++

Hi i am trying to record from a board and i have successfully record 4 seconds. Problem is when i try to record for more time, i got an error telling me that there not enough memory. my target is to record a 5 minutes file. Until now i have create a buffer named snIn[256] where are the samples. i send it to a big buffer of [16K * 4sec] and when it is full, i create the wav file.
#include "SAI_InOut.hpp"
#include "F746_GUI.hpp"
#include "Delay.hpp"
#include "WaveformDisplay.hpp"
#include "SDFileSystem.h"
#include "wavfile.h"
using namespace Mikami;
#define RES_STR_SIZE 0x20
#define WAVFILE_SAMPLES_PER_SECOND 16000
#define REC_TIME 4
//Create an SDFileSystem object
SDFileSystem sd("sd");
bool flag = 1;
int count = 0;
char *res_buf;
int rp = 0;
const int NUM_SAMPLES = WAVFILE_SAMPLES_PER_SECOND * REC_TIME;
Array<int16_t> my_buffer(NUM_SAMPLES);
int j = 0;
static const char *target_filename = "/sd/rectest.wav";
const int SEG_SIZE = 256;
int sent_array = 0;
int rec(const char *filename, Array<int16_t> my_buffer)
{
j = 0;
flag = 0;
sent_array = 0;
WavFileResult result;
wavfile_info_t info;
wavfile_data_t data;
WAVFILE_INFO_AUDIO_FORMAT(&info) = 1;
WAVFILE_INFO_NUM_CHANNELS(&info) = 1;
WAVFILE_INFO_SAMPLE_RATE(&info) = WAVFILE_SAMPLES_PER_SECOND;
WAVFILE_INFO_BITS_PER_SAMPLE(&info) = 16;
WAVFILE_INFO_BYTE_RATE(&info) = WAVFILE_INFO_NUM_CHANNELS(&info) * WAVFILE_INFO_SAMPLE_RATE(&info) * (WAVFILE_INFO_BITS_PER_SAMPLE(&info) / 8);
WAVFILE_INFO_BLOCK_ALIGN(&info) = 2;
WAVFILE *wf = wavfile_open(filename, WavFileModeWrite, &result);
if (result != WavFileResultOK) {
wavfile_result_string(result, res_buf, RES_STR_SIZE);
printf("%s", res_buf);
return result;
} else printf ("Open file success \r\n");
rp = 0;
WAVFILE_DATA_NUM_CHANNELS(&data) = 1;
result = wavfile_write_info(wf, &info);
if (result != WavFileResultOK) {
wavfile_result_string(result, res_buf, RES_STR_SIZE);
printf("%s", res_buf);
return result; } else printf ("Write info success \r\n");
while ( rp < NUM_SAMPLES ) {
WAVFILE_DATA_CHANNEL_DATA(&data, 0) = my_buffer[rp];
result = wavfile_write_data(wf, &data);
rp += 1;
}
if (result != WavFileResultOK) {
wavfile_result_string(result, res_buf, RES_STR_SIZE);
printf("%s", res_buf);
return result; } else printf ("Write Data file success \r\n");
result = wavfile_close(wf);
if (result != WavFileResultOK) {
wavfile_result_string(result, res_buf , RES_STR_SIZE);
printf("%s", res_buf);
return result; } else printf ("Close file success \r\n");
//UnMount the filesystem
sd.unmount();
printf("Success rec !\r\n");
return 0;
}
int main()
{
//Mount the filesystem
sd.mount();
const float MAX_DELAY = 0.5f; // 最大遅延,単位:秒
const int FS = I2S_AUDIOFREQ_16K; // 標本化周波数: 16 kHz
const uint32_t MAX_ARRAY_SIZE = (uint32_t)(MAX_DELAY*FS);
SaiIO mySai(SaiIO::BOTH, 256, FS, INPUT_DEVICE_DIGITAL_MICROPHONE_2);
Label myLabel(185, 10, "Delay System", Label::CENTER, Font16);
// ButtonGroup: "ON", "OFF"
const uint16_t BG_LEFT = 370;
const uint16_t BG_WIDTH = 100;
const uint16_t BG_HEIGHT = 45;
ButtonGroup onOff(BG_LEFT, 40, BG_WIDTH/2, BG_HEIGHT,
2, (string[]){"ON", "OFF"}, 0, 0, 2, 1);
const uint16_t SB_LEFT = BG_LEFT - 320;
const uint16_t SB_WIDTH = 270;
const uint16_t SB_Y0 = 240;
char str[20];
sprintf(str, " %3.1f [s]", MAX_DELAY);
SeekBar barDelay(SB_LEFT, SB_Y0, SB_WIDTH,
0, MAX_ARRAY_SIZE, 0, "0", "", str);
NumericLabel<float> labelDelay(SB_LEFT+SB_WIDTH/2, SB_Y0-40, "DELEY: %4.2f", 0, Label::CENTER);
DelaySystem delaySystem(MAX_ARRAY_SIZE);
WaveformDisplay displayIn(*GuiBase::GetLcdPtr(), SB_LEFT+7, 70, 256, 9,LCD_COLOR_WHITE, LCD_COLOR_CYAN,GuiBase::ENUM_BACK);
Label inLabel(SB_LEFT-30, 65, "IN");
WaveformDisplay displayOut(*GuiBase::GetLcdPtr(), SB_LEFT+7, 130, 256, 9,LCD_COLOR_WHITE, LCD_COLOR_CYAN,GuiBase::ENUM_BACK);
Label outLabel(SB_LEFT-30, 125, "OUT");
int runStop = 1;
Array<int16_t> snIn(mySai.GetLength());
Array<int16_t> snOut(mySai.GetLength());
mySai.RecordIn();
mySai.PlayOut();
mySai.PauseOut();
while (true)
{
// On/OFF
int num;
if (onOff.GetTouchedNumber(num))
if (runStop != num)
{
if (num == 0) mySai.ResumeOut();
else mySai.PauseOut();
runStop = num;
}
if (mySai.IsCompleted())
{
for (int n=0; n<mySai.GetLength() ; n++)
{
int16_t xL, xR;
mySai.Input(xL,xR);
int16_t xn = xL + xR;
snIn[n] = xn;
my_buffer[j] = xn;
j++;
if (j == NUM_SAMPLES && flag == 1) {
rec (target_filename , my_buffer); }
int16_t yn = delaySystem.Execute(xn);
mySai.Output(yn, yn);
snOut[n] = yn;
}
mySai.Reset();
displayIn.Execute(snIn);
}
}
}
I thought about a possible solution, to fill directly the "data field" of the wavefile with the snIn[256] buffer (instead of using my_buffer) again and again and at the end close the wavfile. Please let me know what you think about that and other solutions

things to note: 1) while a write operation is being performed, more data is still coming in.
At the very least I would double buffer that data, so can be writing one buffer while the other one fills.
Usually this means using an interrupt to collect the samples (into which ever buffer is currently being filed.)
the foreground program waits for the current buffer to be 'full', then initiates write operation.,
then waits again for a buffer to be 'full'
The interrupt function tracks which buffer is being filled and the current index into that buffer. When a buffer is full, set a 'global' status to let the foreground program know which buffer is ready to be written.
The foreground program writes the buffer, then resets the status for that buffer.

Related

How to decode using lame (mp3->wav) in c++

Thank you so much for answering this question.
I use lame and I want to decode mp3 file to wav.
I succeeded in decoding mp3 files into wav files through several searches.
However, the size of the wav file is created too large and an error message appears.
Media player error message :
This file cannot be played. The file format may not be supported, the file extension may be incorrect, or the file may be corrupted.
If you know my problem, please give me some advice.
Thank you
HEADER FILE
#pragma once
#ifndef _LAME_HELPER_H_
#define _LAME_HELPER_H_
#include <windows.h>
#include "lame.h"
#define LH_STARTED WM_USER+1
#define LH_COMPUTED WM_USER+2
#define LH_DONE WM_USER+3
#define LH_ERROR WM_USER+4
#define MAX_THREAD_COUNT 5
enum encode_mode_e
{
EM_ABR,
EM_CBR,
EM_VBR
};
enum encode_channel_e
{
EC_MONO,
EC_STEREO
};
enum bitrate_e
{
BR_8kbps = 8,
BR_16kbps = 16,
BR_24kbps = 24,
BR_32kbps = 32,
BR_40kbps = 40,
BR_48kbps = 48,
BR_56kbps = 56,
BR_64kbps = 64,
BR_80kbps = 80,
BR_96kbps = 96,
BR_112kbps = 112,
BR_128kbps = 128,
BR_144kbps = 144,
BR_160kbps = 160,
BR_192kbps = 192,
BR_224kbps = 224,
BR_256kbps = 256,
BR_320kbps = 320
};
enum samplerate_e
{
SR_8khz = 8000,
SR_11khz = 11025,
SR_12khz = 12000,
SR_16khz = 16000,
SR_22khz = 22050,
SR_24khz = 24000,
SR_32khz = 32000,
SR_44khz = 44100,
SR_48khz = 48000
};
struct settings_t
{
char* title;
char* artist;
char* album;
char* comment;
char* year;
char* track;
char* genre;
char* albumart;
encode_channel_e channels;
bitrate_e abr_bitrate;
bitrate_e cbr_bitrate;
int quality;
encode_mode_e enc_mode;
samplerate_e resample_frequency;
samplerate_e in_samplerate;
//The constructor; used to set default values
settings_t();
};
class CLameHelper; //lameHelper prototype, needed because of struct StaticParam_t
//Use to hold parameters for the thread function
struct StaticParam_t
{
char* pcm;
char* mp3;
settings_t settings;
WNDPROC callback_proc;
CLameHelper* lhObj;
};
class CLameHelper
{
public :
static const int PCM_SIZE = 4096;
static const int MP3_SIZE = 4096;
HANDLE m_hThread[MAX_THREAD_COUNT];
StaticParam_t* m_phSParam[MAX_THREAD_COUNT];
static int Decode_s(void* pParam);
void WriteWaveHeader(FILE* const, int, int, int, int);
void Write32BitLowHigh(FILE*, int);
void Write16BitLowHigh(FILE*, int);
int SetID3AlbumArt(lame_t gfp, char const* szFileName);
void errorHandler(char*);
char errMsg[1000];
public:
CLameHelper();
~CLameHelper();
int Decode(char* szMp3_in, char* szPcm_out);
int Decode(char* szMp3_in, char* szPcm_out, WNDPROC callback_proc);
};
#endif
CPP FILE
#include "stdafx.h"
#include "LameHelper.h"
settings_t::settings_t()
{
//Setting the default values
title = "";
artist = "";
album = "";
comment = "";
year = "";
track = "";
genre = "";
albumart = NULL;
channels = EC_STEREO;
abr_bitrate = BR_128kbps;
cbr_bitrate = BR_128kbps;
quality = 5;
enc_mode = EM_CBR;
resample_frequency = SR_44khz;
in_samplerate = SR_44khz;
}
CLameHelper::CLameHelper()
{
//Initialize to NULL, aids deletion/closing later
for(int i = 0; i < MAX_THREAD_COUNT; i++)
{
m_hThread[i] = NULL;
m_phSParam[i] = NULL;
}
}
CLameHelper::~CLameHelper()
{
//Destroy all declared objects
for(int i = 0; i < MAX_THREAD_COUNT; i++)
{
if(m_hThread[i] != NULL)
CloseHandle(m_hThread[i]);
if(m_phSParam[i] != NULL)
delete m_phSParam[i];
}
}
int CLameHelper::SetID3AlbumArt(lame_t gfp, char const* szFileName)
{
int iResult = -1;
FILE *pFileName = 0;
char *szAlbumart = 0;
if(szFileName == NULL)
{
return 0;
}
pFileName = fopen(szFileName, "rb");
if(!pFileName)
{
iResult = 1;
}
else
{
size_t size;
fseek(pFileName, 0, SEEK_END);
size = ftell(pFileName);
fseek(pFileName, 0, SEEK_SET);
szAlbumart = (char*)malloc(size);
if(!szAlbumart)
{
iResult = 2;
}
else
{
if(fread(szAlbumart, 1, size, pFileName) != size)
{
iResult = 3;
}
else
{
iResult = (gfp, szAlbumart, size) ? 4 : 0;
}
free(szAlbumart);
}
fclose(pFileName);
}
switch(iResult)
{
case 1:
sprintf(errMsg, "WARNING: could not find file '%s' for szAlbumart.\n", szFileName);
errorHandler(errMsg);
break;
case 2:
errorHandler("WARNING: insufficient memory for reading the szAlbumart.\n");
break;
case 3:
sprintf(errMsg, "WARNING: read error in '%s' for szAlbumart.\n", szFileName);
errorHandler(errMsg);
break;
case 4:
sprintf(errMsg, "WARNING: unsupported image: '%s' for szAlbumart. Specify JPEG/PNG/GIF image\n", szFileName);
errorHandler(errMsg);
break;
default:
break;
}
return iResult;
}
void CLameHelper::Write16BitLowHigh(FILE * fp, int val)
{
unsigned char bytes[2];
bytes[0] = (val & 0xff);
bytes[1] = ((val >> 8) & 0xff);
fwrite(bytes, 1, 2, fp);
}
void CLameHelper::Write32BitLowHigh(FILE * fp, int val)
{
unsigned char bytes[4];
bytes[0] = (val & 0xff);
bytes[1] = ((val >> 8) & 0xff);
bytes[2] = ((val >> 16) & 0xff);
bytes[3] = ((val >> 24) & 0xff);
fwrite(bytes, 1, 4, fp);
}
void CLameHelper::WriteWaveHeader(FILE * const fp, int pcmbytes, int freq, int channels, int bits)
{
int bytes = (bits + 7) / 8;
/* quick and dirty, but documented */
fwrite("RIFF", 1, 4, fp); /* label */
Write32BitLowHigh(fp, pcmbytes + 44 - 8); /* length in bytes without header */
fwrite("WAVEfmt ", 2, 4, fp); /* 2 labels */
Write32BitLowHigh(fp, 2 + 2 + 4 + 4 + 2 + 2); /* length of PCM format declaration area */
Write16BitLowHigh(fp, 1); /* is PCM? */
Write16BitLowHigh(fp, channels); /* number of channels */
Write32BitLowHigh(fp, freq); /* sample frequency in [Hz] */
Write32BitLowHigh(fp, freq * channels * bytes); /* bytes per second */
Write16BitLowHigh(fp, channels * bytes); /* bytes per sample time */
Write16BitLowHigh(fp, bits); /* bits per sample */
fwrite("data", 1, 4, fp); /* label */
Write32BitLowHigh(fp, pcmbytes); /* length in bytes of raw PCM data */
}
int CLameHelper::Decode(char* szMp3_in, char* szPcm_out)
{
return Decode(szMp3_in, szPcm_out, NULL);
}
//the static function used for the thread
int CLameHelper::Decode_s(void* param)
{
StaticParam_t* sp = (StaticParam_t*)param;
char* szPcm_out = sp->pcm;
char* szMp3_in = sp->mp3;
WNDPROC callback_proc = sp->callback_proc;
CLameHelper* lh = (CLameHelper*)sp->lhObj;
return lh->Decode(szMp3_in, szPcm_out, callback_proc);
}
int CLameHelper::Decode(char* szMp3_in, char* szPcm_out, WNDPROC callback_proc)
{
int read, i, samples;
long wavsize = 0; // use to count the number of mp3 byte read, this is used to write the length of the wave file
long cumulative_read = 0;
short int pcm_l[PCM_SIZE], pcm_r[PCM_SIZE];
unsigned char mp3_buffer[MP3_SIZE];
FILE* mp3 = fopen(szMp3_in, "rb");
if(mp3 == NULL)
{
if(callback_proc != NULL)
{
callback_proc((HWND)GetModuleHandle(NULL), LH_ERROR, -1, NULL);
}
sprintf(errMsg, "FATAL ERROR: file '%s' can't be open for read. Aborting!\n", szMp3_in);
errorHandler(errMsg);
return -1;
}
fseek(mp3, 0, SEEK_END);
long MP3_total_size = ftell(mp3);
fseek(mp3, 0, SEEK_SET);
FILE* pcm = fopen(szPcm_out, "wb");
if(pcm == NULL)
{
if(callback_proc != NULL)
{
callback_proc((HWND)GetModuleHandle(NULL), LH_ERROR, -1, NULL);
}
sprintf(errMsg, "FATAL ERROR: file '%s' can't be open for write. Aborting!\n", szPcm_out);
errorHandler(errMsg);
return -1;
}
lame_t lame = lame_init();
lame_set_decode_only(lame, 1);
if(lame_init_params(lame) == -1)
{
if(callback_proc != NULL)
{
callback_proc((HWND)GetModuleHandle(NULL), LH_ERROR, -2, NULL);
}
sprintf(errMsg, "FATAL ERROR: parameters failed to initialize properly in lame. Aborting!\n", szPcm_out);
errorHandler(errMsg);
return -2;
}
hip_t hip = hip_decode_init();
mp3data_struct mp3data;
memset(&mp3data, 0, sizeof(mp3data));
int nChannels = -1;
int nSampleRate = -1;
int mp3_len;
if(callback_proc != NULL)
{
callback_proc((HWND)GetModuleHandle(NULL), LH_STARTED, NULL, NULL);
}
while((read = fread(mp3_buffer, sizeof(char), MP3_SIZE, mp3)) > 0)
{
mp3_len = read;
cumulative_read += read * sizeof(char);
do
{
samples = hip_decode1_headers(hip, mp3_buffer, mp3_len, pcm_l, pcm_r, &mp3data);
wavsize += samples;
if(mp3data.header_parsed == 1)//header is gotten
{
if(nChannels < 0)//reading for the first time
{
//Write the header
WriteWaveHeader(pcm, 0x7FFFFFFF, mp3data.samplerate, mp3data.stereo, 16); //unknown size, so write maximum 32 bit signed value
}
nChannels = mp3data.stereo;
nSampleRate = mp3data.samplerate;
}
if(samples > 0 && mp3data.header_parsed != 1)
{
errorHandler("WARNING: lame decode error occured!");
break;
}
if(samples > 0)
{
for(i = 0 ; i < samples; i++)
{
fwrite((char*)&pcm_l[i], sizeof(char), sizeof(pcm_l[i]), pcm);
if(nChannels == 2)
{
fwrite((char*)&pcm_r[i], sizeof(char), sizeof(pcm_r[i]), pcm);
}
}
}
mp3_len = 0;
if(callback_proc != NULL)
{
int percentage = ((float)cumulative_read/MP3_total_size)*100;
callback_proc((HWND)GetModuleHandle(NULL), LH_COMPUTED, percentage, NULL);
}
}while(samples>0);
}
i = (16 / 8) * mp3data.stereo;
if (wavsize <= 0)
{
wavsize = 0;
}
else if (wavsize > 0xFFFFFFD0 / i)
{
wavsize = 0xFFFFFFD0;
}
else
{
wavsize *= i;
}
if(!fseek(pcm, 0l, SEEK_SET))//seek back and adjust length
WriteWaveHeader(pcm, (int) wavsize, mp3data.samplerate, mp3data.stereo, 16);
else
errorHandler("WARNING: can't seek back to adjust length in wave header!");
hip_decode_exit(hip);
lame_close(lame);
fclose(mp3);
fclose(pcm);
if(callback_proc != NULL)
{
callback_proc((HWND)GetModuleHandle(NULL), LH_DONE, NULL, NULL);
}
return 0;
}
void CLameHelper::errorHandler(char* msg)
{
printf("%s\n", msg);
}

How to make a thread stop excution (eg: std::this_thread::sleep_for) for an accturate interval

I am currently making a small discord bot that can play music to improve my skill. That's why i don't use any discord lib.
I want the music as smooth as possible, but when i played some piece of music, the music produced is very choppy.
here is my code:
concurrency::task<void> play(std::string id) {
auto shared_token = std::make_shared<concurrency::cancellation_token*>(&p_token);
auto shared_running = std::make_shared<bool*>(&running);
return concurrency::create_task([this, id, shared_token] {
audio* source = new audio(id); // create a s16le binary stream using FFMPEG
speak(); // sending speak packet
printf("creating opus encoder\n");
const unsigned short FRAME_MILLIS = 20;
const unsigned short FRAME_SIZE = 960;
const unsigned short SAMPLE_RATE = 48000;
const unsigned short CHANNELS = 2;
const unsigned int BITRATE = 64000;
#define MAX_PACKET_SIZE FRAME_SIZE * 5
int error;
OpusEncoder* encoder = opus_encoder_create(SAMPLE_RATE, CHANNELS, OPUS_APPLICATION_AUDIO, &error);
if (error < 0) {
throw "failed to create opus encoder: " + std::string(opus_strerror(error));
}
error = opus_encoder_ctl(encoder, OPUS_SET_BITRATE(BITRATE));
if (error < 0) {
throw "failed to set bitrate for opus encoder: " + std::string(opus_strerror(error));
}
if (sodium_init() == -1) {
throw "libsodium initialisation failed";
}
int num_opus_bytes;
unsigned char* pcm_data = new unsigned char[FRAME_SIZE * CHANNELS * 2];
opus_int16* in_data;
std::vector<unsigned char> opus_data(MAX_PACKET_SIZE);
class timer_event {
bool is_set = false;
public:
bool get_is_set() { return is_set; };
void set() { is_set = true; };
void unset() { is_set = false; };
};
timer_event* run_timer = new timer_event();
run_timer->set();
//this is the send loop
concurrency::create_task([run_timer, this, shared_token] {
while (run_timer->get_is_set()) {
speak();
int i = 0;
while (i < 15) {
utils::sleep(1000);
if (run_timer->get_is_set() == false) {
std::cout << "Stop sending speak packet due to turn off\n";
concurrency::cancel_current_task();
return;
}
if ((*shared_token)->is_canceled()) {
std::cout << "Stop sending speak packet due to cancel\n";
concurrency::cancel_current_task();
return;
}
}
}});
std::deque<std::string>* buffer = new std::deque<std::string>();
auto timer = concurrency::create_task([run_timer, this, buffer, FRAME_MILLIS, shared_token] {
while (run_timer->get_is_set() || buffer->size() > 0) {
utils::sleep(5 * FRAME_MILLIS); //std::this_thread::sleep_for
int loop = 0;
int sent = 0;
auto start = boost::chrono::high_resolution_clock::now();
while (buffer->size() > 0) {
if (udpclient.send(buffer->front()) != 0) { //send frame
//udpclient.send ~ winsock sendto
std::cout << "Stop sendding voice data due to udp error\n";
return;
}
buffer->pop_front();
if ((*shared_token)->is_canceled()) {
std::cout << "Stop sending voice data due to cancel\n";
concurrency::cancel_current_task();
}
sent++; //count sent frame
//calculate next time point we should (in theory) send next frame and store in *delay*
long long next_time = (long long)(sent+1) * (long long)(FRAME_MILLIS) * 1000 ;
auto now = boost::chrono::high_resolution_clock::now();
long long mcs_elapsed = (boost::chrono::duration_cast<boost::chrono::microseconds>(now - start)).count(); // elapsed time from start loop
long long delay = std::max((long long)0, (next_time - mcs_elapsed));
//wait for next time point
boost::asio::deadline_timer timer(context_io);
timer.expires_from_now(boost::posix_time::microseconds(delay));
timer.wait();
}
}
});
unsigned short _sequence = 0;
unsigned int _timestamp = 0;
while (1) {
if (buffer->size() >= 50) {
utils::sleep(FRAME_MILLIS);
}
if (source->read((char*)pcm_data, FRAME_SIZE * CHANNELS * 2) != true)
break;
if ((*shared_token)->is_canceled()) {
std::cout << "Stop encoding due to cancel\n";
break;
}
in_data = (opus_int16*)pcm_data;
num_opus_bytes = opus_encode(encoder, in_data, FRAME_SIZE, opus_data.data(), MAX_PACKET_SIZE);
if (num_opus_bytes <= 0) {
throw "failed to encode frame: " + std::string(opus_strerror(num_opus_bytes));
}
opus_data.resize(num_opus_bytes);
std::vector<unsigned char> packet(12 + opus_data.size() + crypto_secretbox_MACBYTES);
packet[0] = 0x80; //Type
packet[1] = 0x78; //Version
packet[2] = _sequence >> 8; //Sequence
packet[3] = (unsigned char)_sequence;
packet[4] = _timestamp >> 24; //Timestamp
packet[5] = _timestamp >> 16;
packet[6] = _timestamp >> 8;
packet[7] = _timestamp;
packet[8] = (unsigned char)(ssrc >> 24); //SSRC
packet[9] = (unsigned char)(ssrc >> 16);
packet[10] = (unsigned char)(ssrc >> 8);
packet[11] = (unsigned char)ssrc;
_sequence++;
_timestamp += SAMPLE_RATE / 1000 * FRAME_MILLIS; //48000Hz / 1000 * 20(ms)
unsigned char nonce[crypto_secretbox_NONCEBYTES];
memset(nonce, 0, crypto_secretbox_NONCEBYTES);
for (int i = 0; i < 12; i++) {
nonce[i] = packet[i];
}
crypto_secretbox_easy(packet.data() + 12, opus_data.data(), opus_data.size(), nonce, key.data());
packet.resize(12 + opus_data.size() + crypto_secretbox_MACBYTES);
std::string msg;
msg.resize(packet.size(), '\0');
for (unsigned int i = 0; i < packet.size(); i++) {
msg[i] = packet[i];
}
buffer->push_back(msg);
}
run_timer->unset();
timer.wait();
unspeak();
delete run_timer;
delete buffer;
opus_encoder_destroy(encoder);
delete[] pcm_data;
});
}
There are 3 possible causes:
I send packet late so server-end buffer run out, so the sound produced has some silence between each each 2 packets. Maybe the timer is not accurate so the sound is out of sync.
The encode process is wrong which causes lost data somehow.
Bad network (i have tested an open source bot written on java, it worked so i can assume that my network is good enough)
So i post this question, hope someone has experienced this situation show me what wrong and what should i do to correct it.
I figured out the problem myself. I want to post solution here for someone who need.
The problem is the timer is unstable so it's usually sleep more than it should, so it makes the music broken.
I changed it to an accurate sleep function which i found somewhere on the internet(i don't remember the source, sorry for that, if you know it please credit it bellow).
Function source code:
#include <math.h>
#include <chrono>
#include <window.h>
static void timerSleep(double seconds) {
using namespace std::chrono;
static HANDLE timer = CreateWaitableTimer(NULL, FALSE, NULL);
static double estimate = 5e-3;
static double mean = 5e-3;
static double m2 = 0;
static int64_t count = 1;
while (seconds - estimate > 1e-7) {
double toWait = seconds - estimate;
LARGE_INTEGER due;
due.QuadPart = -int64_t(toWait * 1e7);
auto start = high_resolution_clock::now();
SetWaitableTimerEx(timer, &due, 0, NULL, NULL, NULL, 0);
WaitForSingleObject(timer, INFINITE);
auto end = high_resolution_clock::now();
double observed = (end - start).count() / 1e9;
seconds -= observed;
++count;
double error = observed - toWait;
double delta = error - mean;
mean += delta / count;
m2 += delta * (error - mean);
double stddev = sqrt(m2 / (count - 1));
estimate = mean + stddev;
}
// spin lock
auto start = high_resolution_clock::now();
while ((high_resolution_clock::now() - start).count() / 1e9 < seconds);
}
Thank you for your support!

Why is msgrcv() feeding garbage characters into the buffer?

right now, I am currently trying to output the contents of buf.mtext so I can make sure take the correct input before moving on with my program. Everything seems to work fine, except one thing; msgrcv() puts garbage characters into the buffer, and the reciever process outputs garbage characters.
Here is my sender process:
int main (void)
{
int i; // loop counter
int status_01; // result status
int msqid_01; // message queue ID (#1)
key_t msgkey_01; // message-queue key (#1)
unsigned int rand_num;
float temp_rand;
unsigned char eight_bit_num;
unsigned char counter = 0;
unsigned char even_counter = 0;
unsigned char odd_counter = 0;
srand(time(0));
struct message {
long mtype;
char mtext[BUFFER_SIZE];
} buf_01;
msgkey_01 = MSG_key_01; // defined at top of file
msqid_01 = msgget(msgkey_01, 0666 | IPC_CREAT)
if ((msqid_01 <= -1) { exit(1); }
/* wait for a key stroke at the keyboard ---- */
eight_bit_num = getchar();
buf_01.mtype = 1;
/* send one eight-bit number, one at a time ------------ */
for (i = 0; i < NUM_REPEATS; i++)
{
temp_rand = ((float)rand()/(float)RAND_MAX)*255.0;
rand_num = (int)temp_rand;
eight_bit_num = (unsigned char)rand_num;
if ((eight_bit_num % 2) == 0)
{
printf("Even number: %d\n", eight_bit_num);
even_counter = even_counter + eight_bit_num;
}
else
{
printf("Odd number: %d\n", eight_bit_num);
odd_counter = odd_counter + eight_bit_num;
}
/* update the counters ------------------------------ */
counter = counter + eight_bit_num;
if((eight_bit_num % 2) == 0) { even_counter = even_counter + eight_bit_num; }
else { odd_counter = odd_counter + eight_bit_num; }
buf_01.mtext[0] = eight_bit_num; // copy the 8-bit number
buf_01.mtext[1] = '\0'; // null-terminate it
status_01 = msgsnd(msqid_01, (struct msgbuf *)&buf_01, sizeof(buf_01.mtext), 0);
status_01 = msgctl(msqid_01, IPC_RMID, NULL);
}
Here is my receiver process:
int main() {
struct message {
long mtype;
char mtext[BUFFER_SIZE];
} buf;
int msqid;
key_t msgkey;
msgkey = MSG_key_01;
msqid = msgget(msgkey, 0666); // connect to message queue
if (msqid < 0) {
printf("Failed\n");
exit(1);
}
else {
printf("Connected\n");
}
if (msgrcv(msqid, &buf, BUFFER_SIZE, 0, 0) < 0) { // read message into buf
perror("msgrcv");
exit(1);
}
printf("Data received is: %s \n", buf.mtext);
printf("Done receiving messages.\n");
return 0;
}
The output is usually something like as follows:
Data received is: ▒
Done receiving messages.
I have made sure to clear my message queues each time after running the sender and receiver processes, as well, since I have come to find out this can cause issues. Thanks in advance for your help.
Turns out neither of the suggested solutions were the issue, as I suspected; the sender process actually works just fine. The problem was that I was trying to print buf.mtext instead of buf.mtext[0] which isn't an actual integer value. I fixed the issue by just doing this:
int temp_num = buf.mtext[0];
printf("Data recieved is %d \n", temp_num);

How to use LZMA SDK in C++?

i have difficulties in using LZMA SDK in my application.
I would like to create a kind of single file compression tool. I dont need any directory support, just need only the LZMA2 stream. But i have no idea on how LZMA SDK is to be used for this.
Please can anyone give me a little example on how the LZMA SDK can be used under C++?
I think that it's a properly little example to use LZMA SDK.
/* LzmaUtil.c -- Test application for LZMA compression
2008-08-05
Igor Pavlov
public domain */
#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "../LzmaDec.h"
#include "../LzmaEnc.h"
#include "../Alloc.h"
const char *kCantReadMessage = "Can not read input file";
const char *kCantWriteMessage = "Can not write output file";
const char *kCantAllocateMessage = "Can not allocate memory";
const char *kDataErrorMessage = "Data error";
static void *SzAlloc(void *p, size_t size) { p = p; return MyAlloc(size); }
static void SzFree(void *p, void *address) { p = p; MyFree(address); }
static ISzAlloc g_Alloc = { SzAlloc, SzFree };
#define kInBufferSize (1 << 15)
#define kOutBufferSize (1 << 15)
unsigned char g_InBuffer[kInBufferSize];
unsigned char g_OutBuffer[kOutBufferSize];
size_t MyReadFile(FILE *file, void *data, size_t size)
{ return fread(data, 1, size, file); }
int MyReadFileAndCheck(FILE *file, void *data, size_t size)
{ return (MyReadFile(file, data, size) == size); }
size_t MyWriteFile(FILE *file, const void *data, size_t size)
{
if (size == 0)
return 0;
return fwrite(data, 1, size, file);
}
int MyWriteFileAndCheck(FILE *file, const void *data, size_t size)
{ return (MyWriteFile(file, data, size) == size); }
long MyGetFileLength(FILE *file)
{
long length;
fseek(file, 0, SEEK_END);
length = ftell(file);
fseek(file, 0, SEEK_SET);
return length;
}
void PrintHelp(char *buffer)
{
strcat(buffer, "\nLZMA Utility 4.58 Copyright (c) 1999-2008 Igor Pavlov 2008-04-11\n"
"\nUsage: lzma <e|d> inputFile outputFile\n"
" e: encode file\n"
" d: decode file\n");
}
int PrintError(char *buffer, const char *message)
{
strcat(buffer, "\nError: ");
strcat(buffer, message);
strcat(buffer, "\n");
return 1;
}
int PrintErrorNumber(char *buffer, SRes val)
{
sprintf(buffer + strlen(buffer), "\nError code: %x\n", (unsigned)val);
return 1;
}
int PrintUserError(char *buffer)
{
return PrintError(buffer, "Incorrect command");
}
#define IN_BUF_SIZE (1 << 16)
#define OUT_BUF_SIZE (1 << 16)
static int Decode(FILE *inFile, FILE *outFile, char *rs)
{
UInt64 unpackSize;
int thereIsSize; /* = 1, if there is uncompressed size in headers */
int i;
int res = 0;
CLzmaDec state;
/* header: 5 bytes of LZMA properties and 8 bytes of uncompressed size */
unsigned char header[LZMA_PROPS_SIZE + 8];
/* Read and parse header */
if (!MyReadFileAndCheck(inFile, header, sizeof(header)))
return PrintError(rs, kCantReadMessage);
unpackSize = 0;
thereIsSize = 0;
for (i = 0; i < 8; i++)
{
unsigned char b = header[LZMA_PROPS_SIZE + i];
if (b != 0xFF)
thereIsSize = 1;
unpackSize += (UInt64)b << (i * 8);
}
LzmaDec_Construct(&state);
res = LzmaDec_Allocate(&state, header, LZMA_PROPS_SIZE, &g_Alloc);
if (res != SZ_OK)
return res;
{
Byte inBuf[IN_BUF_SIZE];
Byte outBuf[OUT_BUF_SIZE];
size_t inPos = 0, inSize = 0, outPos = 0;
LzmaDec_Init(&state);
for (;;)
{
if (inPos == inSize)
{
inSize = MyReadFile(inFile, inBuf, IN_BUF_SIZE);
inPos = 0;
}
{
SizeT inProcessed = inSize - inPos;
SizeT outProcessed = OUT_BUF_SIZE - outPos;
ELzmaFinishMode finishMode = LZMA_FINISH_ANY;
ELzmaStatus status;
if (thereIsSize && outProcessed > unpackSize)
{
outProcessed = (SizeT)unpackSize;
finishMode = LZMA_FINISH_END;
}
res = LzmaDec_DecodeToBuf(&state, outBuf + outPos, &outProcessed,
inBuf + inPos, &inProcessed, finishMode, &status);
inPos += (UInt32)inProcessed;
outPos += outProcessed;
unpackSize -= outProcessed;
if (outFile != 0)
MyWriteFile(outFile, outBuf, outPos);
outPos = 0;
if (res != SZ_OK || thereIsSize && unpackSize == 0)
break;
if (inProcessed == 0 && outProcessed == 0)
{
if (thereIsSize || status != LZMA_STATUS_FINISHED_WITH_MARK)
res = SZ_ERROR_DATA;
break;
}
}
}
}
LzmaDec_Free(&state, &g_Alloc);
return res;
}
typedef struct _CFileSeqInStream
{
ISeqInStream funcTable;
FILE *file;
} CFileSeqInStream;
static SRes MyRead(void *p, void *buf, size_t *size)
{
if (*size == 0)
return SZ_OK;
*size = MyReadFile(((CFileSeqInStream*)p)->file, buf, *size);
/*
if (*size == 0)
return SZE_FAIL;
*/
return SZ_OK;
}
typedef struct _CFileSeqOutStream
{
ISeqOutStream funcTable;
FILE *file;
} CFileSeqOutStream;
static size_t MyWrite(void *pp, const void *buf, size_t size)
{
return MyWriteFile(((CFileSeqOutStream *)pp)->file, buf, size);
}
static SRes Encode(FILE *inFile, FILE *outFile, char *rs)
{
CLzmaEncHandle enc;
SRes res;
CFileSeqInStream inStream;
CFileSeqOutStream outStream;
CLzmaEncProps props;
enc = LzmaEnc_Create(&g_Alloc);
if (enc == 0)
return SZ_ERROR_MEM;
inStream.funcTable.Read = MyRead;
inStream.file = inFile;
outStream.funcTable.Write = MyWrite;
outStream.file = outFile;
LzmaEncProps_Init(&props);
res = LzmaEnc_SetProps(enc, &props);
if (res == SZ_OK)
{
Byte header[LZMA_PROPS_SIZE + 8];
size_t headerSize = LZMA_PROPS_SIZE;
UInt64 fileSize;
int i;
res = LzmaEnc_WriteProperties(enc, header, &headerSize);
fileSize = MyGetFileLength(inFile);
for (i = 0; i < 8; i++)
header[headerSize++] = (Byte)(fileSize >> (8 * i));
if (!MyWriteFileAndCheck(outFile, header, headerSize))
return PrintError(rs, "writing error");
if (res == SZ_OK)
res = LzmaEnc_Encode(enc, &outStream.funcTable, &inStream.funcTable,
NULL, &g_Alloc, &g_Alloc);
}
LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc);
return res;
}
int main2(int numArgs, const char *args[], char *rs)
{
FILE *inFile = 0;
FILE *outFile = 0;
char c;
int res;
int encodeMode;
if (numArgs == 1)
{
PrintHelp(rs);
return 0;
}
if (numArgs < 3 || numArgs > 4 || strlen(args[1]) != 1)
return PrintUserError(rs);
c = args[1][0];
encodeMode = (c == 'e' || c == 'E');
if (!encodeMode && c != 'd' && c != 'D')
return PrintUserError(rs);
{
size_t t4 = sizeof(UInt32);
size_t t8 = sizeof(UInt64);
if (t4 != 4 || t8 != 8)
return PrintError(rs, "LZMA UTil needs correct UInt32 and UInt64");
}
inFile = fopen(args[2], "rb");
if (inFile == 0)
return PrintError(rs, "Can not open input file");
if (numArgs > 3)
{
outFile = fopen(args[3], "wb+");
if (outFile == 0)
return PrintError(rs, "Can not open output file");
}
else if (encodeMode)
PrintUserError(rs);
if (encodeMode)
{
res = Encode(inFile, outFile, rs);
}
else
{
res = Decode(inFile, outFile, rs);
}
if (outFile != 0)
fclose(outFile);
fclose(inFile);
if (res != SZ_OK)
{
if (res == SZ_ERROR_MEM)
return PrintError(rs, kCantAllocateMessage);
else if (res == SZ_ERROR_DATA)
return PrintError(rs, kDataErrorMessage);
else
return PrintErrorNumber(rs, res);
}
return 0;
}
int MY_CDECL main(int numArgs, const char *args[])
{
char rs[800] = { 0 };
int res = main2(numArgs, args, rs);
printf(rs);
return res;
}
Also you can see it at:
http://read.pudn.com/downloads151/sourcecode/zip/656407/7z460/C/LzmaUtil/LzmaUtil.c__.htm
http://read.pudn.com/downloads157/sourcecode/zip/698262/LZMA/LzmaUtil.c__.htm
I recently found a nice example, written in C++. Credit goes to GH user Treeki who published the original gist:
// note: -D_7ZIP_ST is required when compiling on non-Windows platforms
// g++ -o lzma_sample -std=c++14 -D_7ZIP_ST lzma_sample.cpp LzmaDec.c LzmaEnc.c LzFind.c
#include <stdio.h>
#include <stdint.h>
#include <string.h>
#include <memory>
#include "LzmaEnc.h"
#include "LzmaDec.h"
static void *_lzmaAlloc(ISzAllocPtr, size_t size) {
return new uint8_t[size];
}
static void _lzmaFree(ISzAllocPtr, void *addr) {
if (!addr)
return;
delete[] reinterpret_cast<uint8_t *>(addr);
}
static ISzAlloc _allocFuncs = {
_lzmaAlloc, _lzmaFree
};
std::unique_ptr<uint8_t[]> lzmaCompress(const uint8_t *input, uint32_t inputSize, uint32_t *outputSize) {
std::unique_ptr<uint8_t[]> result;
// set up properties
CLzmaEncProps props;
LzmaEncProps_Init(&props);
if (inputSize >= (1 << 20))
props.dictSize = 1 << 20; // 1mb dictionary
else
props.dictSize = inputSize; // smaller dictionary = faster!
props.fb = 40;
// prepare space for the encoded properties
SizeT propsSize = 5;
uint8_t propsEncoded[5];
// allocate some space for the compression output
// this is way more than necessary in most cases...
// but better safe than sorry
// (a smarter implementation would use a growing buffer,
// but this requires a bunch of fuckery that is out of
/// scope for this simple example)
SizeT outputSize64 = inputSize * 1.5;
if (outputSize64 < 1024)
outputSize64 = 1024;
auto output = std::make_unique<uint8_t[]>(outputSize64);
int lzmaStatus = LzmaEncode(
output.get(), &outputSize64, input, inputSize,
&props, propsEncoded, &propsSize, 0,
NULL,
&_allocFuncs, &_allocFuncs);
*outputSize = outputSize64 + 13;
if (lzmaStatus == SZ_OK) {
// tricky: we have to generate the LZMA header
// 5 bytes properties + 8 byte uncompressed size
result = std::make_unique<uint8_t[]>(outputSize64 + 13);
uint8_t *resultData = result.get();
memcpy(resultData, propsEncoded, 5);
for (int i = 0; i < 8; i++)
resultData[5 + i] = (inputSize >> (i * 8)) & 0xFF;
memcpy(resultData + 13, output.get(), outputSize64);
}
return result;
}
std::unique_ptr<uint8_t[]> lzmaDecompress(const uint8_t *input, uint32_t inputSize, uint32_t *outputSize) {
if (inputSize < 13)
return NULL; // invalid header!
// extract the size from the header
UInt64 size = 0;
for (int i = 0; i < 8; i++)
size |= (input[5 + i] << (i * 8));
if (size <= (256 * 1024 * 1024)) {
auto blob = std::make_unique<uint8_t[]>(size);
ELzmaStatus lzmaStatus;
SizeT procOutSize = size, procInSize = inputSize - 13;
int status = LzmaDecode(blob.get(), &procOutSize, &input[13], &procInSize, input, 5, LZMA_FINISH_END, &lzmaStatus, &_allocFuncs);
if (status == SZ_OK && procOutSize == size) {
*outputSize = size;
return blob;
}
}
return NULL;
}
void hexdump(const uint8_t *buf, int size) {
int lines = (size + 15) / 16;
for (int i = 0; i < lines; i++) {
printf("%08x | ", i * 16);
int lineMin = i * 16;
int lineMax = lineMin + 16;
int lineCappedMax = (lineMax > size) ? size : lineMax;
for (int j = lineMin; j < lineCappedMax; j++)
printf("%02x ", buf[j]);
for (int j = lineCappedMax; j < lineMax; j++)
printf(" ");
printf("| ");
for (int j = lineMin; j < lineCappedMax; j++) {
if (buf[j] >= 32 && buf[j] <= 127)
printf("%c", buf[j]);
else
printf(".");
}
printf("\n");
}
}
void testIt(const uint8_t *input, int size) {
printf("Test Input:\n");
hexdump(input, size);
uint32_t compressedSize;
auto compressedBlob = lzmaCompress(input, size, &compressedSize);
if (compressedBlob) {
printf("Compressed:\n");
hexdump(compressedBlob.get(), compressedSize);
} else {
printf("Nope, we screwed it\n");
return;
}
// let's try decompressing it now
uint32_t decompressedSize;
auto decompressedBlob = lzmaDecompress(compressedBlob.get(), compressedSize, &decompressedSize);
if (decompressedBlob) {
printf("Decompressed:\n");
hexdump(decompressedBlob.get(), decompressedSize);
} else {
printf("Nope, we screwed it (part 2)\n");
return;
}
printf("----------\n");
}
void testIt(const char *string) {
testIt((const uint8_t *)string, strlen(string));
}
int main(int argc, char **argv) {
testIt("a");
testIt("here is a cool string");
testIt("here's something that should compress pretty well: abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdef");
return 0;
}
You can refer to this file on how to use lzma2。
https://github.com/Tencent/libpag/blob/aab6391e455193c8ec5b8e2031b495b3fe77b034/test/framework/utils/LzmaUtil.cpp
/////////////////////////////////////////////////////////////////////////////////////////////////
//
// Tencent is pleased to support the open source community by making libpag available.
//
// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// unless required by applicable law or agreed to in writing, software distributed under the
// license is distributed on an "as is" basis, without warranties or conditions of any kind,
// either express or implied. see the license for the specific language governing permissions
// and limitations under the license.
//
/////////////////////////////////////////////////////////////////////////////////////////////////
#include "LzmaUtil.h"
#include "test/framework/lzma/Lzma2DecMt.h"
#include "test/framework/lzma/Lzma2Enc.h"
namespace pag {
static void* LzmaAlloc(ISzAllocPtr, size_t size) {
return new uint8_t[size];
}
static void LzmaFree(ISzAllocPtr, void* address) {
if (!address) {
return;
}
delete[] reinterpret_cast<uint8_t*>(address);
}
static ISzAlloc gAllocFuncs = {LzmaAlloc, LzmaFree};
class SequentialOutStream {
public:
virtual ~SequentialOutStream() = default;
virtual bool write(const void* data, size_t size) = 0;
};
class SequentialInStream {
public:
virtual ~SequentialInStream() = default;
virtual bool read(void* data, size_t size, size_t* processedSize) = 0;
};
struct CSeqInStreamWrap {
ISeqInStream vt;
std::unique_ptr<SequentialInStream> inStream;
};
struct CSeqOutStreamWrap {
ISeqOutStream vt;
std::unique_ptr<SequentialOutStream> outStream;
};
class BuffPtrInStream : public SequentialInStream {
public:
explicit BuffPtrInStream(const uint8_t* buffer, size_t bufferSize)
: buffer(buffer), bufferSize(bufferSize) {
}
bool read(void* data, size_t size, size_t* processedSize) override {
if (processedSize) {
*processedSize = 0;
}
if (size == 0 || position >= bufferSize) {
return true;
}
auto remain = bufferSize - position;
if (remain > size) {
remain = size;
}
memcpy(data, static_cast<const uint8_t*>(buffer) + position, remain);
position += remain;
if (processedSize) {
*processedSize = remain;
}
return true;
}
private:
const uint8_t* buffer = nullptr;
size_t bufferSize = 0;
size_t position = 0;
};
class VectorOutStream : public SequentialOutStream {
public:
explicit VectorOutStream(std::vector<uint8_t>* buffer) : buffer(buffer) {
}
bool write(const void* data, size_t size) override {
auto oldSize = buffer->size();
buffer->resize(oldSize + size);
memcpy(&(*buffer)[oldSize], data, size);
return true;
}
private:
std::vector<uint8_t>* buffer;
};
class BuffPtrSeqOutStream : public SequentialOutStream {
public:
BuffPtrSeqOutStream(uint8_t* buffer, size_t size) : buffer(buffer), bufferSize(size) {
}
bool write(const void* data, size_t size) override {
auto remain = bufferSize - position;
if (remain > size) {
remain = size;
}
if (remain != 0) {
memcpy(buffer + position, data, remain);
position += remain;
}
return remain != 0 || size == 0;
}
private:
uint8_t* buffer = nullptr;
size_t bufferSize = 0;
size_t position = 0;
};
static const size_t kStreamStepSize = 1 << 31;
static SRes MyRead(const ISeqInStream* p, void* data, size_t* size) {
CSeqInStreamWrap* wrap = CONTAINER_FROM_VTBL(p, CSeqInStreamWrap, vt);
auto curSize = (*size < kStreamStepSize) ? *size : kStreamStepSize;
if (!wrap->inStream->read(data, curSize, &curSize)) {
return SZ_ERROR_READ;
}
*size = curSize;
return SZ_OK;
}
static size_t MyWrite(const ISeqOutStream* p, const void* buf, size_t size) {
auto* wrap = CONTAINER_FROM_VTBL(p, CSeqOutStreamWrap, vt);
if (wrap->outStream->write(buf, size)) {
return size;
}
return 0;
}
class Lzma2Encoder {
public:
Lzma2Encoder() {
encoder = Lzma2Enc_Create(&gAllocFuncs, &gAllocFuncs);
}
~Lzma2Encoder() {
Lzma2Enc_Destroy(encoder);
}
std::shared_ptr<Data> code(const std::shared_ptr<Data>& inputData) {
if (encoder == nullptr || inputData == nullptr || inputData->size() == 0) {
return nullptr;
}
auto inputSize = inputData->size();
CLzma2EncProps lzma2Props;
Lzma2EncProps_Init(&lzma2Props);
lzma2Props.lzmaProps.dictSize = inputSize;
lzma2Props.lzmaProps.level = 9;
lzma2Props.numTotalThreads = 4;
Lzma2Enc_SetProps(encoder, &lzma2Props);
std::vector<uint8_t> outBuf;
outBuf.resize(1 + 8);
outBuf[0] = Lzma2Enc_WriteProperties(encoder);
for (int i = 0; i < 8; i++) {
outBuf[1 + i] = static_cast<uint8_t>(inputSize >> (8 * i));
}
CSeqInStreamWrap inWrap = {};
inWrap.vt.Read = MyRead;
inWrap.inStream = std::make_unique<BuffPtrInStream>(
static_cast<const uint8_t*>(inputData->data()), inputSize);
CSeqOutStreamWrap outStream = {};
outStream.vt.Write = MyWrite;
outStream.outStream = std::make_unique<VectorOutStream>(&outBuf);
auto status =
Lzma2Enc_Encode2(encoder, &outStream.vt, nullptr, nullptr, &inWrap.vt, nullptr, 0, nullptr);
if (status != SZ_OK) {
return nullptr;
}
return Data::MakeWithCopy(&outBuf[0], outBuf.size());
}
private:
CLzma2EncHandle encoder = nullptr;
};
std::shared_ptr<Data> LzmaUtil::Compress(const std::shared_ptr<Data>& pixelData) {
Lzma2Encoder encoder;
return encoder.code(pixelData);
}
class Lzma2Decoder {
public:
Lzma2Decoder() {
decoder = Lzma2DecMt_Create(&gAllocFuncs, &gAllocFuncs);
}
~Lzma2Decoder() {
if (decoder) {
Lzma2DecMt_Destroy(decoder);
}
}
std::shared_ptr<Data> code(const std::shared_ptr<Data>& inputData) {
if (decoder == nullptr || inputData == nullptr || inputData->size() == 0) {
return nullptr;
}
auto input = static_cast<const uint8_t*>(inputData->data());
auto inputSize = inputData->size() - 9;
Byte prop = static_cast<const Byte*>(input)[0];
CLzma2DecMtProps props;
Lzma2DecMtProps_Init(&props);
props.inBufSize_ST = inputSize;
props.numThreads = 1;
UInt64 outBufferSize = 0;
for (int i = 0; i < 8; i++) {
outBufferSize |= (input[1 + i] << (i * 8));
}
auto outBuffer = new uint8_t[outBufferSize];
CSeqInStreamWrap inWrap = {};
inWrap.vt.Read = MyRead;
inWrap.inStream = std::make_unique<BuffPtrInStream>(input + 9, inputSize);
CSeqOutStreamWrap outWrap = {};
outWrap.vt.Write = MyWrite;
outWrap.outStream = std::make_unique<BuffPtrSeqOutStream>(outBuffer, outBufferSize);
UInt64 inProcessed = 0;
int isMT = false;
auto res = Lzma2DecMt_Decode(decoder, prop, &props, &outWrap.vt, &outBufferSize, 1, &inWrap.vt,
&inProcessed, &isMT, nullptr);
if (res == SZ_OK && inputSize == inProcessed) {
return Data::MakeAdopted(outBuffer, outBufferSize, Data::DeleteProc);
}
delete[] outBuffer;
return nullptr;
}
private:
CLzma2DecMtHandle decoder = nullptr;
};
std::shared_ptr<Data> LzmaUtil::Decompress(const std::shared_ptr<Data>& data) {
Lzma2Decoder decoder;
return decoder.code(data);
}
} // namespace pag

Reading Multiple Files Parallel into a buffer

Am working in a project where i have to read a set of files and put it in a buffer.The List comprises of small as well as large files.I have to read these files and for more efficiency i tried implementing it in multiple threads.Each thread will take a file from vector of file names and start reading it put it into a buffer and these buffer have to be put in a queue.I happened to have some error in program and i i don't know where exactly in my program the error occurs also don't know why ? Please help me whether there is any mistake in my logic or in my code and how to correct it. Thanks in advance
using namespace std;
#define MAX_THREADS 2
#define BUFFER_SIZE 8388608
vector<string>files;
deque<string>bufferq;
CRITICAL_SECTION Readlock;
int count = 0;
DWORD WINAPI ReadThread(LPVOID s);
int main(int argc,char *argv[])
{
HANDLE ReadT[MAX_THREADS];
char *filelist[5];
DWORD threadid;
filelist[0] = "1.txt";
filelist[1] = "cloudy.jpg";
filelist[2] = "connectify.exe";
filelist[3] = "VMware.exe";
filelist[4] = "Sherlock.mp4";
for(int i=0;i<5;i++)
files.push_back(filelist[i]);
InitializeCriticalSection(&Readlock);
long t1 = GetTickCount();
for(int k = 0; k< MAX_THREADS; k++)
ReadT[k] = CreateThread(NULL,0,ReadThread,NULL,NULL,&threadid);
WaitForMultipleObjects(MAX_THREADS,ReadT,TRUE,INFINITE);
cout << " Time Taken "<< GetTickCount()-t1 << "ms" ;
system("pause");
return 0;
}
DWORD WINAPI ReadThread(LPVOID s)
{
long pending = 0;
//int freespace = BUFFER_SIZE;
char *filename = new char[50];
char fsize[10];
string file;
char *buf;
buf = new char[BUFFER_SIZE];
long filesize = 0;
int numfiles = files.size();
int filled = 0;
int i = 0;
FILE *fp;
char* ptr;
ptr = buf;
while(true)
{
EnterCriticalSection(&Readlock);
if(files.empty())
{
LeaveCriticalSection(&Readlock);
break;
}
else
{
file = files.front();
files.erase(files.begin());
LeaveCriticalSection(&Readlock);
}
bool buff_full = false;
buf = ptr;
int freespace = BUFFER_SIZE;
memset(buf,0,BUFFER_SIZE);
if(!buff_full)
{
if(pending == 0)
{
fp = fopen(file.c_str(),"rb");
if(!fp)
{
cout<<"\nNo such file";
cout<<files[i];
system("pause");
return 0;
}
int r1 =fseek(fp, 0L, SEEK_END);
filesize = ftell(fp);
int r2 =fseek(fp, 0L, SEEK_SET);
sprintf(fsize, "%ld", filesize);
if(freespace >= (strlen(fsize) + strlen(file.c_str()) + 2))
{
count++;
memcpy(buf, file.c_str(), strlen(file.c_str())+1);
freespace = freespace - strlen(file.c_str()) - 1;
buf += strlen(file.c_str()) + 1;
memcpy(buf,fsize,strlen(fsize)+1);
buf += strlen(fsize) + 1;
freespace = freespace - strlen(fsize) - 1;
cout<<"Files read is "<<count<<"\n";
if(freespace == 0)
{
buff_full = true;
pending = filesize;
break;
}
}
else
{
filled = BUFFER_SIZE - freespace;
fclose(fp);
break;
}
if(freespace >= filesize)
{
fread(buf, 1, filesize, fp);
buf += filesize;
freespace = freespace - filesize;
bufferq.push_back(buf);
//cout << "pop"<<bufferq.size();
//i++;
if(files.empty())
{
filled = BUFFER_SIZE - freespace;
fclose(fp);
break;
}
fclose(fp);
}
else
{
fread(buf, 1, freespace, fp);
bufferq.push_back(buf);
//cout <<"pop "<<bufferq.size();
buff_full = true;
}
}
else
{
if(freespace >= pending)
{
fread(buf, 1, pending, fp);
bufferq.push_back(buf);
freespace = freespace - pending;
pending = 0;
//i++;
if(files.empty())
{
filled = BUFFER_SIZE - freespace;
fclose(fp);
break;
}
if(freespace > 0)
buf += pending;
else
buff_full = true;
fclose(fp);
}
else
{
fread(buf, 1, freespace, fp);
bufferq.push_back(buf);
cout << bufferq.size();
pending = pending - freespace;
buff_full = true;
}
}
}
if(buff_full)
{
buf = ptr;
cout << "popping buffer " << bufferq.size();
//bufferq.pop_back();
}
}
return 0;
}
In the context that bug occurs on big files, I suppose that this line can cause problems
sprintf(fsize, "%ld", filesize);
fsize is char[10], and if filesize is >= 1,000,000,000 you'll overwrite fsize array with trailing 0. This will cause "Run-Time Check Failure #2 - Stack around the variable 'fsize' was corrupted.", as you wrote. Please check the sizes of your test files.
Among others, you are filling files in loop on i, and then you wrote:
files.erase(files.begin());
// ...
cout<<"\nNo such file";
cout<<files[i];
files[i] already points to another element as you erased them, and if files are empty on the last iteration it will cause crash.
And what for are you copying file and fsize to buf if you do not copy it to the bufferq?
As bufferq is writable and is shared between threads the access to it should be protected by lock, critical section as you chose.
That's my little code review.