English is not my native language; please excuse typing errors. This is code to read dataset from 400+ files with for loop to input in an big array.
float (*HaloPos)[3];//3D-positon(x-y-z)
int Nhalo= 1000000;//Suppose there are 1000000 particles in total number.
void readh5(int numfiles)
{
char fname[256];
int i, npart_cum;
hid_t hdf5_file, hdf5_grp, hdf5_dataspace_in_file;
hid_t hdf5_datatype, hddf5_dataspace_in_memory, hdf5_dataset;
hsize_t dims[2], count[2];
hsize_t start[2];
for(int task = 0; task < numfiles; task ++)
{
sprintf(fname,"%s.%d.hdf5",FILE_DIR,task);
cout<<"file name is"<<fname<<endl;
cout.flush();
read_header_attributes_in_hdf5(fname);
cout<<"npart = "<<header.npart<<endl;
cout.flush();
if(task == 0)
{
Nhalo = header.npartTotal[1];
npart_cum[1] = 0;
}
if(header.npart[1] > 0){
Group grp = file.openGroup("/PartType1");
read_halo(file, grp, task, npart_cum);
grp.close();
}
cout<<"task="<<task<<"halo particles have been read in."<<endl;
cout.flush();
file.close();
npart_cum += header.npart[1];
}
}
static void read_halo(H5File file, Group grp, int task, int npre)
{
hid_t hdf5_dataset;
/*--*/
if(task == 0)
if(!(HaloPos = malloc(sizeof(float)*Nhalo*3)))
exit(100);
printf("allocate positions\n");
fflush(stdout);
hdf5_dataset = H5Dopen(hdf5_grp,"Coordinates");
> get_float3(hdf5_file, hdf5_grp, hdf5_dataset, &HaloPos[npre][0]);
H5Dclose(hdf5_dataset);
}
static void read_header_attributes_in_hdf5(char *fname)
{
hid_thdf5_file,G hdf5_headergrphdf5_attribute;
hdf5_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
hdf5_headergrp = H5Gopen(hdf5_file, "/Header");
hdf5_attribute = H5Aopen_name(hdf5_headergrp, "NumPart_ThisFile");
H5Aread(hdf5_attribute, H5T_NATIVE_INT, &header.npart);
H5Aclose(hdf5_attribute);
}
static void get_float3(H5File file, Group grp, DataSet dset, float *fvec){
/* Open the dataset */
hid_t hdf5_datatype, hdf5_dataspace;
H5T_class_t hdf5_class;
H5T_order_t hdf5_order;
int status_n, rank;
hsize_t dims_out[2];
herr_t status;
hdf5_datatype = H5Dget_type(hdf5_dataset);
hdf5_class = H5Tget_class(hdf5_datatype);
if(hdf5_class != H5T_FLOAT)
exit(53);
hdf5_dataspace = H5Dget_space(hdf5_dataset); /* dataspace handle */
rank = H5Sget_simple_extent_ndims(hdf5_dataspace);
if(rank != 2)
exit(54);
status_n = H5Sget_simple_extent_dims(hdf5_dataspace, dims_out, NULL);
if(dims_out[1] != 3)
exit(55);
status = H5Dread(hdf5_dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL,
H5P_DEFAULT, fvec);
H5Tclose(hdf5_datatype);
}
}
The code is not complete, but the main thing I want to ask everyone is how to use dataset.read instead of H5Dread to read the data and add after the array after one loop
In my limited experience with C++
actually I tried using vector replace array, but I want to know how to write in c++
After I changed background in c++. The code I just changed
status = H5Dread(hdf5_dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL,
H5P_DEFAULT, fvec);
to
dataset.read(fvec,datatype);
And I found the dataset read defination is different in c and c++
in c++
void read ( void * buf,
const DataType & mem_type,
const DataSpace & mem_space = DataSpace::ALL,
const DataSpace & file_space = DataSpace::ALL,
const DSetMemXferPropList & xfer_plist = DSetMemXferPropList::DEFAULT
) const
void read ( H5std_string & buf,
const DataType & mem_type,
const DataSpace & mem_space = DataSpace::ALL,
const DataSpace & file_space = DataSpace::ALL,
const DSetMemXferPropList & xfer_plist = DSetMemXferPropList::DEFAULT
) const
and in c
H5Dread (hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t dxpl_id, void *buf)
Could someone help about how to write in c++?Thanks in advance.
Related
I'm trying to receive audio from the soundcard via RtAudio Api. It has a callback function that gets called once the audio has enough bytes received and the user can then copy the data to a custom object. This custom object can be sent to the callback via pointer. My class that encapsulates RtAudio looks like this:
class Audio {
private:
AudioConfiguration config;
AudioData data;
RtAudio* rt;
// the void* d is the casted AudioData-object
static int input( void*, void* inputBuffer, unsigned int bufferSize, double, RtAudioStreamStatus status, void* d );
void openStream( RtAudio::StreamParameters& params, RtAudio::StreamOptions& options, AudioConfiguration& config );
bool isDeviceOk();
public:
// ctor & dtor
Audio( AudioConfiguration& c );
~Audio();
// copy ctor & assignment
Audio( const Audio& other );
Audio& operator=( const Audio& a );
// move ctor & assignment
Audio( Audio&& other );
Audio& operator=( Audio&& a);
AudioConfiguration& getConfiguration();
AudioData& getData();
void start();
void stop();
};
This is the implementation of the static function that gets called from inside the audio thread
int Audio::input( void*, void* inputBuffer, unsigned int bufferSize, double, RtAudioStreamStatus status, void* d ){
if( status == RTAUDIO_INPUT_OVERFLOW )
std::cout << "Audio Thread: Input overflow detected." << std::endl;
//std::cout << "Audio Thread: Received input from soundcard" << std::endl;
float* in = static_cast<float*>( inputBuffer );
AudioData* data = static_cast<AudioData*>( d );
boost::lock_guard<boost::mutex> lock{data->getMutex()};
unsigned int i = 0;
while( i < bufferSize ){
data->getBuffer().push_back( *in );
in++;
i++;
}
return 0;
}
The custom object that I share between the threads is of the class AudioData, which looks like this:
class AudioData {
private:
boost::circular_buffer<float> buffer;
boost::mutex mutex;
public:
AudioData();
~AudioData();
boost::circular_buffer<float>& getBuffer();
boost::mutex& getMutex();
};
The audio-object gets embedded in a Recorder-Object, which then reads the buffer in the AudioData member variable of Audio.
typedef boost::container::vector<boost::circular_buffer<float>> Buffers;
class Recorder {
private:
Audio audio;
Buffers buffers;
/*
* Reads n samples per channel from audio buffers to NUM_CHANNELS distinct buffers
* When done, it returns the number of samples read into each channel
* Why? We want to keep audio buffer to be locked as minimal time as possible
*/
unsigned int read( unsigned int samples );
/*
* Deletes n samples from channel buffer
* When done, it returns the number of samples deleted
*/
unsigned int deleteBegin( unsigned int ch, unsigned int samples );
/*
* Detects the trigger on TRIGGER_CHANNEL
* Returns true, if trigger was found and its position
*/
bool detectTrigger( unsigned int* pos );
public:
Recorder( AudioConfiguration& c );
Recorder( Audio&& a );
boost::container::vector<float> record( RecorderConfiguration& config );
};
The function record(..) looks like this:
boost::container::vector<float> Recorder::record( RecorderConfiguration& config ){
AudioConfiguration audioConfig = audio.getConfiguration();
unsigned int length = ( audioConfig.fs * config.length ) / 1000;
boost::container::vector<float> recording;
recording.resize( length );
// Tell Audio to start recording
audio.start();
// State
unsigned int times = 0; // Count averages
unsigned int left = length; // Samples left on current average
bool triggered = false; // Trigger has been read
while( true ){
// Read into local buffer
unsigned int samplesToRead = length / 10;
unsigned int samplesRead = read( samplesToRead );
// if not enough samples, wait for more
if( samplesRead < 100 )
continue;
// Actual logic
unsigned int triggerPos = 0;
if( !triggered && detectTrigger( &triggerPos ) ){
std::cout << "Recorder: Trigger detected." << std::endl;
triggered = true;
// delete everything that comes before trigger on both channels
for( unsigned int i = 0 ; i < NUM_CHANNELS ; i++ ){
deleteBegin( i, triggerPos - 1);
}
}
// Copy from buffer if trigger was found beforehand
if( triggered ){
boost::circular_buffer<float>& buffer = buffers[ EEG_CHANNEL ];
unsigned int samplesToCopy = buffer.size();
if( samplesToCopy > left )
samplesToCopy = left;
for( unsigned int i = 0 ; i < samplesToCopy ; i++ ){
recording[ length - left ] = recording[ left - left ] + buffer.front();
buffer.pop_front();
left--;
}
}
// current average done
if( left <= 0 ){
// increment times
times++;
// check if recording is done
if( times >= config.times )
break;
// if not
else {
triggered = false;
left = length;
}
}
}
// Stop receiving input from audio
audio.stop();
return recording;
}
I read that the heap is the place to hold data that is shared between threads, but in the example by rtaudio they use a global variable that gets allocated on the stack for pushing the data to Link. So I am a little bit confused. Help would be gladly accepted!
Edit: When i debug my app. I can see that the input function of the audio-thread gets called and it writes to the buffer. Also the record function works as expected. Only the buffer (of AudioData) does not seem to have any data in it...
Edit2: Here is the code where I register the callback in the rtaudio api.
void Audio::openStream( RtAudio::StreamParameters& params, RtAudio::StreamOptions& options, AudioConfiguration& config ){
try {
rt->openStream( nullptr, ¶ms, RTAUDIO_FLOAT32, config.fs, &config.bufferSize, &this->input, &data, &options, nullptr );
} catch( RtAudioError& e ){
std::cout << "Audio::openStream(): Cannot open stream." << std::endl;
throw e;
}
}
Hi i am trying to record from a board and i have successfully record 4 seconds. Problem is when i try to record for more time, i got an error telling me that there not enough memory. my target is to record a 5 minutes file. Until now i have create a buffer named snIn[256] where are the samples. i send it to a big buffer of [16K * 4sec] and when it is full, i create the wav file.
#include "SAI_InOut.hpp"
#include "F746_GUI.hpp"
#include "Delay.hpp"
#include "WaveformDisplay.hpp"
#include "SDFileSystem.h"
#include "wavfile.h"
using namespace Mikami;
#define RES_STR_SIZE 0x20
#define WAVFILE_SAMPLES_PER_SECOND 16000
#define REC_TIME 4
//Create an SDFileSystem object
SDFileSystem sd("sd");
bool flag = 1;
int count = 0;
char *res_buf;
int rp = 0;
const int NUM_SAMPLES = WAVFILE_SAMPLES_PER_SECOND * REC_TIME;
Array<int16_t> my_buffer(NUM_SAMPLES);
int j = 0;
static const char *target_filename = "/sd/rectest.wav";
const int SEG_SIZE = 256;
int sent_array = 0;
int rec(const char *filename, Array<int16_t> my_buffer)
{
j = 0;
flag = 0;
sent_array = 0;
WavFileResult result;
wavfile_info_t info;
wavfile_data_t data;
WAVFILE_INFO_AUDIO_FORMAT(&info) = 1;
WAVFILE_INFO_NUM_CHANNELS(&info) = 1;
WAVFILE_INFO_SAMPLE_RATE(&info) = WAVFILE_SAMPLES_PER_SECOND;
WAVFILE_INFO_BITS_PER_SAMPLE(&info) = 16;
WAVFILE_INFO_BYTE_RATE(&info) = WAVFILE_INFO_NUM_CHANNELS(&info) * WAVFILE_INFO_SAMPLE_RATE(&info) * (WAVFILE_INFO_BITS_PER_SAMPLE(&info) / 8);
WAVFILE_INFO_BLOCK_ALIGN(&info) = 2;
WAVFILE *wf = wavfile_open(filename, WavFileModeWrite, &result);
if (result != WavFileResultOK) {
wavfile_result_string(result, res_buf, RES_STR_SIZE);
printf("%s", res_buf);
return result;
} else printf ("Open file success \r\n");
rp = 0;
WAVFILE_DATA_NUM_CHANNELS(&data) = 1;
result = wavfile_write_info(wf, &info);
if (result != WavFileResultOK) {
wavfile_result_string(result, res_buf, RES_STR_SIZE);
printf("%s", res_buf);
return result; } else printf ("Write info success \r\n");
while ( rp < NUM_SAMPLES ) {
WAVFILE_DATA_CHANNEL_DATA(&data, 0) = my_buffer[rp];
result = wavfile_write_data(wf, &data);
rp += 1;
}
if (result != WavFileResultOK) {
wavfile_result_string(result, res_buf, RES_STR_SIZE);
printf("%s", res_buf);
return result; } else printf ("Write Data file success \r\n");
result = wavfile_close(wf);
if (result != WavFileResultOK) {
wavfile_result_string(result, res_buf , RES_STR_SIZE);
printf("%s", res_buf);
return result; } else printf ("Close file success \r\n");
//UnMount the filesystem
sd.unmount();
printf("Success rec !\r\n");
return 0;
}
int main()
{
//Mount the filesystem
sd.mount();
const float MAX_DELAY = 0.5f; // 最大遅延,単位:秒
const int FS = I2S_AUDIOFREQ_16K; // 標本化周波数: 16 kHz
const uint32_t MAX_ARRAY_SIZE = (uint32_t)(MAX_DELAY*FS);
SaiIO mySai(SaiIO::BOTH, 256, FS, INPUT_DEVICE_DIGITAL_MICROPHONE_2);
Label myLabel(185, 10, "Delay System", Label::CENTER, Font16);
// ButtonGroup: "ON", "OFF"
const uint16_t BG_LEFT = 370;
const uint16_t BG_WIDTH = 100;
const uint16_t BG_HEIGHT = 45;
ButtonGroup onOff(BG_LEFT, 40, BG_WIDTH/2, BG_HEIGHT,
2, (string[]){"ON", "OFF"}, 0, 0, 2, 1);
const uint16_t SB_LEFT = BG_LEFT - 320;
const uint16_t SB_WIDTH = 270;
const uint16_t SB_Y0 = 240;
char str[20];
sprintf(str, " %3.1f [s]", MAX_DELAY);
SeekBar barDelay(SB_LEFT, SB_Y0, SB_WIDTH,
0, MAX_ARRAY_SIZE, 0, "0", "", str);
NumericLabel<float> labelDelay(SB_LEFT+SB_WIDTH/2, SB_Y0-40, "DELEY: %4.2f", 0, Label::CENTER);
DelaySystem delaySystem(MAX_ARRAY_SIZE);
WaveformDisplay displayIn(*GuiBase::GetLcdPtr(), SB_LEFT+7, 70, 256, 9,LCD_COLOR_WHITE, LCD_COLOR_CYAN,GuiBase::ENUM_BACK);
Label inLabel(SB_LEFT-30, 65, "IN");
WaveformDisplay displayOut(*GuiBase::GetLcdPtr(), SB_LEFT+7, 130, 256, 9,LCD_COLOR_WHITE, LCD_COLOR_CYAN,GuiBase::ENUM_BACK);
Label outLabel(SB_LEFT-30, 125, "OUT");
int runStop = 1;
Array<int16_t> snIn(mySai.GetLength());
Array<int16_t> snOut(mySai.GetLength());
mySai.RecordIn();
mySai.PlayOut();
mySai.PauseOut();
while (true)
{
// On/OFF
int num;
if (onOff.GetTouchedNumber(num))
if (runStop != num)
{
if (num == 0) mySai.ResumeOut();
else mySai.PauseOut();
runStop = num;
}
if (mySai.IsCompleted())
{
for (int n=0; n<mySai.GetLength() ; n++)
{
int16_t xL, xR;
mySai.Input(xL,xR);
int16_t xn = xL + xR;
snIn[n] = xn;
my_buffer[j] = xn;
j++;
if (j == NUM_SAMPLES && flag == 1) {
rec (target_filename , my_buffer); }
int16_t yn = delaySystem.Execute(xn);
mySai.Output(yn, yn);
snOut[n] = yn;
}
mySai.Reset();
displayIn.Execute(snIn);
}
}
}
I thought about a possible solution, to fill directly the "data field" of the wavefile with the snIn[256] buffer (instead of using my_buffer) again and again and at the end close the wavfile. Please let me know what you think about that and other solutions
things to note: 1) while a write operation is being performed, more data is still coming in.
At the very least I would double buffer that data, so can be writing one buffer while the other one fills.
Usually this means using an interrupt to collect the samples (into which ever buffer is currently being filed.)
the foreground program waits for the current buffer to be 'full', then initiates write operation.,
then waits again for a buffer to be 'full'
The interrupt function tracks which buffer is being filled and the current index into that buffer. When a buffer is full, set a 'global' status to let the foreground program know which buffer is ready to be written.
The foreground program writes the buffer, then resets the status for that buffer.
I am using an array of structs to store different binary data, from different clients. While I am debugging, I can do a few iterations with success (in memcpy). But at some point the debug crashes with "unhandled exception".
struct Buffer {
int size_ = 0;
int capacity_total = 200000;
int beg_index = 0
int end_index = 0;
char data_[200000];
} buffer_audio[3];
int writing_bufer(Buffer& buffers, const char *data, int nbytes) {
if (nbytes == 0) return 0;
int capacity = buffers.capacity_total;
if (nbytes <= capacity - buffers.end_index)
{
memcpy(buffers.data_ + buffers.end_index, data, nbytes); //crashes here
buffers.end_index += nbytes;
if (buffers.end_index == capacity) printf("full");
}
else {
return 0; }
return buffers.end_index;
}
The buffer never is full or close of that.
The full routine:
void buffering(const FunctionCallbackInfo<v8::Value>& args) {
Isolate* isolate = Isolate::GetCurrent();
HandleScope scope(isolate);
int size = args[1]->NumberValue();
int final_read = args[2]->NumberValue();
int client_id = args[3]->NumberValue();
int nbytes = args[4]->NumberValue();
(...)
buf = node::Buffer::Data(bufferObj);
buffering_mem(buf,size, final_read, client_id,nbytes);
Local<String> devolve = String::NewFromUtf8(isolate, "buffering_com_sucesso");//C++--->JS
args.GetReturnValue().Set(devolve);
}
void buffering_mem(char* chunk,int size_chunk, int close_file, int client, int total_size){
int check_bytes = writing_bufer(buffer_audio[client], chunk, size_chunk);
//other code}
You are copying the wrong amount in your code:
memcpy(buffers.data_ + buffers.end_index, data, buffers.end_index+nbytes);
That should be just
memcpy(buffers.data_ + buffers.end_index, data, nbytes);
With the huge help of #JSF
void buffering_mem(char* chunk,int size_chunk, int close_file, int client, int total_size){
//old place of invocation
if (close_file == 3) {
fp0 = fopen("buffer_inteiro", "wb");
fwrite(buffer_audio[client].data_, 1,total_size,fp0);
fflush(fp0); return;
}
int check_bytes = writing_bufer(buffer_audio[client], chunk, size_chunk);
}
I am trying to create a function that uncompresses LZAM2 compressed data. I inspired myself from this tutorial which works great for LZMA and I tried to adapt it for LZMA2. I successfully created the compression function for LZMA2, but i have no success for the uncompression one.
Here is the compression function:
static void Compress2Inc(std::vector<unsigned char> &outBuf,
const std::vector<unsigned char> &inBuf)
{
CLzma2EncHandle enc = Lzma2Enc_Create(&SzAllocForLzma, &SzAllocForLzma2);
assert(enc);
CLzma2EncProps props;
Lzma2EncProps_Init(&props);
props.lzmaProps.writeEndMark = 1; // 0 or 1
SRes res = Lzma2Enc_SetProps(enc, &props);
assert(res == SZ_OK);
unsigned propsSize = LZMA_PROPS_SIZE;
outBuf.resize(propsSize);
res = Lzma2Enc_WriteProperties(enc);
//cout << res;
//assert(res == SZ_OK && propsSize == LZMA_PROPS_SIZE);
VectorInStream inStream = { &VectorInStream_Read, &inBuf, 0 };
VectorOutStream outStream = { &VectorOutStream_Write, &outBuf };
res = Lzma2Enc_Encode(enc,
(ISeqOutStream*)&outStream, (ISeqInStream*)&inStream,
0);
assert(res == SZ_OK);
Lzma2Enc_Destroy(enc);
}
Where:
static void *AllocForLzma2(void *, size_t size) { return BigAlloc(size); }
static void FreeForLzma2(void *, void *address) { BigFree(address); }
static ISzAlloc SzAllocForLzma2 = { AllocForLzma2, FreeForLzma2 };
static void *AllocForLzma(void *, size_t size) { return MyAlloc(size); }
static void FreeForLzma(void *, void *address) { MyFree(address); }
static ISzAlloc SzAllocForLzma = { AllocForLzma, FreeForLzma };
typedef struct
{
ISeqInStream SeqInStream;
const std::vector<unsigned char> *Buf;
unsigned BufPos;
} VectorInStream;
SRes VectorInStream_Read(void *p, void *buf, size_t *size)
{
VectorInStream *ctx = (VectorInStream*)p;
*size = min(*size, ctx->Buf->size() - ctx->BufPos);
if (*size)
memcpy(buf, &(*ctx->Buf)[ctx->BufPos], *size);
ctx->BufPos += *size;
return SZ_OK;
}
typedef struct
{
ISeqOutStream SeqOutStream;
std::vector<unsigned char> *Buf;
} VectorOutStream;
size_t VectorOutStream_Write(void *p, const void *buf, size_t size)
{
VectorOutStream *ctx = (VectorOutStream*)p;
if (size)
{
unsigned oldSize = ctx->Buf->size();
ctx->Buf->resize(oldSize + size);
memcpy(&(*ctx->Buf)[oldSize], buf, size);
}
return size;
}
Here is what I have so far with the uncompression function but Lzma2Dec_DecodeToBuf function returns error code 1(SZ_ERROR_DATA) and I just couldn't find anything on the web regarding this on the web.
static void Uncompress2Inc(std::vector<unsigned char> &outBuf,
const std::vector<unsigned char> &inBuf)
{
CLzma2Dec dec;
Lzma2Dec_Construct(&dec);
SRes res = Lzma2Dec_Allocate(&dec, outBuf.size(), &SzAllocForLzma);
assert(res == SZ_OK);
Lzma2Dec_Init(&dec);
outBuf.resize(UNCOMPRESSED_SIZE);
unsigned outPos = 0, inPos = LZMA_PROPS_SIZE;
ELzmaStatus status;
const unsigned BUF_SIZE = 10240;
while (outPos < outBuf.size())
{
unsigned destLen = min(BUF_SIZE, outBuf.size() - outPos);
unsigned srcLen = min(BUF_SIZE, inBuf.size() - inPos);
unsigned srcLenOld = srcLen, destLenOld = destLen;
res = Lzma2Dec_DecodeToBuf(&dec,
&outBuf[outPos], &destLen,
&inBuf[inPos], &srcLen,
(outPos + destLen == outBuf.size()) ? LZMA_FINISH_END : LZMA_FINISH_ANY,
&status);
assert(res == SZ_OK);
inPos += srcLen;
outPos += destLen;
if (status == LZMA_STATUS_FINISHED_WITH_MARK)
break;
}
Lzma2Dec_Free(&dec, &SzAllocForLzma);
outBuf.resize(outPos);
}
I am using Visual Studio 2008 and LZMA SDK downloaded from here. Someone here had the exact same problem, but i couldn't make use of his code...
Did anyone ever successfully uncompressed LZMA2 compressed files using LZMA SDK?
Please help!
A temporary workaround would be to replace SRes res = Lzma2Dec_Allocate(&dec, outBuf.size(), &SzAllocForLzma); with SRes res = Lzma2Dec_Allocate(&dec, 8, &SzAllocForLzma); in Uncompress2Inc function where 8 is a magic number...
However this is not the right way to solve the problem...
First mistake is that Lzma2Enc_WriteProperties doesn't return a result but a property byte which will have to be used as the second parameter of the Lzma2Dec_Allocate call in the Uncompress2Inc function. As a result we replace the magic number 8 with the property byte and everything works as expected.
In order to achieve this a 5 byte header must be added to the encoded data which will be extracted in the decoding function. Here is an example that works in VS2008(not the most perfect code but it works...I will get back later, when I have time, with a better example):
void Lzma2Benchmark::compressChunk(std::vector<unsigned char> &outBuf, const std::vector<unsigned char> &inBuf)
{
//! \todo This is a temporary workaround, size needs to be added to the
m_uncompressedSize = inBuf.size();
std::cout << "Uncompressed size is: " << inBuf.size() << std::endl;
DWORD tickCountBeforeCompression = GetTickCount();
CLzma2EncHandle enc = Lzma2Enc_Create(&m_szAllocForLzma, &m_szAllocForLzma2);
assert(enc);
CLzma2EncProps props;
Lzma2EncProps_Init(&props);
props.lzmaProps.writeEndMark = 1; // 0 or 1
props.lzmaProps.level = 9;
props.lzmaProps.numThreads = 3;
//props.numTotalThreads = 2;
SRes res = Lzma2Enc_SetProps(enc, &props);
assert(res == SZ_OK);
// LZMA_PROPS_SIZE == 5 bytes
unsigned propsSize = LZMA_PROPS_SIZE;
outBuf.resize(propsSize);
// I think Lzma2Enc_WriteProperties returns the encoding properties in 1 Byte
Byte properties = Lzma2Enc_WriteProperties(enc);
//! \todo This is a temporary workaround
m_propByte = properties;
//! \todo Here m_propByte and m_uncompressedSize need to be added to outBuf's 5 byte header so simply add those 2 values to outBuf and start the encoding from there.
BenchmarkUtils::VectorInStream inStream = { &BenchmarkUtils::VectorInStream_Read, &inBuf, 0 };
BenchmarkUtils::VectorOutStream outStream = { &BenchmarkUtils::VectorOutStream_Write, &outBuf };
res = Lzma2Enc_Encode(enc,
(ISeqOutStream*)&outStream,
(ISeqInStream*)&inStream,
0);
std::cout << "Compress time is: " << GetTickCount() - tickCountBeforeCompression << " milliseconds.\n";
assert(res == SZ_OK);
Lzma2Enc_Destroy(enc);
std::cout << "Compressed size is: " << outBuf.size() << std::endl;
}
void Lzma2Benchmark::unCompressChunk(std::vector<unsigned char> &outBuf, const std::vector<unsigned char> &inBuf)
{
DWORD tickCountBeforeUncompression = GetTickCount();
CLzma2Dec dec;
Lzma2Dec_Construct(&dec);
//! \todo Heere the property size and the uncompressed size need to be extracted from inBuf, which is the compressed data.
// The second parameter is a temporary workaround.
SRes res = Lzma2Dec_Allocate(&dec, m_propByte/*8*/, &m_szAllocForLzma);
assert(res == SZ_OK);
Lzma2Dec_Init(&dec);
outBuf.resize(m_uncompressedSize);
unsigned outPos = 0, inPos = LZMA_PROPS_SIZE;
ELzmaStatus status;
const unsigned BUF_SIZE = 10240;
while(outPos < outBuf.size())
{
SizeT destLen = std::min(BUF_SIZE, outBuf.size() - outPos);
SizeT srcLen = std::min(BUF_SIZE, inBuf.size() - inPos);
SizeT srcLenOld = srcLen, destLenOld = destLen;
res = Lzma2Dec_DecodeToBuf(&dec,
&outBuf[outPos],
&destLen,
&inBuf[inPos],
&srcLen,
(outPos + destLen == outBuf.size()) ? LZMA_FINISH_END : LZMA_FINISH_ANY,
&status);
assert(res == SZ_OK);
inPos += srcLen;
outPos += destLen;
if(status == LZMA_STATUS_FINISHED_WITH_MARK)
{
break;
}
}
Lzma2Dec_Free(&dec, &m_szAllocForLzma);
outBuf.resize(outPos);
std::cout << "Uncompress time is: " << GetTickCount() - tickCountBeforeUncompression << " milliseconds.\n";
}
I have a SSD and I am trying to use it to simulate my program I/O performance, however, IOPS calculated from my program is much much faster than IOMeter.
My SSD is PLEXTOR PX-128M3S, by IOMeter, its max 512B random read IOPS is around 94k (queue depth is 32).
However my program (32 windows threads) can reach around 500k 512B IOPS, around 5 times of IOMeter! I did data validation but didn't find any error in data fetching. It's because my data fetching in order?
I paste my code belwo (it mainly fetch 512B from file and release it; I did use 4bytes (an int) to validate program logic and didn't find problem), can anybody help me figure out where I am wrong?
Thanks so much in advance!!
#include <stdio.h>
#include <Windows.h>
//Global variables
long completeIOs = 0;
long completeBytes = 0;
int threadCount = 32;
unsigned long long length = 1073741824; //4G test file
int interval = 1024;
int resultArrayLen = 320000;
int *result = new int[resultArrayLen];
//Method declarison
double GetSecs(void); //Calculate out duration
int InitPool(long long,char*,int); //Initialize test data for testing, if successful, return 1; otherwise, return a non 1 value.
int * FileRead(char * path);
unsigned int DataVerification(int*, int sampleItem); //Verify data fetched from pool
int main()
{
int sampleItem = 0x1;
char * fPath = "G:\\workspace\\4G.bin";
unsigned int invalidIO = 0;
if (InitPool(length,fPath,sampleItem)!= 1)
printf("File write err... \n");
//start do random I/Os from initialized file
double start = GetSecs();
int * fetchResult = FileRead(fPath);
double end = GetSecs();
printf("File read IOPS is %.4f per second.. \n",completeIOs/(end - start));
//start data validation, for 4 bytes fetch only
// invalidIO = DataVerification(fetchResult,sampleItem);
// if (invalidIO !=0)
// {
// printf("Total invalid data fetch IOs are %d", invalidIO);
// }
return 0;
}
int InitPool(long long length, char* path, int sample)
{
printf("Start initializing test data ... \n");
FILE * fp = fopen(path,"wb");
if (fp == NULL)
{
printf("file open err... \n");
exit (-1);
}
else //initialize file for testing
{
fseek(fp,0L,SEEK_SET);
for (int i=0; i<length; i++)
{
fwrite(&sample,sizeof(int),1,fp);
}
fclose(fp);
fp = NULL;
printf("Data initialization is complete...\n");
return 1;
}
}
double GetSecs(void)
{
LARGE_INTEGER frequency;
LARGE_INTEGER start;
if(! QueryPerformanceFrequency(&frequency))
printf("QueryPerformanceFrequency Failed\n");
if(! QueryPerformanceCounter(&start))
printf("QueryPerformanceCounter Failed\n");
return ((double)start.QuadPart/(double)frequency.QuadPart);
}
class input
{
public:
char *path;
int starting;
input (int st, char * filePath):starting(st),path(filePath){}
};
//Workers
DWORD WINAPI FileReadThreadEntry(LPVOID lpThreadParameter)
{
input * in = (input*) lpThreadParameter;
char* path = in->path;
FILE * fp = fopen(path,"rb");
int sPos = in->starting;
// int * result = in->r;
if(fp != NULL)
{
fpos_t pos;
for (int i=0; i<resultArrayLen/threadCount;i++)
{
pos = i * interval;
fsetpos(fp,&pos);
//For 512 bytes fetch each time
unsigned char *c =new unsigned char [512];
if (fread(c,512,1,fp) ==1)
{
InterlockedIncrement(&completeIOs);
delete c;
}
//For 4 bytes fetch each time
/*if (fread(&result[sPos + i],sizeof(int),1,fp) ==1)
{
InterlockedIncrement(&completeIOs);
}*/
else
{
printf("file read err...\n");
exit(-1);
}
}
fclose(fp);
fp = NULL;
}
else
{
printf("File open err... \n");
exit(-1);
}
}
int * FileRead(char * p)
{
printf("Starting reading file ... \n");
HANDLE mWorkThread[256]; //max 256 threads
completeIOs = 0;
int slice = int (resultArrayLen/threadCount);
for(int i = 0; i < threadCount; i++)
{
mWorkThread[i] = CreateThread(
NULL,
0,
FileReadThreadEntry,
(LPVOID)(new input(i*slice,p)),
0,
NULL);
}
WaitForMultipleObjects(threadCount, mWorkThread, TRUE, INFINITE);
printf("File read complete... \n");
return result;
}
unsigned int DataVerification(int* result, int sampleItem)
{
unsigned int invalid = 0;
for (int i=0; i< resultArrayLen/interval;i++)
{
if (result[i]!=sampleItem)
{
invalid ++;
continue;
}
}
return invalid;
}
I didn't look in enough detail to be certain, but I didn't see any code there to flush the data to the disk and/or ensure your reads actually came from the disk. That being the case, it appears that what you're measuring is primarily the performance of the operating system's disk caching. While the disk might contribute a little to the performance you're measuring, it's probably only a small contributor, with other factors dominating.
Since the code is apparently written for Windows, you might consider (for one example) opening the file with CreateFile, and passing the FILE_FLAG_NO_BUFFERING flag when you do so. This will (at least mostly) remove the operating system cache from the equation, and force each read or write to deal directly with the disk itself.