I am trying to receive and save file over 5gb size using c++. But during the course of the process, memory used by the application is increasing and sometimes the application crashes. Is there something I am doing wrong ? Or is there a better way to do this?
char * buffer = channel->cread(&len);
__int64_t lengthFile = *(__int64_t * ) buffer;
__int64_t received = 0;
__int64_t offset = 0;
__int64_t length;
string filename = "received/testFile";
ofstream ifs;
ifs.open(filename.c_str(),ios::binary | ios::out);
int len = 0;
while(1){
length = 256;
if(offset + MAX_MESSAGE >= lengthFile){
length = lengthFile - offset;
breakCondition = true;
}
file = new filemsg(offset,length);
channel->cwrite((char *)file,sizeof (*file));
buffer = channel->cread(&len);
ifs.write(buffer,length);
received = received + length;
offset = offset + MAX_MESSAGE;
if(breakCondition)
break;
}
Related
This was solved by changing the buffer from int16_t to int8_t since I was trying to write 8bit audio.
I'm trying to fill a buffer for a mono wave file with two different frequencies but failing at it. I'm using CLion in Ubuntu 18.04.
I know, the buffer size is equal to duration*sample_rate so I'm creating a int16_t vector with that size. I tried filling it with one note first.
for(int i = 0; i < frame_total; i++)
audio[i] = static_cast<int16_t>(128 + 127 * sin(i));
which generated a nice long beeep. And then I changed it with:
for(int i = 0; i < frame_total; i++)
audio[i] = static_cast<int16_t>(128 + 127 * sin(i*2));
which generated a higher beeeep, but when trying to do the following:
for(int i = 0; i < frame_total/2; i++)
audio[i] = static_cast<int16_t>(128 + 127 * sin(i*2));
for(int i = frame_total/2; i < frame_total; i++)
audio[i] = static_cast<int16_t>(128 + 127 * sin(i));
I expect it to write the higher beep in the first half of the audio, and fill the another fall with the "normal" beep. The *.wav file just plays the first note the entire time.
#define FORMAT_AUDIO 1
#define FORMAT_SIZE 16
struct wave_header{
// Header
char riff[4];
int32_t file_size;
char wave[4];
// Format
char fmt[4];
int32_t format_size;
int16_t format_audio;
int16_t num_channels;
int32_t sample_rate;
int32_t byte_rate;
int16_t block_align;
int16_t bits_per_sample;
// Data
char data[4]
int32_t data_size;
};
void write_header(ofstream &music_file ,int16_t bits, int32_t samples, int32_t duration){
wave_header wav_header{};
int16_t channels_quantity = 1;
int32_t total_data = duration * samples * channels_quantity * bits/8;
int32_t file_data = 4 + 8 + FORMAT_SIZE + 8 + total_data;
wav_header.riff[0] = 'R';
wav_header.riff[1] = 'I';
wav_header.riff[2] = 'F';
wav_header.riff[3] = 'F';
wav_header.file_size = file_data;
wav_header.wave[0] = 'W';
wav_header.wave[1] = 'A';
wav_header.wave[2] = 'V';
wav_header.wave[3] = 'E';
wav_header.fmt[0] = 'f';
wav_header.fmt[1] = 'm';
wav_header.fmt[2] = 't';
wav_header.fmt[3] = ' ';
wav_header.format_size = FORMAT_SIZE;
wav_header.format_audio = FORMAT_AUDIO;
wav_header.num_channels = channels_quantity;
wav_header.sample_rate = samples;
wav_header.byte_rate = samples * channels_quantity * bits/8;
wav_header.block_align = static_cast<int16_t>(channels_quantity * bits / 8);
wav_header.bits_per_sample = bits;
wav_header.data[0] = 'd';
wav_header.data[1] = 'a';
wav_header.data[2] = 't';
wav_header.data[3] = 'a';
wav_header.data_size = total_data;
music_file.write((char*)&wav_header, sizeof(wave_header));
}
int main(int argc, char const *argv[]) {
int16_t bits = 8;
int32_t samples = 44100;
int32_t duration = 4;
ofstream music_file("music.wav", ios::out | ios::binary);
int32_t frame_total = samples * duration;
auto* audio = new int16_t[frame_total];
for(int i = 0; i < frame_total/2; i++)
audio[i] = static_cast<int16_t>(128 + 127 * sin(i*2));
for(int i = frame_total/2; i < frame_total; i++)
audio[i] = static_cast<int16_t>(128 + 127 * sin(i));
write_header(music_file, bits, samples, duration);
music_file.write(reinterpret_cast<char*>(audio),sizeof(int16_t)*frame_total);
return 0;
}
There are two major issues with your code.
The first one is that you may be writing an invalid header depending on your compiler settings and environment.
The reason is that the wave_header struct is not packed in memory, and may contain padding between members. Therefore, when you do:
music_file.write((char*)&wav_header, sizeof(wave_header));
It may write something that isn't a valid WAV header. Even if you are lucky enough to get exactly what you wanted, it is a good idea to fix it, because it may change at any moment and surely it isn't portable.
The second issue is that the call to write the actual wave:
music_file.write(reinterpret_cast<char*>(audio),sizeof(char)*frame_total);
Is writing exactly half the amount of data you are expecting. The actual size of the data pointed by audio is sizeof(int16_t) * frame_total.
This explains why you are only hearing the first part of the wave you wrote.
This was solved by changing the buffer (audio) from int16_t to int8_t since I was trying to write 8bit audio.
I am testing Broadcast Upload Extension under iOS11.4.1 or 12.0 with iPad6.
After extracting the YUV data from CMSampleBufferRef, and saving those data to a file, I get some bad images. The issue seems like that the last frame image still remains on the new frame, and not refreshed. Maybe the data is not ready, when coming out from callback.I wonders how to avoid this issue.
one bad image here
another bad image
- (void)processSampleBuffer:(CMSampleBufferRef)sampleBuffer withType:(RPSampleBufferType)sampleBufferType {
switch (sampleBufferType) {
case RPSampleBufferTypeVideo:
{
CFRetain(sampleBuffer);
size_t bytes = 0;
char* data = NULL;
size_t bufwidth, bufheight, bufstride;
CVPixelBufferRef pixelbuf = CMSampleBufferGetImageBuffer(sampleBuffer);
CVReturn cr = CVPixelBufferLockBaseAddress(pixelbuf, kCVPixelBufferLock_ReadOnly);
for (size_t i = 0; i < CVPixelBufferGetPlaneCount(pixelbuf); i++)
{
bufwidth = CVPixelBufferGetWidthOfPlane(pixelbuf, i);
bufheight = CVPixelBufferGetHeightOfPlane(pixelbuf, i);
bufstride = CVPixelBufferGetBytesPerRowOfPlane(pixelbuf,i);
data = (char*)CVPixelBufferGetBaseAddressOfPlane(pixelbuf,i);
if(bufwidth == bufstride)
{
size_t ylen = bufwidth*bufheight;
fwrite(data, ylen, 1, _file_yuv);
}
else
{
size_t factor = bufstride/bufwidth;
bytes = bufwidth * factor;
for (j = 0; j < bufheight; j++)
{
fwrite(data, bytes, 1, _file_yuv);
data += bufstride;
}
}
}
CVPixelBufferUnlockBaseAddress(pixelbuf, kCVPixelBufferLock_ReadOnly);
CFRelease(sampleBuffer);
}
break;
case RPSampleBufferTypeAudioApp:
break;
default:
break;
}
}
I hope you can help me. I've got this function that I use to save bitmap images but I'm having trouble with the file name. I would like to save the image with the current date on the file name but how I'm currently doing it it's giving me the "An invalid parameter was passed to a function that considers invalid parameters fatal" error and the program crashes.
Here is the function that gets the name from the main program:
void DoSave(char *Name)
{
time_t rawTime;
struct tm *timeInfo;
char buffer[100];
string date;
time(&rawTime);
timeInfo = localtime(&rawTime);
strftime(buffer, 100, " %c", timeInfo);
date = buffer;
string url("Images/");
string name(Name);
string extension(".bmp");
string path = url + name + date + extension;
// Open the file for writing
FILE *f = fopen((const char*)&path, "wb");
// Save the image as bmp file
BmpHelper::SaveTo8bppBmpFile(f, (LONG)m_ImageSizeX, (LONG)m_ImageSizeY, (unsigned char*)videoImage);
fclose(f);
}
Here is the function that saves the bitmaps
void BmpHelper::SaveTo8bppBmpFile(FILE *output, LONG width, LONG height, unsigned char *buffer)
{
f8bppHeader_.bfSize = (DWORD)(s8bppHeader_ + width * height);
i8bppHeader_.biWidth = (LONG)width;
i8bppHeader_.biHeight = -(LONG)height;
fwrite(&f8bppHeader_, sizeof(BITMAPFILEHEADER), 1, output);
fwrite(&i8bppHeader_, sizeof(BITMAPINFOHEADER), 1, output); // I get the error in this line
fwrite(p8bpp, sizeof(RGBQUAD), 256, output);
fwrite(buffer, width, height, output);
}
Not sure you need this bit but I'll add it as well.
void BmpHelper::Init8bppHeaders()
{
f8bppHeader_.bfType = 'MB'; // Will be inverted during the fwrite
s8bppHeader_ = (DWORD)(sizeof(BITMAPFILEHEADER) + sizeof(BITMAPINFOHEADER) + 256 * sizeof(RGBQUAD) + 256 * sizeof(RGBQUAD));
f8bppHeader_.bfReserved1 = 0;
f8bppHeader_.bfReserved2 = 0;
f8bppHeader_.bfOffBits = sizeof(BITMAPFILEHEADER) + sizeof(BITMAPINFOHEADER) + 256 * sizeof(RGBQUAD);
i8bppHeader_.biSize = sizeof(BITMAPINFOHEADER);
i8bppHeader_.biPlanes = 1;
i8bppHeader_.biBitCount = 8;
i8bppHeader_.biCompression = BI_RGB;
i8bppHeader_.biSizeImage = 0;
i8bppHeader_.biXPelsPerMeter = 0;
i8bppHeader_.biYPelsPerMeter = 0;
i8bppHeader_.biClrUsed = 0;
i8bppHeader_.biClrImportant = 0;
for (size_t index = 0; index < 256; ++index)
{
p8bpp[index].rgbBlue = (BYTE)index;
p8bpp[index].rgbGreen = (BYTE)index;
p8bpp[index].rgbRed = (BYTE)index;
p8bpp[index].rgbReserved = 0;
}
}
If I remove the date from the path it works all right but it only saves the file with the url + name given and file format. I need the date to differentiate between files. Can you guys spot anything wrong with the code? Thanks in advance.
Your date/time string includes : characters. Windows file names do not accept any of of these characters \/:*?"<>|. You have to replace : with a different character.
Also your code does not account for bitmap padding. It works only if bitmap width in bytes is a multiple of 4. Use the following formula to calculate size (instead of width * height):
int size = ((width * bpp + 31) / 32) * 4 * height;
s8bppHeader_ = (DWORD)... is a typo! bfSize is not calculated properly.
You can use C++ functions std::fstream instead of mixing C++ and C
Example:
#include <iostream>
#include <fstream>
#include <string>
#include <vector>
#include <ctime>
#include <windows.h>
...
std::time_t t = std::time(nullptr);
char date[100];
std::strftime(date, sizeof(date), " %c", std::localtime(&t));
for(size_t i = 0, len = strlen(date); i < len; i++)
if(date[i] == ':')
date[i] = ','; //use a different character
char *Name = "_abc";
string url("c:\\test\\");
string name(Name);
string extension(".bmp");
string path = url + name + date + extension;
int width = i8bppHeader_.biWidth;
int height = i8bppHeader_.biHeight;
int bpp = i8bppHeader_.biBitCount;
int size = ((width * bpp + 31) / 32) * 4 * height;
f8bppHeader_.bfType = 'MB'; // Will be inverted during the fwrite
f8bppHeader_.bfSize = sizeof(BITMAPFILEHEADER) + sizeof(BITMAPINFOHEADER)
+ 256 * sizeof(RGBQUAD) + size;
f8bppHeader_.bfReserved1 = 0;
f8bppHeader_.bfReserved2 = 0;
f8bppHeader_.bfOffBits = sizeof(BITMAPFILEHEADER) + sizeof(BITMAPINFOHEADER)
+ 256 * sizeof(RGBQUAD);
ofstream fout(path, ios::binary);
fout.write((char*)&f8bppHeader_, sizeof(f8bppHeader_));
fout.write((char*)&i8bppHeader_, sizeof(i8bppHeader_));
fout.write((char*)p8bpp, 256 * sizeof(RGBQUAD));
fout.write((char*)buffer, size);
I am using libsndfile to read .caf file. I am able to read the file properly with number of items in the audio file. However, when I save those numbers in a text file and try to verify my values with MATLAB, they look a lot different. I have attached the code in C++ and the values I obtain from C++ and MATLAB.
void ofApp::setup(){
const char* fn = "/Users/faiyadhshahid/Desktop/Desktopdemo.caf";
SNDFILE *sf;
SF_INFO info;
int num_channels, num, num_items, *buf, f, sr,c, i , j;
FILE *out;
/* Open the WAV file. */
info.format = 0;
sf = sf_open(fn,SFM_READ,&info);
if (sf == NULL)
{
printf("Failed to open the file.\n");
}
/* Print some of the info, and figure out how much data to read. */
f = info.frames;
sr = info.samplerate;
c = info.channels;
printf("frames=%d\n",f);
printf("samplerate=%d\n",sr);
printf("channels=%d\n",c);
num_items = f*c;
printf("num_items=%d\n",num_items);
/* Allocate space for the data to be read, then read it. */
buf = (int *) malloc(num_items*sizeof(int));
num = sf_read_int(sf,buf,num_items);
sf_close(sf);
printf("Read %d items\n",num);
/* Write the data to filedata.out. */
out = fopen("/Users/faiyadhshahid/Desktop/filedata.txt","w");
for (i = 0; i < num; i += c)
{
for (j = 0; j < c; ++j)
fprintf(out,"%d ",buf[i+j]);
fprintf(out,"\n");
}
fclose(out);
return 0;
}
Values of C++ (on left) vs MATLAB (on right):
I figured it out by myself. I was comparing apples with oranges.
The changes I needed to make were to convert the buffer saving the values to read float values. `int num_channels, num, num_items,f, sr,c, i , j;
float *buf;
FILE *out;
/* Open the WAV file. */
info.format = 0;
sf = sf_open(fn,SFM_READ,&info);
if (sf == NULL)
{
printf("Failed to open the file.\n");
}
/* Print some of the info, and figure out how much data to read. */
f = info.frames;
sr = info.samplerate;
c = info.channels;
printf("frames=%d\n",f);
printf("samplerate=%d\n",sr);
printf("channels=%d\n",c);
num_items = f*c;
printf("num_items=%d\n",num_items);
/* Allocate space for the data to be read, then read it. */
buf = (float *) malloc(num_items*sizeof(float));
num = sf_read_float(sf,buf,num_items);
sf_close(sf);
printf("Read %d items\n",num);
/* Write the data to filedata.out. */
out = fopen("/Users/faiyadhshahid/Desktop/filedata.txt","w");
for (i = 0; i < num; i += c)
{
for (j = 0; j < c; ++j)
fprintf(out,"%f \n",buf[i]);
// fprintf(out,"\n");
}
fclose(out);
`
I need to read huge 35G file from disc line by line in C++. Currently I do it the following way:
ifstream infile("myfile.txt");
string line;
while (true) {
if (!getline(infile, line)) break;
long linepos = infile.tellg();
process(line,linepos);
}
But it gives me about 2MB/sec performance, though file manager copies the file with 100Mb/s speed. I guess that getline() is not doing buffering correctly. Please propose some sort of buffered line-by-line reading approach.
UPD: process() is not a bottleneck, code without process() works with the same speed.
You won't get anywhere close to line speed with the standard IO streams. Buffering or not, pretty much ANY parsing will kill your speed by orders of magnitude. I did experiments on datafiles composed of two ints and a double per line (Ivy Bridge chip, SSD):
IO streams in various combinations: ~10 MB/s. Pure parsing (f >> i1 >> i2 >> d) is faster than a getline into a string followed by a sstringstream parse.
C file operations like fscanf get about 40 MB/s.
getline with no parsing: 180 MB/s.
fread: 500-800 MB/s (depending on whether or not the file was cached by the OS).
I/O is not the bottleneck, parsing is. In other words, your process is likely your slow point.
So I wrote a parallel parser. It's composed of tasks (using a TBB pipeline):
fread large chunks (one such task at a time)
re-arrange chunks such that a line is not split between chunks (one such task at a time)
parse chunk (many such tasks)
I can have unlimited parsing tasks because my data is unordered anyway. If yours isn't then this might not be worth it to you.
This approach gets me about 100 MB/s on an 4-core IvyBridge chip.
I've translated my own buffering code from my java project and it does what I need. I had to put defines to overcome problems with M$VC 2010 compiler tellg, that always gives wrong negative values on huge files. This algorithm gives desired speed ~100MB/s, though it does some usless new[].
void readFileFast(ifstream &file, void(*lineHandler)(char*str, int length, __int64 absPos)){
int BUF_SIZE = 40000;
file.seekg(0,ios::end);
ifstream::pos_type p = file.tellg();
#ifdef WIN32
__int64 fileSize = *(__int64*)(((char*)&p) +8);
#else
__int64 fileSize = p;
#endif
file.seekg(0,ios::beg);
BUF_SIZE = min(BUF_SIZE, fileSize);
char* buf = new char[BUF_SIZE];
int bufLength = BUF_SIZE;
file.read(buf, bufLength);
int strEnd = -1;
int strStart;
__int64 bufPosInFile = 0;
while (bufLength > 0) {
int i = strEnd + 1;
strStart = strEnd;
strEnd = -1;
for (; i < bufLength && i + bufPosInFile < fileSize; i++) {
if (buf[i] == '\n') {
strEnd = i;
break;
}
}
if (strEnd == -1) { // scroll buffer
if (strStart == -1) {
lineHandler(buf + strStart + 1, bufLength, bufPosInFile + strStart + 1);
bufPosInFile += bufLength;
bufLength = min(bufLength, fileSize - bufPosInFile);
delete[]buf;
buf = new char[bufLength];
file.read(buf, bufLength);
} else {
int movedLength = bufLength - strStart - 1;
memmove(buf,buf+strStart+1,movedLength);
bufPosInFile += strStart + 1;
int readSize = min(bufLength - movedLength, fileSize - bufPosInFile - movedLength);
if (readSize != 0)
file.read(buf + movedLength, readSize);
if (movedLength + readSize < bufLength) {
char *tmpbuf = new char[movedLength + readSize];
memmove(tmpbuf,buf,movedLength+readSize);
delete[]buf;
buf = tmpbuf;
bufLength = movedLength + readSize;
}
strEnd = -1;
}
} else {
lineHandler(buf+ strStart + 1, strEnd - strStart, bufPosInFile + strStart + 1);
}
}
lineHandler(0, 0, 0);//eof
}
void lineHandler(char*buf, int l, __int64 pos){
if(buf==0) return;
string s = string(buf, l);
printf(s.c_str());
}
void loadFile(){
ifstream infile("file");
readFileFast(infile,lineHandler);
}
Use a line parser or write the same. here is a sample in the sourceforge http://tclap.sourceforge.net/ and put in a buffer if necessary.