Reading Multiple Files Parallel into a buffer - c++

Question

Am working in a project where i have to read a set of files and put it in a buffer.The List comprises of small as well as large files.I have to read these files and for more efficiency i tried implementing it in multiple threads.Each thread will take a file from vector of file names and start reading it put it into a buffer and these buffer have to be put in a queue.I happened to have some error in program and i i don't know where exactly in my program the error occurs also don't know why ? Please help me whether there is any mistake in my logic or in my code and how to correct it. Thanks in advance
using namespace std;
#define MAX_THREADS 2
#define BUFFER_SIZE 8388608
vector<string>files;
deque<string>bufferq;
CRITICAL_SECTION Readlock;
int count = 0;
DWORD WINAPI ReadThread(LPVOID s);
int main(int argc,char *argv[])
{
HANDLE ReadT[MAX_THREADS];
char *filelist[5];
DWORD threadid;
filelist[0] = "1.txt";
filelist[1] = "cloudy.jpg";
filelist[2] = "connectify.exe";
filelist[3] = "VMware.exe";
filelist[4] = "Sherlock.mp4";
for(int i=0;i<5;i++)
files.push_back(filelist[i]);
InitializeCriticalSection(&Readlock);
long t1 = GetTickCount();
for(int k = 0; k< MAX_THREADS; k++)
ReadT[k] = CreateThread(NULL,0,ReadThread,NULL,NULL,&threadid);
WaitForMultipleObjects(MAX_THREADS,ReadT,TRUE,INFINITE);
cout << " Time Taken "<< GetTickCount()-t1 << "ms" ;
system("pause");
return 0;
}
DWORD WINAPI ReadThread(LPVOID s)
{
long pending = 0;
//int freespace = BUFFER_SIZE;
char *filename = new char[50];
char fsize[10];
string file;
char *buf;
buf = new char[BUFFER_SIZE];
long filesize = 0;
int numfiles = files.size();
int filled = 0;
int i = 0;
FILE *fp;
char* ptr;
ptr = buf;
while(true)
{
EnterCriticalSection(&Readlock);
if(files.empty())
{
LeaveCriticalSection(&Readlock);
break;
}
else
{
file = files.front();
files.erase(files.begin());
LeaveCriticalSection(&Readlock);
}
bool buff_full = false;
buf = ptr;
int freespace = BUFFER_SIZE;
memset(buf,0,BUFFER_SIZE);
if(!buff_full)
{
if(pending == 0)
{
fp = fopen(file.c_str(),"rb");
if(!fp)
{
cout<<"\nNo such file";
cout<<files[i];
system("pause");
return 0;
}
int r1 =fseek(fp, 0L, SEEK_END);
filesize = ftell(fp);
int r2 =fseek(fp, 0L, SEEK_SET);
sprintf(fsize, "%ld", filesize);
if(freespace >= (strlen(fsize) + strlen(file.c_str()) + 2))
{
count++;
memcpy(buf, file.c_str(), strlen(file.c_str())+1);
freespace = freespace - strlen(file.c_str()) - 1;
buf += strlen(file.c_str()) + 1;
memcpy(buf,fsize,strlen(fsize)+1);
buf += strlen(fsize) + 1;
freespace = freespace - strlen(fsize) - 1;
cout<<"Files read is "<<count<<"\n";
if(freespace == 0)
{
buff_full = true;
pending = filesize;
break;
}
}
else
{
filled = BUFFER_SIZE - freespace;
fclose(fp);
break;
}
if(freespace >= filesize)
{
fread(buf, 1, filesize, fp);
buf += filesize;
freespace = freespace - filesize;
bufferq.push_back(buf);
//cout << "pop"<<bufferq.size();
//i++;
if(files.empty())
{
filled = BUFFER_SIZE - freespace;
fclose(fp);
break;
}
fclose(fp);
}
else
{
fread(buf, 1, freespace, fp);
bufferq.push_back(buf);
//cout <<"pop "<<bufferq.size();
buff_full = true;
}
}
else
{
if(freespace >= pending)
{
fread(buf, 1, pending, fp);
bufferq.push_back(buf);
freespace = freespace - pending;
pending = 0;
//i++;
if(files.empty())
{
filled = BUFFER_SIZE - freespace;
fclose(fp);
break;
}
if(freespace > 0)
buf += pending;
else
buff_full = true;
fclose(fp);
}
else
{
fread(buf, 1, freespace, fp);
bufferq.push_back(buf);
cout << bufferq.size();
pending = pending - freespace;
buff_full = true;
}
}
}
if(buff_full)
{
buf = ptr;
cout << "popping buffer " << bufferq.size();
//bufferq.pop_back();
}
}
return 0;
}

Answer 1

In the context that bug occurs on big files, I suppose that this line can cause problems
sprintf(fsize, "%ld", filesize);
fsize is char[10], and if filesize is >= 1,000,000,000 you'll overwrite fsize array with trailing 0. This will cause "Run-Time Check Failure #2 - Stack around the variable 'fsize' was corrupted.", as you wrote. Please check the sizes of your test files.
Among others, you are filling files in loop on i, and then you wrote:
files.erase(files.begin());
// ...
cout<<"\nNo such file";
cout<<files[i];
files[i] already points to another element as you erased them, and if files are empty on the last iteration it will cause crash.
And what for are you copying file and fsize to buf if you do not copy it to the bufferq?
As bufferq is writable and is shared between threads the access to it should be protected by lock, critical section as you chose.
That's my little code review.