Is this proper usage of the windows file API? (Multiple overlapped requests) - c++

Question

I have encountered some strange behavior when using the windows file API, specifically ReadFile with overlapped IO.
Under certain conditions, GetOverlappedResult will successfully read data into the provided buffer, but set lpNumberOfBytesTransferred to zero, instead of the correct amount that was read.
This only seems to happen when multiple overlapped read request are issued on the same handle, when the file was previously opened with FILE_FLAG_NO_BUFFERING.
Here is a full sample of code that illustrates the issue...
#include <Windows.h>
#include <string>
#include <iostream>
const int PageSize = 4096;
const int BufferSize = PageSize * 4;
struct OperationSlot
{
OVERLAPPED state;
unsigned char* buffer;
};
bool enableFail;
bool ReadTest(std::string filename, DWORD flags, int queueSize)
{
bool result = true;
if (enableFail)
{
HANDLE temp = CreateFile(filename.c_str(), GENERIC_READ | GENERIC_WRITE, 0, 0, OPEN_EXISTING, FILE_FLAG_NO_BUFFERING, 0);
CloseHandle(temp);
}
OperationSlot* slots = new OperationSlot[queueSize];
for (int i = 0; i < queueSize; ++i)
{
slots[i].buffer = (unsigned char*)_aligned_malloc(BufferSize, PageSize);
ZeroMemory(slots[i].buffer, BufferSize);
}
HANDLE file = CreateFile(filename.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, flags, NULL);
HANDLE controlFile = CreateFile(filename.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
unsigned char* controlBuffer = new unsigned char[BufferSize];
// Start async read operations...
for (int i = 0; i < queueSize; ++i)
{
ZeroMemory(&slots[i].state, sizeof(OVERLAPPED));
slots[i].state.Offset = i * BufferSize;
bool ok = ReadFile(file, slots[i].buffer, BufferSize, NULL, &slots[i].state);
if (!ok)
{
DWORD err = GetLastError();
if (err != ERROR_IO_PENDING)
{
std::cout << "ReadFile set error code " << err << std::endl;
}
}
}
int readId = 0;
while (true)
{
OperationSlot& active_slot = slots[readId % queueSize];
DWORD bytes = 0;
bool ok = GetOverlappedResult(file, &active_slot.state, &bytes, true);
DWORD err = GetLastError();
DWORD controlBytes = 0;
ReadFile(controlFile, controlBuffer, BufferSize, &controlBytes, NULL);
bool dataok = memcmp(active_slot.buffer, controlBuffer, controlBytes) == 0;
if (!dataok)
std::cout << "Data mismatch." << std::endl;
if (bytes != controlBytes)
{
std::cout << "Error with QueueSize (" << queueSize << ") and flags: ";
if (flags & FILE_FLAG_OVERLAPPED)
{
std::cout << "FILE_FLAG_OVERLAPPED";
}
if (flags & FILE_FLAG_NO_BUFFERING)
{
std::cout << " | FILE_FLAG_NO_BUFFERING";
}
if (flags & FILE_FLAG_SEQUENTIAL_SCAN)
{
std::cout << " | FILE_FLAG_SEQUENTIAL_SCAN";
}
std::cout << std::endl;
std::cout << "Read size error, expected " << controlBytes << ", got " << bytes << std::endl;
std::cout << "GetOverlappedResult returned " << ok << " with error code " << err << std::endl;
result = false;
}
if (controlBytes < BufferSize)
break;
ZeroMemory(&active_slot.state, sizeof(OVERLAPPED));
active_slot.state.Offset = (readId + queueSize) * BufferSize;
ReadFile(file, active_slot.buffer, BufferSize, NULL, &active_slot.state);
++readId;
}
CloseHandle(file);
CloseHandle(controlFile);
delete[] controlBuffer;
for (int i = 0; i < queueSize; ++i)
{
_aligned_free(slots[i].buffer);
}
delete[] slots;
return !result;
}
int main()
{
enableFail = false;
int totalfail = 0;
std::cout << "Testing without fail." << std::endl;
totalfail += ReadTest("C:\\Test\\SmallFile.txt.txt", FILE_FLAG_OVERLAPPED, 1);
totalfail += ReadTest("C:\\Test\\SmallFile.txt.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_SEQUENTIAL_SCAN, 1);
totalfail += ReadTest("C:\\Test\\SmallFile.txt.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_NO_BUFFERING, 1);
totalfail += ReadTest("C:\\Test\\SmallFile.txt.txt", FILE_FLAG_OVERLAPPED, 4);
totalfail += ReadTest("C:\\Test\\SmallFile.txt.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_SEQUENTIAL_SCAN, 4);
totalfail += ReadTest("C:\\Test\\SmallFile.txt.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_NO_BUFFERING, 4);
totalfail += ReadTest("C:\\Test\\LargeFile.txt", FILE_FLAG_OVERLAPPED, 1);
totalfail += ReadTest("C:\\Test\\LargeFile.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_SEQUENTIAL_SCAN, 1);
totalfail += ReadTest("C:\\Test\\LargeFile.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_NO_BUFFERING, 1);
totalfail += ReadTest("C:\\Test\\LargeFile.txt", FILE_FLAG_OVERLAPPED, 4);
totalfail += ReadTest("C:\\Test\\LargeFile.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_SEQUENTIAL_SCAN, 4);
totalfail += ReadTest("C:\\Test\\LargeFile.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_NO_BUFFERING, 4);
std::cout << totalfail << " calls failed." << std::endl;
enableFail = true;
totalfail = 0;
std::cout << "Testing with fail enabled." << std::endl;
totalfail += ReadTest("C:\\Test\\SmallFile.txt.txt", FILE_FLAG_OVERLAPPED, 1);
totalfail += ReadTest("C:\\Test\\SmallFile.txt.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_SEQUENTIAL_SCAN, 1);
totalfail += ReadTest("C:\\Test\\SmallFile.txt.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_NO_BUFFERING, 1);
totalfail += ReadTest("C:\\Test\\SmallFile.txt.txt", FILE_FLAG_OVERLAPPED, 4);
totalfail += ReadTest("C:\\Test\\SmallFile.txt.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_SEQUENTIAL_SCAN, 4);
totalfail += ReadTest("C:\\Test\\SmallFile.txt.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_NO_BUFFERING, 4);
totalfail += ReadTest("C:\\Test\\LargeFile.txt", FILE_FLAG_OVERLAPPED, 1);
totalfail += ReadTest("C:\\Test\\LargeFile.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_SEQUENTIAL_SCAN, 1);
totalfail += ReadTest("C:\\Test\\LargeFile.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_NO_BUFFERING, 1);
totalfail += ReadTest("C:\\Test\\LargeFile.txt", FILE_FLAG_OVERLAPPED, 4);
totalfail += ReadTest("C:\\Test\\LargeFile.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_SEQUENTIAL_SCAN, 4);
totalfail += ReadTest("C:\\Test\\LargeFile.txt", FILE_FLAG_OVERLAPPED | FILE_FLAG_NO_BUFFERING, 4);
std::cout << totalfail << " calls failed." << std::endl;
system("pause");
return 0;
}
On my system, this results in 4 of the calls 'failing'. (The ones with a 'queue size' of more than one.)
What happens is that the overlapped version reports that it only reads 0 bytes, while the 'normal' file handle reads 20. (A small text file that says "this is a test".)
The other strange thing is that it is actually reading the data properly. The correct buffer is populated with the correct data, only the reported amount of data transferred is wrong...
And this only happens if the file opened and closed with FILE_FLAG_NO_BUFFERING right before the file is opened.
Why would previously touching a file cause subsequent access to behave differently like this?
Am I doing something that's not supported, or is the API not functioning the way it is supposed to? (Or possibly there is a mistake in my code I have overlooked...?)
EDIT: It seems I was lulled into thinking I was using the API correctly by the fact that it was miraculously working under most conditions. The proper way to do this is to specify unique events for each overlapped structure, as pointed out in the accepted answer. After giving each overlapped request it's own event, it works consistently.

Answer 1

You're not giving your OVERLAPPED structures unique events, so all GetOverlappedResult() has to wait on is the file handle - and with multiple requests outstanding there's no guarantee the request you've asked for will actually be the one that completed when the file handle gets signalled.
Each OVERLAPPED structure should have its hEvent member initialized to a new event handle created with CreateEvent().