How to use ReadFileScatter - c++

I've been attempting to use ReadFileScatter today in my code (which sounds like exactly what I need), so far without a lot of luck. Google'ing the internet for what goes wrong doesn't give me much insight.
The documentation states:
The array must contain enough elements to store nNumberOfBytesToRead bytes of data, plus one element for the terminating NULL. For example, if there are 40 KB to be read and the page size is 4 KB, the array must have 11 elements that includes 10 for the data and one for the NULL.
Each buffer must be at least the size of a system memory page and must be aligned on a system memory page size boundary. The system reads one system memory page of data into each buffer.
The function stores the data in the buffers in sequential order. For example, it stores data into the first buffer, then into the second buffer, and so on until each buffer is filled and all the data is stored, or there are no more buffers.
So far I've been attempting to do just that. I allocated a bunch of bytes using VirtualAlloc (which ensures the page boundary constraint), add a terminator NULL to the list, ensure the data on disk is on the system boundary (and implicitly a disk sector size boundary) as well and issue the call.
Without further due, here's the minimum test case in C++:
// Setup: c:\tmp\test.dat is a file with at least 12K of stuff.
// I attempt to read page 2/3, e.g. offset [4096-4096+8192>
// TEST:
SYSTEM_INFO systemInfo;
GetSystemInfo(&systemInfo);
auto pageSize = systemInfo.dwPageSize;
std::cout << "Page size: "<< pageSize << std::endl;
// Allocate 2 pages that are aligned with one in the middle:
auto buffer = reinterpret_cast<char*>(VirtualAlloc(NULL, pageSize * 3, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
// Create read buffer:
std::vector<FILE_SEGMENT_ELEMENT> elements;
{
FILE_SEGMENT_ELEMENT element1;
element1.Buffer = buffer;
elements.push_back(element1);
}
{
FILE_SEGMENT_ELEMENT element2;
element2.Buffer = buffer + pageSize * 2;
elements.push_back(element2);
}
{
FILE_SEGMENT_ELEMENT terminator;
terminator.Buffer = NULL;
elements.push_back(terminator);
}
// [..] Physical sector size is normally checked as well. In my case it's 512 bytes,
// so I guess that's irrelevant here.
//
// Open file:
auto fileHandle = CreateFile(
"c:\\tmp\\test.dat",
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ,
NULL,
OPEN_ALWAYS,
FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH,
NULL);
auto err = GetLastError();
if (err != ERROR_ALREADY_EXISTS && err != ERROR_SUCCESS)
{
throw std::exception(); // FIXME.
}
OVERLAPPED overlapped;
memset(&overlapped, 0, sizeof(OVERLAPPED));
LARGE_INTEGER tmp;
tmp.QuadPart = 4096; // Read from disk page 1
overlapped.Offset = tmp.LowPart;
overlapped.OffsetHigh = tmp.HighPart;
overlapped.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
auto succes = ReadFileScatter(fileHandle, elements.data(), pageSize * 2, NULL, &overlapped);
err = GetLastError();
if (!succes && err != ERROR_IO_PENDING && err != ERROR_SUCCESS)
{
throw std::exception(); // The call always ends up here with error 87: Invalid parameter
}
WaitForSingleObject(overlapped.hEvent, INFINITE);
std::cout << "Call succeeded!" << std::endl;
// FIXME: Proper exception handling.
// Clean up:
VirtualFree(buffer, pageSize * 3, MEM_DECOMMIT | MEM_RELEASE);
CloseHandle(overlapped.hEvent);
CloseHandle(fileHandle);
In the code, the error is noted with the comment // The call always ends up here with error 87: Invalid parameter. However, as far as I can see, I check all the boxes that they describe on MSDN... so...
What am I doing wrong here?

Related

How to initialize disk on Windows server 2008/2012 through a C++ program

We are trying to initialize disk with the properties of some existing disk on Windows server 2008/2012 through a C++ program.
We are using DeviceIoControl() method and IOCTL_DISK_CREATE_DISK, IOCTL_DISK_SET_DRIVE_LAYOUT_EX, IOCTL_DISK_SET_PARTITION_INFO_EX codes from Disk management control codes to make the disk available for use.
Got the following code snippet by searching a bit
//To open the drive
hDevice = CreateFile( TEXT("\\\\.\\PhysicalDrive7"),
GENERIC_READ | GENERIC_WRITE, // no access to the drive
FILE_SHARE_READ | FILE_SHARE_WRITE, // share mode
NULL, // default security attributes
OPEN_EXISTING, // disposition
0, // file attributes
NULL); // do not copy file attributes
CREATE_DISK dsk;
dsk.PartitionStyle = PARTITION_STYLE_MBR; //It can also be PARTITION_STYLE_GPT
dsk.Mbr.Signature = 1;
// Initialize disk
bResult = DeviceIoControl( hDevice, // device to be queried
IOCTL_DISK_CREATE_DISK, // operation to perform
&dsk, sizeof(dsk),
NULL, 0, // no output buffer
&junk, // # bytes returned
NULL
);
LARGE_INTEGER lgPartitionSize;
lgPartitionSize.QuadPart = (1024 * 1024 * 1024);
DWORD dwDriverLayoutInfoExLen = sizeof (DRIVE_LAYOUT_INFORMATION_EX) + 3 * sizeof(PARTITION_INFORMATION_EX);
DRIVE_LAYOUT_INFORMATION_EX *pdg = (DRIVE_LAYOUT_INFORMATION_EX *)new BYTE[dwDriverLayoutInfoExLen];
SecureZeroMemory(pdg, dwDriverLayoutInfoExLen);
pdg->PartitionStyle = PARTITION_STYLE_MBR;
pdg->PartitionCount = 1;
pdg->Mbr.Signature = 1;
pdg->PartitionEntry[0].PartitionStyle = PARTITION_STYLE_MBR;
pdg->PartitionEntry[0].StartingOffset.QuadPart = 1048576;
pdg->PartitionEntry[0].PartitionLength.QuadPart = lgPartitionSize.QuadPart * 200;
pdg->PartitionEntry[0].PartitionNumber = 1;
pdg->PartitionEntry[0].RewritePartition = TRUE;
pdg->PartitionEntry[0].Mbr.PartitionType = PARTITION_NTFT; // PARTITION_IFS (NTFS partition or logical drive)
pdg->PartitionEntry[0].Mbr.BootIndicator = TRUE;
pdg->PartitionEntry[0].Mbr.RecognizedPartition = 1;
pdg->PartitionEntry[0].Mbr.HiddenSectors = 32256 / 512;
// Partition a disk
bResult = DeviceIoControl( hDevice, // device to be queried
IOCTL_DISK_SET_DRIVE_LAYOUT_EX, // operation to perform
pdg, sizeof DRIVE_LAYOUT_INFORMATION_EX, //output buffer
NULL, 0, // no output buffer
&junk, // # bytes returned
NULL
);
bResult = DeviceIoControl( hDevice,
IOCTL_DISK_UPDATE_PROPERTIES,
NULL, 0, NULL, 0, &junk, NULL);
PARTITION_INFORMATION_EX dskinfo;
PARTITION_INFORMATION_MBR mbrinfo;
mbrinfo.PartitionType = PARTITION_NTFT;
mbrinfo.HiddenSectors = (32256 / 512);
mbrinfo.BootIndicator = 1;
mbrinfo.RecognizedPartition = 1;
dskinfo.PartitionStyle = PARTITION_STYLE_MBR;
dskinfo.StartingOffset.QuadPart = 1048576;//0;
dskinfo.PartitionLength.QuadPart = lgPartitionSize.QuadPart * 200;
dskinfo.PartitionNumber = 1;
dskinfo.RewritePartition = TRUE;
dskinfo.Mbr = mbrinfo;
bResult = DeviceIoControl( hDevice, // device to be queried
IOCTL_DISK_SET_PARTITION_INFO_EX, // operation to perform
&dskinfo, sizeof(dskinfo), // output buffer
NULL, 0, // no output buffer
&junk, // # bytes returned
NULL
);
All the calls to DeviceIoControl() are getting succeeded except the last one with IOCTL_DISK_SET_PARTITION_INFO_EX code with error 1 (i.e Incorrect function). What could be the reason for this?
If we comment out the last call, the disk is being initialized as raw disk, But this won't meet our requirements.
The above sample is for MBR partition style only. We could not find any sample for GPT,... styles. Please give a link if someone is aware of one.
You're using the wrong structure type with IOCTL_DISK_SET_PARTITION_INFO_EX. It takes a SET_PARTITION_INFORMATION_EX structure, not a PARTITION_INFORMATION_EX structure.
You probably don't need to use IOCTL_DISK_SET_PARTITION_INFO_EX, since it just sets the partition type, which should have already been set with IOCTL_DISK_SET_DRIVE_LAYOUT_EX. Unfortunately you've used it to set the wrong partition type. NTFS partitions have the partition type PARTITION_IFS.
Multiplying lgPartitionSize by 200 is almost certainly wrong. If lgPartitionSize is supposed the size in sectors then you need to multiply this by the sector size of the disk. The sector size of hard drives used to be always 512 bytes (0x200 bytes), but modern drives use a 4096 byte sector size.
Correctly creating partition tables is not easy, and mindlessly copying other people's code like you've done isn't going to work. Even after you fix the problems I've mentioned above you're still likely to encounter other issues. You really need to understand the all the restrictions placed on how partitions should be laid out.
You might want to consider using the diskpart command to programically initialize disks instead of C++ code.

CreateFileMapping returns ERROR_INVALID_HANDLE

I am trying to use CreateFileMapping for the first time and it is giving me this error when I use GetLastError():
ERROR_INVALID_HANDLE: The handle is invalid.
Here is my code:
// create the name of our file-mapping object
nTry++; // Ensures a unique string is used in case user closes and reopens
wsprintfA(szName, FS6IPC_MSGNAME1 ":%X:%X", GetCurrentProcessId(), nTry);
// stuff the name into a global atom
m_atom = GlobalAddAtomA(szName);
if (m_atom == 0)
{ *pdwResult = ERR_ATOM;
return FALSE;
}
// create the file-mapping object
m_hMap = CreateFileMappingA(
(HANDLE)0xFFFFFFFF, // use system paging file
NULL, // security
PAGE_READWRITE, // protection
0, MAX_SIZE+256, // size
szName); //
EDIT:
The first issue was resolved, but now my program crashes somewhere else.
#define FS6IPC_MESSAGE_SUCCESS 1
#define FS6IPC_MESSAGE_FAILURE 0
// IPC message types
#define FS6IPC_READSTATEDATA_ID 1
#define FS6IPC_WRITESTATEDATA_ID 2
// read request structure
typedef struct tagFS6IPC_READSTATEDATA_HDR
{
DWORD dwId; // FS6IPC_READSTATEDATA_ID
DWORD dwOffset; // state table offset
DWORD nBytes; // number of bytes of state data to read
void* pDest; // destination buffer for data (client use only)
} FS6IPC_READSTATEDATA_HDR;
// write request structure
typedef struct tagFS6IPC_WRITESTATEDATA_HDR
{
DWORD dwId; // FS6IPC_WRITESTATEDATA_ID
DWORD dwOffset; // state table offset
DWORD nBytes; // number of bytes of state data to write
} FS6IPC_WRITESTATEDATA_HDR;
while (*pdw)
{ switch (*pdw)
{ case FS6IPC_READSTATEDATA_ID:
pHdrR = (FS6IPC_READSTATEDATA_HDR *) pdw;
m_pNext += sizeof(FS6IPC_READSTATEDATA_HDR);
if (pHdrR->pDest && pHdrR->nBytes)
CopyMemory(pHdrR->pDest, m_pNext, pHdrR->nBytes);
m_pNext += pHdrR->nBytes; // Debugger says the issue is here
break;
case FS6IPC_WRITESTATEDATA_ID:
// This is a write, so there's no returned data to store
pHdrW = (FS6IPC_WRITESTATEDATA_HDR *) pdw;
m_pNext += sizeof(FS6IPC_WRITESTATEDATA_HDR) + pHdrW->nBytes;
break;
default:
// Error! So terminate the scan
*pdw = 0;
break;
}
pdw = (DWORD *) m_pNext;
}
I'm guessing you're running on a 64-bit system, on which HANDLEs are 64 bits. The OS is quite right—the handle value 0x00000000FFFFFFFF is an invalid handle value for your process.
What exactly are you trying to do? If you want to create a file mapping backed by an actual file, pass in the handle for that file. If you want to a create a file mapping backed by the paging file instead, pass in INVALID_HANDLE_VALUE. INVALID_HANDLE_VALUE happens to be (HANDLE)-1, which is 0xFFFFFFFF on 32-bit systems but 0xFFFFFFFFFFFFFFFF on 64-bit systems, but that doesn't really matter since you should just use the symbolic value INVALID_HANDLE_VALUE in any case.
If your application is crashing when you pass in INVALID_HANDLE_VALUE, it's not because the call to CreateFileMapping is failing, it's for some other reason, and you should debug that.

When calling EnumServicesStatusEx() twice, i still get EROR_MORE_DATA in C++

I call EnumServicesStatusEx() twice in my code, the first time should fail and put the correct buffer size in dwBuffNeeded so that when i call it the second time the buffer size should be correct. But, sometimes, not always i still get ERROR_MORE_DATA after the second call. Any ideas why? Thanks
DWORD pId=GetCurrentProcessId();
SC_HANDLE hSCM = NULL;
PUCHAR pBuf = NULL;
ULONG dwBufSize = 0x00;
ULONG dwBufNeed = 0x00;
ULONG dwNumberOfService = 0x00;
LPENUM_SERVICE_STATUS_PROCESS pInfo = NULL;
hSCM = OpenSCManager( NULL, NULL, SC_MANAGER_ENUMERATE_SERVICE | SC_MANAGER_CONNECT );
if (hSCM == NULL)
{
GetCustomLog().Log( SV_ERROR, 10004807, "Could not open Service Control Manager: %s", GetLastOSErrorString().c_str() );
return;
}
//Query services once to get correct buffer size, always fails
if ( EnumServicesStatusEx(
hSCM,
SC_ENUM_PROCESS_INFO,
SERVICE_WIN32,
SERVICE_ACTIVE,
NULL,
dwBufSize,
&dwBufNeed,
&dwNumberOfService,
NULL,
NULL) == 0 )
{
DWORD err = GetLastError();
if ( ERROR_MORE_DATA == err )
{
dwBufSize = dwBufNeed + 0x10;
pBuf = (PUCHAR) malloc(dwBufSize);
//Query services again with correct buffer size
if ( EnumServicesStatusEx(
hSCM,
SC_ENUM_PROCESS_INFO,
SERVICE_WIN32,
SERVICE_ACTIVE,
pBuf,
dwBufSize,
&dwBufNeed,
&dwNumberOfService,
NULL,
NULL ) == 0 )
{
//FAILS HERE
GetCustomLog().Log( SV_ERROR, 10004808, "Could not enumerate services, error: %s", GetLastOSErrorString().c_str() );
free(pBuf);
return;
}
}
else
{
GetCustomLog().Log( SV_ERROR, 10004809, "Could not enumerate services, error: %s", GetLastOSErrorString().c_str() );
return;
}
I too had the same problem, but when querying SERVICE_STATE_ALL, which I thought would require the same amount of memory on each call (unless a service had been installed/uninstalled). It wasn't enough to simply retry with a buffer of the size returned in the pcbBytesNeeded argument:
BOOL WINAPI EnumServicesStatusEx(
_In_ SC_HANDLE hSCManager,
_In_ SC_ENUM_TYPE InfoLevel,
_In_ DWORD dwServiceType,
_In_ DWORD dwServiceState,
_Out_opt_ LPBYTE lpServices,
_In_ DWORD cbBufSize,
_Out_ LPDWORD pcbBytesNeeded,
_Out_ LPDWORD lpServicesReturned,
_Inout_opt_ LPDWORD lpResumeHandle,
_In_opt_ LPCTSTR pszGroupName
);
Unlike other WIN32 API calls, this is not returning the absolute number of bytes needed, but the additional bytes needed relative to the cbBufSize parameter. As an experiment, I provided a deliberately small buffer and just doubled it each time to find out what the system would return in pcbBytesNeeded in response. On this system sizeof(ENUM_SERVICE_STATUS_PROCESS) is 56 bytes. The last row is the smallest buffer resulting in a successful call.
+-----------+----------------+
| cbBufSize | pcbBytesNeeded |
+-----------+----------------+
| 112 | 37158 |
| 224 | 37013 |
| 448 | 36766 |
| 896 | 36374 |
| 1792 | 35280 |
| 3584 | 33202 |
| 7168 | 28972 |
| 14336 | 20765 |
| 28672 | 4215 |
| 32032 | 0 |
+-----------+----------------+
You can see each row roughly adds up to the required buffer size, and also that the required buffer size as far as the system is concerned is not very predictable. The number of service entries returned in the successful call in this case was 233, which only requires 13048 bytes. It turns out the strings pointed to by the ENUM_SERVICE_STATUS_PROCESS lpDisplayName and lpServiceName pointers are just stored at the tail-end of the provided buffer (I always wondered where the backing for those was, and why it was stable and didn't need to be separately freed). Anyway this explains the somewhat non-deterministic responses as well as the odd numbers: the system likely has the current entry which doesn't fit and knows exactly how much size it needs for that, but guesses about the remainder.
The code below is reliable and usually takes exactly two calls but sometimes can take three (even with SERVICE_STATE_ALL). I don't know why three are needed and the underestimation by the system sticks around for multiple minutes at a time but eventually resolves itself. I've never seen it need four calls.
int EnumerateAllServices(SC_HANDLE hSCM) {
void* buf = NULL;
DWORD bufSize = 0;
DWORD moreBytesNeeded, serviceCount;
for (;;) {
printf("Calling EnumServiceStatusEx with bufferSize %d\n", bufSize);
if (EnumServicesStatusEx(
hSCM,
SC_ENUM_PROCESS_INFO,
SERVICE_WIN32,
SERVICE_STATE_ALL,
(LPBYTE)buf,
bufSize,
&moreBytesNeeded,
&serviceCount,
NULL,
NULL)) {
ENUM_SERVICE_STATUS_PROCESS* services = (ENUM_SERVICE_STATUS_PROCESS*)buf;
for (DWORD i = 0; i < serviceCount; ++i) {
printf("%s\n", services[i].lpServiceName);
}
free(buf);
return 0;
}
int err = GetLastError();
if (ERROR_MORE_DATA != err) {
free(buf);
return err;
}
bufSize += moreBytesNeeded;
free(buf);
buf = malloc(bufSize);
}
}
The "+=" is the 'trick', and is not documented very clearly (but, in hindsight, I now understand the nuance in the MSDN explanation of the pcbBytesNeeded parameter):
pcbBytesNeeded [out]
A pointer to a variable that receives the number of bytes
needed to return the remaining service entries, if the buffer
is too small.
Because service information isn't static and theoretically anything can happen between two calls to EnumServicesStatusEx(), this API is best used iteratively.
As Kat Marsen suggests, you may of course repeatedly try to fetch all services (looping and growing a buffer as long as the API returns ERROR_MORE_DATA).
But this has 2 drawbacks:
You tell the SCM to constantly fill in information it actually has sent you back already (even it's rather rare, I actually want to avoid it).
According to the MSDN documentation, there's a hard upper limit of 256K bytes for the buffer. As I mentioned in a comment above, the latter is practically very unlikely; still you want to play safe as a programmer.
The iterative approach
In contrary we can IMO only presume that the author responsible for EnumServicesStatusEx() had a reason to design it this way:
A buffer which is filled immediately if valid and big enough for at least one service.
An output parameter for the remaining bytes the SCM assumes for the services yet to be iterated.
Most importantly there's an output parameter telling you the resume point.
Both, the resume point and the remainder allow for a much easier iterative approach.
Let me showcase with complete real-world functions, working reasonably well, and featuring some C++ goodies.
chunk_fetch_all_win32_services pre-allocates some amount of memory, invokes a callback for every chunk, and lets the callback decide whether it consumes the memory or leaves it in order to be reused.
enum_all_win32_services consumes each chunk and in turn invokes a callback for each single service.
#include <type_traits>
#include <memory>
#include <stddef.h>
#include <Windows.h>
using std::unique_ptr;
using std::error_code;
using std::system_error;
using std::system_category;
using std::function;
using std::make_unique;
/** #short Fetch all win32 services in chunks.
This function fetches all services of type `SERVICE_WIN32=(SERVICE_WIN32_OWN_PROCESS | SERVICE_WIN32_SHARE_PROCESS)`,
in chunks as decided by the SCM (refer to EnumServicesStatusEx()).
#param scmHandle Handle to the SCM with access right SC_MANAGER_ENUMERATE_SERVICE
#param stateMask One of SERVICE_ACTIVE, SERVICE_INACTIVE, SERVICE_STATE_ALL
In the callback you can decide whether you want to consume the passed memory or leave
it in order to be reused.
#note This is most probably rare but expect the callback being invoked multiple times
in case the SCM didn't return information about all services at once.
*/
bool chunk_fetch_all_win32_services(SC_HANDLE scmHandle, DWORD stateMask, const function<bool(unique_ptr<ENUM_SERVICE_STATUS_PROCESS[]>&, DWORD /*nServices*/)>& cb, error_code* ec)
{
// (optionally) preallocate
// (the amount stems from Win XP's upper size of 64k;
// on a typical Win7 system there are about 220 services (about 34k))
unique_ptr<BYTE[]> mem = make_unique<BYTE[]>(64 * 1024);
DWORD nAllocated = 64 * 1024, nRemaining;
DWORD resumePoint = 0;
do
{
DWORD nServices;
if (!EnumServicesStatusEx(scmHandle, SC_ENUM_PROCESS_INFO, SERVICE_WIN32, stateMask, mem.get(), nAllocated, &nRemaining, &nServices, &resumePoint, nullptr))
{
const int errorCode = GetLastError();
if (errorCode != ERROR_MORE_DATA)
{
if (!ec)
throw system_error{ errorCode, system_category(), "Can't enumerate services" };
ec->assign(errorCode, system_category());
return false;
}
}
if (nServices)
{
// memory initialized, transfer ownership to typed pointer
unique_ptr<ENUM_SERVICE_STATUS_PROCESS[]> cache{ LPENUM_SERVICE_STATUS_PROCESS(mem.release()) };
if (!cb(cache, nServices))
// early bail-out requested
return true;
// document that the typed pointer can be 'downgraded' again without the need to destroy objects
static_assert(std::is_trivially_destructible_v<ENUM_SERVICE_STATUS_PROCESS>);
// possibly reuse existing buffer
mem.reset(PBYTE(cache.release()));
}
if (nRemaining)
{
// re-allocate if buffer too small or consumed by callback
if (!mem || nAllocated < nRemaining)
mem = make_unique<BYTE[]>(nRemaining),
nAllocated = nRemaining;
}
// loop as long as there are more services to be enumerated
} while (nRemaining);
return true;
}
/** #short Enumerate all win32 services.
This function enumerates all services of type `SERVICE_WIN32=(SERVICE_WIN32_OWN_PROCESS | SERVICE_WIN32_SHARE_PROCESS)`.
#param scmHandle Handle to the SCM with access right SC_MANAGER_ENUMERATE_SERVICE
#param stateMask One of SERVICE_ACTIVE, SERVICE_INACTIVE, SERVICE_STATE_ALL
#param cb Callback receiving process information of a single service process information and the number of services.
#note This is most probably rare but expect the number of services passed to the callback to change
in case the SCM didn't return information about all services at once; if, then this of course happens
after the provided number of information items have been iterated.
*/
bool enum_all_win32_services(SC_HANDLE scmHandle, DWORD stateMask, const function<bool(ENUM_SERVICE_STATUS_PROCESS&, DWORD /*nServices*/)>& cb, error_code* ec)
{
return chunk_fetch_all_win32_services(scmHandle, stateMask, [&cb](unique_ptr<ENUM_SERVICE_STATUS_PROCESS[]>& cache, DWORD nServices)
{
for (size_t idx = 0; idx < nServices; ++idx)
{
if (!cb(cache[idx], nServices))
// early bail-out requested
return true;
}
return true;
}, ec);
}

C++ using 7zip.dll

I'm developing an app which will need to work with different types of archives. As many of the archive types as possible is good. I have choosen a 7zip.dll as an engine of archive-worker. But there is a problem, does anyone knows how to uncompress a file from archive to memory buffer? As I see, 7zip.dll supports only uncompressing to hard disk. Also, it would be nice to load archive from memory buffer. Has anyone tried to do something like that?
Not sure if I completely understand your needs (for example, don't you need the decompressed file on disk?).
I was looking at LZMA SDK 9.20 and its lzma.txt readme file, and there are plenty of hints that decompression to memory is possible - you may just need to use the C API rather than the C++ interface. Check out, for example, the section called Single-call Decompressing:
When to use: RAM->RAM decompressing
Compile files: LzmaDec.h + LzmaDec.c + Types.h
Compile defines: no defines
Memory Requirements:
- Input buffer: compressed size
- Output buffer: uncompressed size
- LZMA Internal Structures: state_size (16 KB for default settings)
Also, there is this function:
SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen,
const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status);
You can utilize these by memory-mapping the archive file. To the best of my knowledge, if your process creates a memory-mapped file with exclusive access (so no other process can access it) and does no explicit flushing, all changes to the file will be kept in memory until the mapping is destroyed or the file closed. Alternatively, you could just load the archive contents in memory.
For the sake of completeness, I hacked together several examples into a demo of using memory mapping in Windows.
#include <stdio.h>
#include <time.h>
#include <Windows.h>
#include <WinNT.h>
// This demo will limit the file to 4KiB
#define FILE_SIZE_MAX_LOWER_DW 4096
#define FILE_SIZE_MAX_UPPER_DW 0
#define MAP_OFFSET_LOWER_DW 0
#define MAP_OFFSET_UPPER_DW 0
#define TEST_ITERATIONS 1000
#define INT16_SIZE 2
typedef short int int16;
// NOTE: This will not work for Windows less than XP or 2003 Server!
int main()
{
HANDLE hFile, hFileMapping;
PBYTE mapViewStartAddress;
// Note: with no explicit security attributes, the process needs to have
// the necessary rights (e.g. read, write) to this location.
LPCSTR path = "C:\\Users\\mcmlxxxvi\\Desktop\\test.dat";
// First, open a file handle.
hFile = CreateFile(path,
GENERIC_READ | GENERIC_WRITE, // The file is created with Read/Write permissions
FILE_SHARE_READ, // Set this to 0 for exclusive access
NULL, // Optional security attributes
CREATE_ALWAYS, // File is created if not found, overwritten otherwise
FILE_ATTRIBUTE_TEMPORARY, // This affects the caching behaviour
0); // Attributes template, can be left NULL
if ((hFile) == INVALID_HANDLE_VALUE)
{
fprintf(stderr, "Unable to open file");
return 1;
}
// Then, create a memory mapping for the opened file.
hFileMapping = CreateFileMapping(hFile, // Handle for an opened file
NULL, // Optional security attributes
PAGE_READWRITE, // File can be mapped for Read/Write access
FILE_SIZE_MAX_UPPER_DW, // Maximum file size split in DWORDs.
FILE_SIZE_MAX_LOWER_DW, // NOTE: I may have these two mixed up!
NULL); // Optional name
if (hFileMapping == 0)
{
CloseHandle(hFile);
fprintf(stderr, "Unable to open file for mapping.");
return 1;
}
// Next, map a view (a continuous portion of the file) to a memory region
// The view must start and end at an offset that is a multiple of
// the allocation granularity (roughly speaking, the machine page size).
mapViewStartAddress = (PBYTE)MapViewOfFile(hFileMapping, // Handle to a memory-mapped file
FILE_MAP_READ | FILE_MAP_WRITE, // Maps the view for Read/Write access
MAP_OFFSET_UPPER_DW, // Offset in the file from which
MAP_OFFSET_LOWER_DW, // the view starts, split in DWORDs.
FILE_SIZE_MAX_LOWER_DW); // Size of the view (here, entire file)
if (mapViewStartAddress == 0)
{
CloseHandle(hFileMapping);
CloseHandle(hFile);
fprintf(stderr, "Couldn't map a view of the file.");
return 1;
}
// This is where actual business stuff belongs.
// This example application does iterations of reading and writing
// random numbers for the entire length of the file.
int16 value;
errno_t result = 0;
srand((int)time(NULL));
for (int i = 0; i < TEST_ITERATIONS; i++)
{
// Write
for (int j = 0; j < FILE_SIZE_MAX_LOWER_DW / INT16_SIZE; j++)
{
value = rand();
result = memcpy_s(mapViewStartAddress + j * INT16_SIZE, INT16_SIZE, &value, INT16_SIZE);
if (result != 0)
{
CloseHandle(hFileMapping);
CloseHandle(hFile);
fprintf(stderr, "File write error during iteration #%d, error %d", i, GetLastError());
return 1;
}
}
// Read
SetFilePointer(hFileMapping, 0, 0, FILE_BEGIN);
for (int j = 0; j < FILE_SIZE_MAX_LOWER_DW / sizeof(int); j++)
{
result = memcpy_s(&value, INT16_SIZE, mapViewStartAddress + j * INT16_SIZE, INT16_SIZE);
if (result != 0)
{
CloseHandle(hFileMapping);
CloseHandle(hFile);
fprintf(stderr, "File read error during iteration #%d, error %d", i, GetLastError());
return 1;
}
}
}
// End business stuff
CloseHandle(hFileMapping);
CloseHandle(hFile);
return 0;
}

Mapping non-contiguous blocks from a file into contiguous memory addresses

I am interested in the prospect of using memory mapped IO, preferably
exploiting the facilities in boost::interprocess for cross-platform
support, to map non-contiguous system-page-size blocks in a file into
a contiguous address space in memory.
A simplified concrete scenario:
I've a number of 'plain-old-data' structures, each of a fixed length
(less than the system page size.) These structures are concatenated
into a (very long) stream with the type & location of structures
determined by the values of those structures that proceed them in the
stream. I'm aiming to minimize latency and maximize throughput in a
demanding concurrent environment.
I can read this data very effectively by memory-mapping it in blocks
of at least twice the system-page-size... and establishing a new
mapping immediately having read a structure extending beyond the
penultimate system-page-boundary. This allows the code that interacts
with the plain-old-data structures to be blissfully unaware that these
structures are memory mapped... and, for example, could compare two
different structures using memcmp() directly without having to care
about page boundaries.
Where things get interesting is with respect to updating these data
streams... while they're being (concurrently) read. The strategy I'd
like to use is inspired by 'Copy On Write' on a system-page-size
granularity... essentially writing 'overlay-pages' - allowing one
process to read the old data while another reads the updated data.
While managing which overlay pages to use, and when, isn't necessarily
trivial... that's not my main concern. My main concern is that I may
have a structure spanning pages 4 and 5, then update a
structure wholly contained in page 5... writing the new page in
location 6... leaving page 5 to be 'garbage collected' when it is
determined to be no-longer reachable. This means that, if I map page
4 into location M, I need to map page 6 into memory location
M+page_size... in order to be able to reliably process structures that
cross page boundaries using existing (non-memory-mapping-aware) functions.
I'm trying to establish the best strategy, and I'm hampered by
documentation I feel is incomplete. Essentially, I need to decouple
allocation of address space from memory mapping into that address
space. With mmap(), I'm aware that I can use MAP_FIXED - if I wish to
explicitly control the mapping location... but I'm unclear how I
should reserve address space in order to do this safely. Can I map
/dev/zero for two pages without MAP_FIXED, then use MAP_FIXED twice to
map two pages into that allocated space at explicit VM addresses? If
so, should I call munmap() three times too? Will it leak resources
and/or have any other untoward overhead? To make the issue even more
complex, I'd like comparable behaviour on Windows... is there any way
to do this? Are there neat solutions if I were to compromise my
cross-platform ambitions?
--
Thanks for your answer, Mahmoud... I've read, and think I've understood that code... I've compiled it under Linux and it behaves as you suggest.
My main concerns are with line 62 - using MAP_FIXED. It makes some assumptions about mmap, which I've been unable to confirm when I read the documentation I can find. You're mapping the 'update' page into the same address space as mmap() returned initially - I assume that this is 'correct' - i.e. not something that just happens to work on Linux? I'd also need to assume that it works cross-platform for file-mappings as well as anonymous mappings.
The sample definitely moves me forwards... documenting that what I ultimately need is probably achievable with mmap() on Linux - at least. What I'd really like is a pointer to documentation that shows that the MAP_FIXED line will work as the sample demonstrates... and, idealy, a transformation from the Linux/Unix specific mmap() to a platform independent (Boost::interprocess) approach.
Your question is a little confusing. From what I understood, this code will do what you need:
#define PAGESIZE 4096
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <errno.h>
#include <sys/types.h>
#include <fcntl.h>
#include <unistd.h>
#include <assert.h>
struct StoredObject
{
int IntVal;
char StrVal[25];
};
int main(int argc, char **argv)
{
int fd = open("mmapfile", O_RDWR | O_CREAT | O_TRUNC, (mode_t) 0600);
//Set the file to the size of our data (2 pages)
lseek(fd, PAGESIZE*2 - 1, SEEK_SET);
write(fd, "", 1); //The final byte
unsigned char *mapPtr = (unsigned char *) mmap(0, PAGESIZE * 2, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
struct StoredObject controlObject;
controlObject.IntVal = 12;
strcpy(controlObject.StrVal, "Mary had a little lamb.\n");
struct StoredObject *mary1;
mary1 = (struct StoredObject *)(mapPtr + PAGESIZE - 4); //Will fall on the boundary between first and second page
memcpy(mary1, &controlObject, sizeof(StoredObject));
printf("%d, %s", mary1->IntVal, mary1->StrVal);
//Should print "12, Mary had a little lamb.\n"
struct StoredObject *john1;
john1 = mary1 + 1; //Comes immediately after mary1 in memory; will start and end in the second page
memcpy(john1, &controlObject, sizeof(StoredObject));
john1->IntVal = 42;
strcpy(john1->StrVal, "John had a little lamb.\n");
printf("%d, %s", john1->IntVal, john1->StrVal);
//Should print "12, Mary had a little lamb.\n"
//Make sure the data's on the disk, as this is the initial, "read-only" data
msync(mapPtr, PAGESIZE * 2, MS_SYNC);
//This is the inital data set, now in memory, loaded across two pages
//At this point, someone could be reading from there. We don't know or care.
//We want to modify john1, but don't want to write over the existing data
//Easy as pie.
//This is the shadow map. COW-like optimization will take place:
//we'll map the entire address space from the shared source, then overlap with a new map to modify
//This is mapped anywhere, letting the system decide what address we'll be using for the new data pointer
unsigned char *mapPtr2 = (unsigned char *) mmap(0, PAGESIZE * 2, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
//Map the second page on top of the first mapping; this is the one that we're modifying. It is *not* backed by disk
unsigned char *temp = (unsigned char *) mmap(mapPtr2 + PAGESIZE, PAGESIZE, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED | MAP_ANON, 0, 0);
if (temp == MAP_FAILED)
{
printf("Fixed map failed. %s", strerror(errno));
}
assert(temp == mapPtr2 + PAGESIZE);
//Make a copy of the old data that will later be changed
memcpy(mapPtr2 + PAGESIZE, mapPtr + PAGESIZE, PAGESIZE);
//The two address spaces should still be identical until this point
assert(memcmp(mapPtr, mapPtr2, PAGESIZE * 2) == 0);
//We can now make our changes to the second page as needed
struct StoredObject *mary2 = (struct StoredObject *)(((unsigned char *)mary1 - mapPtr) + mapPtr2);
struct StoredObject *john2 = (struct StoredObject *)(((unsigned char *)john1 - mapPtr) + mapPtr2);
john2->IntVal = 52;
strcpy(john2->StrVal, "Mike had a little lamb.\n");
//Test that everything worked OK
assert(memcmp(mary1, mary2, sizeof(struct StoredObject)) == 0);
printf("%d, %s", john2->IntVal, john2->StrVal);
//Should print "52, Mike had a little lamb.\n"
//Now assume our garbage collection routine has detected that no one is using the original copy of the data
munmap(mapPtr, PAGESIZE * 2);
mapPtr = mapPtr2;
//Now we're done with all our work and want to completely clean up
munmap(mapPtr2, PAGESIZE * 2);
close(fd);
return 0;
}
My modified answer should address your safety concerns.
Only use MAP_FIXED on the second mmap call (like I have above). The cool thing about MAP_FIXED is that it lets you overwrite an existing mmap address section. It'll unload the range you're overlapping and replace it with your new mapped content:
MAP_FIXED
[...] If the memory
region specified by addr and len overlaps pages of any existing
mapping(s), then the overlapped part of the existing mapping(s) will be
discarded. [...]
This way, you let the OS take care of finding a contiguous memory block of hundreds of megs for you (never call MAP_FIXED on address you don't know for sure isn't available). Then you call MAP_FIXED on a subsection of that now-mapped huge space with the data that you will be modifying. Tada.
On Windows, something like this should work (I'm on a Mac at the moment, so untested):
int main(int argc, char **argv)
{
HANDLE hFile = CreateFile(L"mmapfile", GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
//Set the file to the size of our data (2 pages)
SetFilePointer(hFile, PAGESIZE*2 - 1, 0, FILE_BEGIN);
DWORD bytesWritten = -1;
WriteFile(hFile, "", 1, &bytesWritten, NULL);
HANDLE hMap = CreateFileMapping(hFile, NULL, PAGE_READWRITE, 0, PAGESIZE * 2, NULL);
unsigned char *mapPtr = (unsigned char *) MapViewOfFile(hMap, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, PAGESIZE * 2);
struct StoredObject controlObject;
controlObject.IntVal = 12;
strcpy(controlObject.StrVal, "Mary had a little lamb.\n");
struct StoredObject *mary1;
mary1 = (struct StoredObject *)(mapPtr + PAGESIZE - 4); //Will fall on the boundary between first and second page
memcpy(mary1, &controlObject, sizeof(StoredObject));
printf("%d, %s", mary1->IntVal, mary1->StrVal);
//Should print "12, Mary had a little lamb.\n"
struct StoredObject *john1;
john1 = mary1 + 1; //Comes immediately after mary1 in memory; will start and end in the second page
memcpy(john1, &controlObject, sizeof(StoredObject));
john1->IntVal = 42;
strcpy(john1->StrVal, "John had a little lamb.\n");
printf("%d, %s", john1->IntVal, john1->StrVal);
//Should print "12, Mary had a little lamb.\n"
//Make sure the data's on the disk, as this is the initial, "read-only" data
//msync(mapPtr, PAGESIZE * 2, MS_SYNC);
//This is the inital data set, now in memory, loaded across two pages
//At this point, someone could be reading from there. We don't know or care.
//We want to modify john1, but don't want to write over the existing data
//Easy as pie.
//This is the shadow map. COW-like optimization will take place:
//we'll map the entire address space from the shared source, then overlap with a new map to modify
//This is mapped anywhere, letting the system decide what address we'll be using for the new data pointer
unsigned char *reservedMem = (unsigned char *) VirtualAlloc(NULL, PAGESIZE * 2, MEM_RESERVE, PAGE_READWRITE);
HANDLE hMap2 = CreateFileMapping(hFile, NULL, PAGE_READWRITE, 0, PAGESIZE, NULL);
unsigned char *mapPtr2 = (unsigned char *) MapViewOfFileEx(hMap2, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, PAGESIZE, reservedMem);
//Map the second page on top of the first mapping; this is the one that we're modifying. It is *not* backed by disk
unsigned char *temp = (unsigned char *) MapViewOfFileEx(hMap2, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, PAGESIZE, reservedMem + PAGESIZE);
if (temp == NULL)
{
printf("Fixed map failed. 0x%x\n", GetLastError());
return -1;
}
assert(temp == mapPtr2 + PAGESIZE);
//Make a copy of the old data that will later be changed
memcpy(mapPtr2 + PAGESIZE, mapPtr + PAGESIZE, PAGESIZE);
//The two address spaces should still be identical until this point
assert(memcmp(mapPtr, mapPtr2, PAGESIZE * 2) == 0);
//We can now make our changes to the second page as needed
struct StoredObject *mary2 = (struct StoredObject *)(((unsigned char *)mary1 - mapPtr) + mapPtr2);
struct StoredObject *john2 = (struct StoredObject *)(((unsigned char *)john1 - mapPtr) + mapPtr2);
john2->IntVal = 52;
strcpy(john2->StrVal, "Mike had a little lamb.\n");
//Test that everything worked OK
assert(memcmp(mary1, mary2, sizeof(struct StoredObject)) == 0);
printf("%d, %s", john2->IntVal, john2->StrVal);
//Should print "52, Mike had a little lamb.\n"
//Now assume our garbage collection routine has detected that no one is using the original copy of the data
//munmap(mapPtr, PAGESIZE * 2);
mapPtr = mapPtr2;
//Now we're done with all our work and want to completely clean up
//munmap(mapPtr2, PAGESIZE * 2);
//close(fd);
return 0;
}
but I'm unclear how I should reserve address space in order to do this safely
That's going to vary by OS, but a little digging on msdn for mmap (I started with "xp mmap" on the msdn search) shows Microsoft have their usual VerboseAndHelpfullyCapitalizedNames for (the many) functions that implement pieces of mmap. Both the file- and anonymous- mappers can handle fixed-address requests just the same as any POSIX-2001 system can, i.e. if something else in your address space is talking to the kernel, you get to sort it out. No way I'm going to touch "safely", there's no such thing as "safely" with code you're wanting to port to unspecified platforms. You're going to have to build your own pool of pre-mapped anonymous memory that you can unmap and parcel out later under your own control.
I tested the windows code from #Mahmoud, well actually I tested the following similar code, and it doesn't work (the Linux code works.) If you uncomment VirtualFree, it will work. As mentioned in my comment above, on windows you can reserve the address space with VirtualAlloc, but you can't use MapViewOfFileEx with an already mapped address, so you need to VirtualFree it first. Then there's a race condition where another thread can grab the memory address before you do, so you have to do everything in a loop, e.g. try up to 1000 times and then give up.
package main
import (
"fmt"
"os"
"syscall"
)
func main() {
const size = 1024 * 1024
file, err := os.Create("foo.dat")
if err != nil {
panic(err)
}
if err := file.Truncate(size); err != nil {
panic(err)
}
const MEM_COMMIT = 0x1000
addr, err := virtualAlloc(0, size, MEM_COMMIT, protReadWrite)
if err != nil {
panic(err)
}
fd, err := syscall.CreateFileMapping(
syscall.Handle(file.Fd()),
nil,
uint32(protReadWrite),
0,
uint32(size),
nil,
)
//if err := virtualFree(addr); err != nil {
// panic(err)
//}
base, err := mapViewOfFileEx(fd, syscall.FILE_MAP_READ|syscall.FILE_MAP_WRITE, 0, 0, size, addr)
if base == 0 {
panic("mapViewOfFileEx returned 0")
}
if err != nil {
panic(err)
}
fmt.Println("success!")
}
type memProtect uint32
const (
protReadOnly memProtect = 0x02
protReadWrite memProtect = 0x04
protExecute memProtect = 0x20
protAll memProtect = 0x40
)
var (
modkernel32 = syscall.MustLoadDLL("kernel32.dll")
procMapViewOfFileEx = modkernel32.MustFindProc("MapViewOfFileEx")
procVirtualAlloc = modkernel32.MustFindProc("VirtualAlloc")
procVirtualFree = modkernel32.MustFindProc("VirtualFree")
procVirtualProtect = modkernel32.MustFindProc("VirtualProtect")
)
func mapViewOfFileEx(handle syscall.Handle, prot memProtect, offsetHigh uint32, offsetLow uint32, length uintptr, target uintptr) (addr uintptr, err error) {
r0, _, e1 := syscall.Syscall6(procMapViewOfFileEx.Addr(), 6, uintptr(handle), uintptr(prot), uintptr(offsetHigh), uintptr(offsetLow), length, target)
addr = uintptr(r0)
if addr == 0 {
if e1 != 0 {
err = error(e1)
} else {
err = syscall.EINVAL
}
}
return addr, nil
}
func virtualAlloc(addr, size uintptr, allocType uint32, prot memProtect) (mem uintptr, err error) {
r0, _, e1 := syscall.Syscall6(procVirtualAlloc.Addr(), 4, addr, size, uintptr(allocType), uintptr(prot), 0, 0)
mem = uintptr(r0)
if e1 != 0 {
return 0, error(e1)
}
return mem, nil
}
func virtualFree(addr uintptr) error {
const MEM_RELEASE = 0x8000
_, _, e1 := syscall.Syscall(procVirtualFree.Addr(), 3, addr, 0, MEM_RELEASE)
if e1 != 0 {
return error(e1)
}
return nil
}