I want to use shared memory to transfer large images (10M or more) between two processes, but I can only request 4096 bytes of BUF_SIZE when requesting shared memory.beyond 4096 bytes, the program will crash.
Here is my code:
hMapFile = CreateFileMappingW(
INVALID_HANDLE_VALUE, // use paging file
NULL, // default security
PAGE_READWRITE, // read/write access
0, // maximum object size (high-order DWORD)
BUF_SIZE, // maximum object size (low-order DWORD)超过4096就不行了,不知道怎么解决
szMapName);
My system is windos64, Use C++ in qt.anyone have the same problem as me?
I think I know why the program crashed. It's because there's something wrong with the code below.
hMapFile = CreateFileMappingW(
INVALID_HANDLE_VALUE, // use paging file
NULL, // default security
PAGE_READWRITE, // read/write access
0, // maximum object size (high-order DWORD)
BUF_SIZE, // maximum object size (low-order DWORD)
szMapName); // name of mapping object
if (hMapFile == NULL){
qDebug() << (TEXT("Could not create file mapping object (%d).\n"),GetLastError());
return -1;
}
pBuf = (LPTSTR)MapViewOfFile(
hMapFile, // handle to map object
FILE_MAP_ALL_ACCESS, // read/write permission
0,
0,
4097);//if this number > 4096, the program will stuck
if (pBuf == NULL){
qDebug() << (TEXT("Could not map view of file (%d).\n"),
GetLastError());
CloseHandle(hMapFile);
return -1;
}
You called CreateFileMappingW with BUF_SIZE as the low order DWORD size (with the high order DWORD being 0, so BUF_SIZE is the total number of bytes). That's the limit on what you can map later with MapViewOfFile. I'm guessing you defined BUF_SIZE as a constant 4096 somewhere, so that's the limit on what MapViewOfFile can map. Use a larger number for CreateFileMappingW and MapViewOfFile can increase to match.
Related
Half of the buffer used with ReadFile is corrupt. Regardless of the size of the buffer, half of it has the same corrupted character. I have look for anything that could be causing the read to stop early, etc. If I increase the size of the buffer, I see more of the file so it is not failing on a particular part of the file.
Visual Studio 2019. Windows 10.
#define MAXBUFFERSIZE 1024
DWORD bufferSize = MAXBUFFERSIZE;
_int64 fileRemaining;
HANDLE hFile;
DWORD dwBytesRead = 0;
//OVERLAPPED ol = { 0 };
LARGE_INTEGER dwPosition;
TCHAR* buffer;
hFile = CreateFile(
inputFilePath, // file to open
GENERIC_READ, // open for reading
FILE_SHARE_READ, // share for reading
NULL, // default security
OPEN_EXISTING, // existing file only
FILE_ATTRIBUTE_NORMAL, // normal file | FILE_FLAG_OVERLAPPED
NULL); // no attr. template
if (hFile == INVALID_HANDLE_VALUE)
{
DisplayErrorBox((LPWSTR)L"CreateFile");
return 0;
}
LARGE_INTEGER size;
GetFileSizeEx(hFile, &size);
_int64 fileSize = (__int64)size.QuadPart;
double gigabytes = fileSize * 9.3132e-10;
sendToReportWindow(L"file size: %lld bytes \(%.1f gigabytes\)\n", fileSize, gigabytes);
if(fileSize > MAXBUFFERSIZE)
{
buffer = new TCHAR[MAXBUFFERSIZE];
}
else
{
buffer = new TCHAR[fileSize];
}
fileRemaining = fileSize;
sendToReportWindow(L"file remaining: %lld bytes\n", fileRemaining);
while (fileRemaining) // outer loop. while file remaining, read file chunk to buffer
{
sendToReportWindow(L"fileRemaining:%d\n", fileRemaining);
if (bufferSize > fileRemaining) // as fileremaining gets smaller as file is processed, it eventually is smaller than the buffer
bufferSize = fileRemaining;
if (FALSE == ReadFile(hFile, buffer, bufferSize, &dwBytesRead, NULL))
{
sendToReportWindow(L"file read failed\n");
CloseHandle(hFile);
return 0;
}
fileRemaining -= bufferSize;
// bunch of commented out code (verified that it does not cause the corruption)
}
delete [] buffer;
Debugger html view (512 byte buffer)
Debugger html view (1024 byte buffer). This shows that file is probably not the source of the corruption.
Misc notes: I have been told that memory mapping the file does not provide an advantage since I am sequentially processing the file. Another advantage to this method is that when I detect particular and reoccurring tags in the WARC file I can skip ahead ~500 bytes and resume processing. This improves speed.
The reason is that you use a buffer array of type TCHAR, and the size of TCHAR type is 2 bytes. So the bufferSize set when you call the ReadFile function is actually filled into the buffer array every 2 bytes.
But the actual size of the buffer is sizeof(TCHAR) * fileSize, so half of the buffer array you see is "corrupted"
I would like to resize a memory mapped file on windows, without invalidating the pointer retrieved from a previous call to MapViewOfFileEx. This way, all pointers to any file data that are stored throughout the application are not invalidated by the resize operation.
I found a solution for the problem but im not sure whether this approach is actually guaranteed to work in all cases.
This is my approach:
I reserve a large memory region with VirtualAlloc:
reserved_pages_ptr = (char*)VirtualAlloc(nullptr, MAX_FILE_SIZE, MEM_RESERVE, PAGE_NOACCESS);
base_address = reserved_pages_ptr;
Every time the memory map is resized, I close the old file mapping, release the reserved pages and reserve the rest pages, that are not needed for the current size of the file:
filemapping_handle = CreateFileMappingW(...);
SYSTEM_INFO info;
GetSystemInfo(&info);
const DWORD page_size = info.dwAllocationGranularity;
const DWORD pages_needed = file_size / page_size + size_t(file_size % page_size != 0);
// release reserved pages:
VirtualFree(reserved_pages_ptr, 0, MEM_RELEASE);
// reserve rest pages:
reserved_pages_ptr = (char*)VirtualAlloc(
base_address + pages_needed * page_size,
MAX_FILE_SIZE - pages_needed * page_size,
MEM_RESERVE, PAGE_NOACCESS
);
if(reserved_pages_ptr != base_address + pages_needed * page_size)
{
//I hope this never happens...
}
Then i can map the view with MapViewOfFileEx:
data_ = (char*)MapViewOfFileEx(filemapping_handle, ... , base_address);
if (data_ != base_address)
{
//I hope this also never happens...
}
Is this approach stable enough to guarantee, that the potential problems never occur?
Do I need any synchronization to avoid problems with multithreading?
EDIT: I know that the most stable approach would be to change the rest of the application to allow invalidating all the file data pointers, but this solution could be an easy approach that mirrors the behavior of mmap on Linux.
The solution depends on whether you use file mapping object is backed by the operating system paging file (the hFile parameter is INVALID_HANDLE_VALUE), or by some file on disk.
In this case, you use file mapping object is backed by the operating system paging file, you need use the SEC_RESERVE flag:
specifies that when a view of the file is mapped into a process
address space, the entire range of pages is reserved for later use by
the process rather than committed. Reserved pages can be committed to
subsequent calls to the VirtualAlloc function. After the pages are
committed, they cannot be freed or decommitted with the VirtualFree
function.
The code can look like:
#define MAX_FILE_SIZE 0x10000000
void ExtendInMemorySection()
{
if (HANDLE hSection = CreateFileMapping(INVALID_HANDLE_VALUE, 0,
PAGE_READWRITE|SEC_RESERVE, 0, MAX_FILE_SIZE, NULL))
{
PVOID pv = MapViewOfFile(hSection, FILE_MAP_WRITE, 0, 0, 0);
CloseHandle(hSection);
if (pv)
{
SYSTEM_INFO info;
GetSystemInfo(&info);
PBYTE pb = (PBYTE)pv;
int n = MAX_FILE_SIZE / info.dwPageSize;
do
{
if (!VirtualAlloc(pb, info.dwPageSize, MEM_COMMIT, PAGE_READWRITE))
{
break;
}
pb += info.dwPageSize;
} while (--n);
UnmapViewOfFile(pv);
}
}
}
But from SEC_RESERVE
This attribute has no effect for file mapping objects that are backed
by executable image files or data files (the hfile parameter is a
handle to a file).
For this (and only this) case exists undocumented API:
NTSYSCALLAPI
NTSTATUS
NTAPI
NtExtendSection(
_In_ HANDLE SectionHandle,
_Inout_ PLARGE_INTEGER NewSectionSize
);
This API lets you extend section size (and backed file). Also, SectionHandle must have SECTION_EXTEND_SIZE access right in this case, but CreateFileMapping creates a section handle without this access. So we need use only NtCreateSection here, then we need use ZwMapViewOfSection api with AllocationType = MEM_RESERVE and ViewSize = MAX_FILE_SIZE - this reserve ViewSize region of memory but not commit it, but after calling NtExtendSection the valid data (commit pages) in view will be auto extended.
Before win 8.1, the MapViewOfFile not such functionality for the pass MEM_RESERVE allocation type to ZwMapViewOfSection, but begin from win 8 (or 8.1) exist undocumented flag FILE_MAP_RESERVE which let do this.
In general, demonstration code can look like:
#define MAX_FILE_SIZE 0x10000000
void ExtendFileSection()
{
HANDLE hFile = CreateFile(L"d:/ee.tmp", GENERIC_ALL, 0, 0, CREATE_ALWAYS, 0, 0);
if (hFile != INVALID_HANDLE_VALUE)
{
HANDLE hSection;
SYSTEM_INFO info;
GetSystemInfo(&info);
// initially only 1 page in the file
LARGE_INTEGER SectionSize = { info.dwPageSize };
NTSTATUS status = NtCreateSection(&hSection,
SECTION_EXTEND_SIZE|SECTION_MAP_READ|SECTION_MAP_WRITE, 0,
&SectionSize, PAGE_READWRITE, SEC_COMMIT, hFile);
CloseHandle(hFile);
if (0 <= status)
{
PVOID BaseAddress = 0;
SIZE_T ViewSize = MAX_FILE_SIZE;
//MapViewOfFile(hSection, FILE_MAP_WRITE|FILE_MAP_RESERVE, 0, 0, MAX_FILE_SIZE);
status = ZwMapViewOfSection(hSection, NtCurrentProcess(), &BaseAddress, 0, 0, 0,
&ViewSize, ViewUnmap, MEM_RESERVE, PAGE_READWRITE);
if (0 <= status)
{
SIZE_T n = MAX_FILE_SIZE / info.dwPageSize - 1;
do
{
SectionSize.QuadPart += info.dwPageSize;
if (0 > NtExtendSection(hSection, &SectionSize))
{
break;
}
} while (--n);
UnmapViewOfFile(BaseAddress);
}
CloseHandle(hSection);
}
}
}
I need a mechanism to share memory between some threads (usually in the same process, but sometimes not).
This code, which seems very basic, fails with error 5 (access denied) on MapViewOfFile:
HANDLE hSharedMemCreated = CreateFileMapping(
INVALID_HANDLE_VALUE, // use paging file
NULL, // default security
PAGE_READWRITE, // read/write access
0, // maximum object size (high-order DWORD)
10000, // maximum object size (low-order DWORD)
"testFileMapping"); // name of mapping object
HANDLE hSharedMemOpened = OpenFileMapping(
PAGE_READWRITE, // read/write access
FALSE,
"testFileMapping"
);
void* location = MapViewOfFile(
hSharedMemOpened, // handle to map object
FILE_MAP_ALL_ACCESS, // read/write permission
0,
0,
10);
MapViewOfFile succeeds with the opened handle hSharedMemOpened with permission FILE_MAP_READ.
MapViewOfFile succeeds with the created handle hSharedMemCreated with permission FILE_MAP_ALL_ACCESS.
MapViewOfFile fails with the opened handle hSharedMemOpened with permission FILE_MAP_ALL_ACCESS.
The answer is in the comments:
PAGE_READWRITE is not a valid argument for OpenFileMapping(). You probably want FILE_MAP_ALL_ACCESS instead.
I am trying to follow this tutorial on MSDN: Creating Named Shared Memory.
Unfortunately, I get NULL from CreateFileMapping(). The file shmfile.txt exists on my file system, so I thought no problems should occur, the mapping would be created, and the file would take its role as my shared memory object.
What am I doing wrong?
This is my code:
...
#define BUF_SIZE 256
TCHAR szName[]=TEXT("C:\\Users\\joe\\shmfolder\\shmfile.txt");
int main(){
HANDLE hMapFile; // filehandle
LPCTSTR pBuf;
hMapFile = CreateFileMapping(
INVALID_HANDLE_VALUE, // use paging file
NULL, // default security
PAGE_READWRITE, // read/write access
0, // maximum object size (high-order DWORD)
BUF_SIZE, // maximum object size (low-order DWORD)
szName); // name of mapping object
if (hMapFile == NULL)
{
_tprintf(TEXT("Could not create file mapping object (%d).\n"), GetLastError());
return 1;
}
...
}
Right now, you're telling CreateFileMapping to create the mapping in the page file, then use the name of your existing file as the name of the file mapping.
The name you give for a file mapping object can have local\ or global\ as a prefix, but can't contain any other back-slashes.
If you want to map your pre-existing file as a shared memory region, you'd do something on this general order:
TCHAR szName[]=TEXT("C:\\Users\\joe\\shmfolder\\shmfile.txt");
TCHAR szMapName[]=TEXT("SharedFile");
HANDLE file = CreateFile(szName, ...);
HANDLE mapping = CreateFileMapping(file, ..., szMapName);
I want to make a share memory pool for all the other process to share the data, but after I read about the CreateFileMapping API document, I was confused about that it will need to specify the size of the share memory. I actually want it to be dynamic allocate and free which looks more like a service. Is there some way to do the share memory dynamic using createFileMapping or not?
Creating Named Shared Memory.
First Process
The first process creates the file mapping object by calling the CreateFileMapping function with INVALID_HANDLE_VALUE and a name for the object. By using the PAGE_READWRITE flag, the process has read/write permission to the memory through any file views that are created.
Then the process uses the file mapping object handle that CreateFileMapping returns in a call to MapViewOfFile to create a view of the file in the process address space. The MapViewOfFile function returns a pointer to the file view, pBuf. The process then uses the CopyMemory function to write a string to the view that can be accessed by other processes.
Process 1 code:
#include <windows.h>
#include <stdio.h>
#include <conio.h>
#include <tchar.h>
#define BUF_SIZE 256
TCHAR szName[]=TEXT("Global\\MyFileMappingObject");
TCHAR szMsg[]=TEXT("Message from first process.");
int _tmain()
{
HANDLE hMapFile;
LPCTSTR pBuf;
hMapFile = CreateFileMapping(
INVALID_HANDLE_VALUE, // use paging file
NULL, // default security
PAGE_READWRITE, // read/write access
0, // maximum object size (high-order DWORD)
BUF_SIZE, // maximum object size (low-order DWORD)
szName); // name of mapping object
if (hMapFile == NULL)
{
_tprintf(TEXT("Could not create file mapping object (%d).\n"),
GetLastError());
return 1;
}
pBuf = (LPTSTR) MapViewOfFile(hMapFile, // handle to map object
FILE_MAP_ALL_ACCESS, // read/write permission
0,
0,
BUF_SIZE);
if (pBuf == NULL)
{
_tprintf(TEXT("Could not map view of file (%d).\n"),
GetLastError());
CloseHandle(hMapFile);
return 1;
}
CopyMemory((PVOID)pBuf, szMsg, (_tcslen(szMsg) * sizeof(TCHAR)));
_getch();
UnmapViewOfFile(pBuf);
CloseHandle(hMapFile);
return 0;
}
Second Process
A second process can access the string written to the shared memory by the first process by calling the OpenFileMapping function specifying the same name for the mapping object as the first process. Then it can use the MapViewOfFile function to obtain a pointer to the file view, pBuf. The process can display this string as it would any other string. In this example, the message box displayed contains the message "Message from first process" that was written by the first process.
Process 2 code:
#include <windows.h>
#include <stdio.h>
#include <conio.h>
#include <tchar.h>
#pragma comment(lib, "user32.lib")
#define BUF_SIZE 256
TCHAR szName[]=TEXT("Global\\MyFileMappingObject");
int _tmain()
{
HANDLE hMapFile;
LPCTSTR pBuf;
hMapFile = OpenFileMapping(
FILE_MAP_ALL_ACCESS, // read/write access
FALSE, // do not inherit the name
szName); // name of mapping object
if (hMapFile == NULL)
{
_tprintf(TEXT("Could not open file mapping object (%d).\n"),
GetLastError());
return 1;
}
pBuf = (LPTSTR) MapViewOfFile(hMapFile, // handle to map object
FILE_MAP_ALL_ACCESS, // read/write permission
0,
0,
BUF_SIZE);
if (pBuf == NULL)
{
_tprintf(TEXT("Could not map view of file (%d).\n"),
GetLastError());
CloseHandle(hMapFile);
return 1;
}
MessageBox(NULL, pBuf, TEXT("Process2"), MB_OK);
UnmapViewOfFile(pBuf);
CloseHandle(hMapFile);
return 0;
}
Source : http://msdn.microsoft.com/en-us/library/windows/desktop/aa366551(v=vs.85).aspx