My goal is to create a simple addon that opens a memory mapped file and gives node raw access to the buffer. The memory mapped file already exists so I don't need to create a new file -- error will be thrown if file not already created (by another process).
Here is what I have so far:
#include <nan.h>
#include <windows.h>
using namespace v8;
void get_shm(const Nan::FunctionCallbackInfo <v8::Value> &info) {
if (info.Length() != 1) {
return Nan::ThrowTypeError("Wrong number of arguments. You must specify size of mapped file!");
}
if (!info[0]->IsNumber()) {
return Nan::ThrowTypeError("Bad argument. Size of mapped file must be a number!");
}
int len = info[0]->Uint32Value();
HANDLE h = OpenFileMappingA(FILE_MAP_WRITE | FILE_MAP_READ, 1, "Global\\_GB_SharedMemory_Read");
if (h == NULL) {
Nan::ThrowTypeError("Couldn't open memory mapped file");
}
char *buf = (char *) MapViewOfFile(h, FILE_MAP_WRITE | FILE_MAP_READ, 0, 0, len);
Nan::MaybeLocal<Object> jsbuf = Nan::NewBuffer(buf, len);
info.GetReturnValue().Set(jsbuf.ToLocalChecked());
}
void Init(v8::Local <v8::Object> exports) {
exports->Set(Nan::New("get_shm").ToLocalChecked(),
Nan::New<v8::FunctionTemplate>(get_shm)->GetFunction());
}
NODE_MODULE(addon_shm, Init)
Unfortunately node is crashing (presumably due to access violation) at some point after the NewBuffer is returned. It doesn't crash immediately but some unpredictable time after.
I'm not attempting to read/write from the buffer in Javascript. The only line using the add on is this:
const buf=shm.get_shm(10);
Mapped file created by the other process is bigger than 10.
If I use malloc to create the buffer, problem doesn't occur.
If I just call get_shm(10) without storing the result, problem doesn't occur.
Struggling to understand the issue...
Related
Accordingly to the ''best practices'' I have learned, we should load the resources we need to our programs into RAM, avoiding unnecessary requests to user's hard drive. Using SDL2, I always free image files after loading them into RAM. (File -> Surface -> Texture -> Free File/Surface). So, if I other application changes the file, my program ignores it, as the file is not in use by it anymore.
Now in lesson 16 I am learning to use the Add-on SDL_ttf.
However, using SDL_ttf addon I could not find a way to free the font.ttf file, loading it into RAM too. I can only see it through a pointer. It seems to me that the file keeps being read each time I render a text.
How can I load it into RAM, so the rendering calls a RAM position, instead of the file in HD?
Full code
#define SDL_MAIN_HANDLED
#include <SDL2/SDL.h>
#include <SDL2/SDL_ttf.h>
int G = 255;
int main (void) {SDL_SetMainReady();
int SCREEN_WIDTH = 800;
int SCREEN_HEIGHT = 600;
bool QUIT_APPLICATION = false;
SDL_Event union_Event_manager;
SDL_Color str_White_colour = {255,255,255,255};
SDL_Window * ptr_Window = nullptr;
SDL_Surface * ptr_Text_Surface = nullptr;
SDL_Surface * ptr_Main_surface = nullptr;
SDL_RWops * ptr_str_rwops = nullptr;
TTF_Font * ptr_Font = nullptr;
SDL_Init(SDL_INIT_VIDEO);
TTF_Init();
ptr_Window = SDL_CreateWindow("Lesson 16 - TrueTypeFonts", SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, SCREEN_WIDTH, SCREEN_HEIGHT, SDL_WINDOW_SHOWN);
ptr_Main_surface = SDL_GetWindowSurface(ptr_Window);
ptr_str_rwops = SDL_RWFromFile("FreeMono.ttf", "r");
ptr_Font = TTF_OpenFontIndexRW(ptr_str_rwops, 1, 72, 0);
ptr_Text_Surface = TTF_RenderText_Solid(ptr_Font, "Hello World", str_White_colour);
while(!QUIT_APPLICATION){
while(SDL_PollEvent(&union_Event_manager) != 0 ){
if (union_Event_manager.type == SDL_QUIT) {QUIT_APPLICATION = true;}
/*END WHILE*/}
SDL_BlitSurface(ptr_Text_Surface, NULL, ptr_Main_surface, NULL);
SDL_UpdateWindowSurface(ptr_Window);
/*END WHILE*/}
TTF_CloseFont(ptr_Font);
// if called before any rendering, the app crashes, as supposed to.
// So, how free the **file** and keep using its content from RAM?
SDL_RWclose(ptr_str_rwops);
SDL_FreeSurface(ptr_Text_Surface);
SDL_FreeSurface(ptr_Main_surface);
SDL_DestroyWindow(ptr_Window);
ptr_Font = nullptr;
ptr_str_rwops = nullptr;
ptr_Text_Surface = nullptr;
ptr_Main_surface = nullptr;
ptr_Window = nullptr;
TTF_Quit();
SDL_Quit();
return (0);}
Failure 1:
Create a structure to hold information from file.
TTF_Font str_Font; // Error in compilation ''incomplete type''
str_Font = *ptr_Font;
TTF_CloseFont(ptr_Font);
ptr_Font = nullptr;
ptr_Font = &str_Font;
Reason to failure:
I misunderstood how the file works. The structure only holds information about the file, not the media itself.
This approach is useless, and crash the program just after freeing the pointer (the rendering tries to dereference a nullptr).
Failure 2:
Use built in function to free resource.
ptr_Font = TTF_OpenFontIndexRW(SDL_RWFromFile("FreeMono.ttf", "r"), 1, 72, 0);
Reason to failure:
I do not understand why, as the second argument (non-zero) specifies it should free the resource after usage. It also happens in the completed source code above, where I merely separated the functions in two lines.
Failure 3:
Create structure to hold information about pointer.
ptr_str_rwops = SDL_RWFromFile("FreeMono.ttf", "r");
str_rwops = *ptr_str_rwops;
SDL_RWclose(ptr_str_rwops); // crashes the program
ptr_str_rwops = nullptr;
ptr_str_rwops = &str_rwops; // useless: file still in use.
Reason to failure:
The structure RWops seems to not hold the file, only information about it. So it is the sum of failure 1 and 2.
Failure 4:
Tried to load file as object.
ptr_LoadObject = (TTF_Font*)SDL_LoadObject("FreeMono.ttf");
ptr_str_rwops = SDL_RWFromFile((const char *)ptr_LoadObject, "r");
Reason to failure:
This function works with shared operational system files. Wrong usage of function.
Update 2019-04-05
Failure 5
Tried to make a copy of file directly into RAM useing memcpy
long int func_discover_file_size(char* file){
long int var_file_size = 0;
FILE * ptr_file = nullptr;
ptr_file = fopen(file, "rb");
fseek(ptr_file , 0L , SEEK_END);
var_file_size = ftell(ptr_file);
fclose(ptr_file);
return var_file_size;
/*END func_discover_file_size*/}
int main (void) {
/*cut unrelated code*/
void * ptr_load_file = nullptr;
void * ptr_File_copy = nullptr;
long int var_file_size = 0;
/*cut unrelated code*/
var_file_size = func_discover_file_size("FreeMono.ttf");
// works fine and returns correct size of file.
ptr_File_copy = (char*) calloc (1, var_file_size);
// memory allocation works fine (tested)
ptr_load_file = fopen("FreeMono.ttf", "rb");
// file loaded correctly. Test with FOR LOOP shows content of file in console.
memcpy(ptr_File_copy, ptr_load_file, var_file_size);
// program crashes in line above
Reason to failure:
It looks like I do not know how to correctly use memcpy. I tried many many casts to function and pointers (void, char), tried to change type of pointers to char, void, FILE, tried to output to a third pointer...
Now I am looking for a good soul to enlight my ways... :-p
note: C tagged because SDL
While freetype (which SDL_ttf uses) will not read font more than once (which it can't, since its API doesn't provide seek functionality), SDL_ttf will not close file/RWops until font closes. You can achieve what you've described via manually loading file into memory buffer and using that memory as RWops to feed data to SDL_ttf, e.g. (no error checking, no free, etc. - this is just an example):
/* 'slurp' file (read entire file into memory buffer)
* there are multiple ways to do so
*/
SDL_RWops *file_rw = SDL_RWFromFile("font.ttf", "rb");
Sint64 file_sz = file_rw->size(file_rw);
void *membuf = malloc(file_sz);
file_rw->read(file_rw, membuf, 1, file_sz);
file_rw->close(file_rw);
/* use memory buffer as RWops */
SDL_RWops *mem_rw = SDL_RWFromConstMem(membuf, file_sz);
TTF_Font *font = TTF_OpenFontRW(mem_rw, 1, font_size);
/* free(membuf) when you're done with the font */
The secondary question about memcpy can be solved in the following way:
memcpy copies a file object, not its contents. In order to read from it:
Use fread function to read from FILE*: fread(ptr_File_copy, 1,
var_file_size, ptr_load_file) instead of memcpy.
I'm trying to share memory between two processes C# (EXE) and C++ (DLL).
The C++ DLL is dynamically loaded by a Citrix receiver process.
Below is my C# and C++ code:
private void Test()
{
string data = "Hello";
_memoryMap = MemoryMappedFile.CreateOrOpen("14614C87-8D7F-45FC-8D59-DCBF1715A715", data.Length, MemoryMappedFileAccess.ReadWrite);
var stream = _memoryMap.CreateViewStream();
using (BinaryWriter binReader = new BinaryWriter(stream))
{
binReader.Write(data);
}
}
void readFromMemoryMap()
{
HANDLE hMapObject = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"14614C87-8D7F-45FC-8D59-DCBF1715A715");
char buffer[2000];
if (hMapObject == NULL)
{
sprintf(buffer, "OpenFileMapping: GetLastError=%d", GetLastError());
}
char *fileBuffer = (char *)MapViewOfFile(hMapObject, FILE_MAP_ALL_ACCESS, 0, 0, 6);
UnmapViewOfFile(fileBuffer);
CloseHandle(hMapObject);
}
I took care of writing and reading an object sequence. My C# application's memory map handle didn't crash. I have run this code using an administrator account which has rights.
I am facing the below error. Can anyone help me with the case of this error?
I followed below link for fix this issue.
OpenFileMapping ERROR_FILE_NOT_FOUND
Try prefixing the object name with Global\ like it says here: https://learn.microsoft.com/en-us/windows/win32/memory/creating-named-shared-memory
I'm currently working on a mac application to the transfer file from mac to mtp device such as usb.
I use the code below to add it:
LIBMTP_file_t *genfile;
char *AbsolutePath = NULL;
char *FileNameOnly = NULL;
QFileInfo fileInfo(filename);
FileNameOnly = convertQStr2char(fileInfo.fileName());
AbsolutePath = convertQStr2char(fileInfo.absoluteFilePath());
genfile = LIBMTP_new_file_t();
genfile->filesize = filesize;
genfile->filename = FileNameOnly;
genfile->filetype = file_type;
genfile->parent_id = CurrentFolderID;
genfile->storage_id = PulsDeviceMngr->devicestorage->id;
if ((genfile->filetype != LIBMTP_FILETYPE_ALBUM) && (genfile->filetype != LIBMTP_FILETYPE_PLAYLIST)) {
ret = LIBMTP_Send_File_From_File(PulsDeviceMngr->device, AbsolutePath, genfile, ProgressBar, NULL);
if (ret != 0) {
qDebug() << "Error sending file";
LIBMTP_Dump_Errorstack(PulsDeviceMngr->device);
LIBMTP_Clear_Errorstack(PulsDeviceMngr->device);
} else {
PulsDeviceMngr->devicestorage->FreeSpaceInBytes -= filesize;
PulsDeviceMngr->devicestorage->FreeSpaceInObjects--;
}
}
LIBMTP_destroy_file_t(genfile);
This basically what is required by mtp to make it works.
I'm usine QT/C++ to develop the Software.
it seems that the QString is not impacted but only the char . char is needed because the api used only support char . I'm converting QString to char using the following code.
char *mtp_wrapper::convertQStr2char(QString str) {
QByteArray latin_str = str.toLatin1();
char *return_str = latin_str.data();
return return_str;
}
I got an issue to release genfile ?? it seems that the runtime consider that I can't free an not allocated pointer but I use to do it using the line below:
genfile = LIBMTP_new_file_t();
LIBMTP_destroy_file_t(genfile);
There is no precondition to do it. when genfile happens, destroy happened. Some data are allocated on the genfile.
I'm able now to open the file for PTP stat complain that I can not send object
Any idea ?
The problem lies in the convertQStr2char() method. The latin_str object is created on the stack and will be destroyed at the end of the enclosing code block. This object is responsible for management of its internal data.
When you call char *return_str = latin_str.data();, you only copy the pointer to latin_str's internal data. As your method returns, latin_str is destroyed and the memory pointed to by data() is freed, so your returned pointer now points to a freed memory as well.
LIBMTP_destroy_file_t is documented to deallocate memory used by the structure, including any strings. In particular, when you call it, it also tries to free the memory pointed to by FileNameOnly, but - as I explained above - the memory has been already freed by QByteArray's destructor, hence the error.
Since using freed pointer results in an undefined behavior, this may be also the cause of your second issue.
I use function CreateFileMapping MapViewOfFile UnmapViewOfFile to create a shared block on the disk. Can I placement new a class object on this block?
I use VS2003 IDE.
try
{
Sphere *pData = m_pBVH->GetFirstHalfData();
Sphere *p = new(pData)Sphere(center, radius, index);
}
catch (std::exception& e)
{
// Can't catch
}
Placement new error:
#ifdef _DEBUG
#define new DEBUG_NEW
#endif
At the Debug Mode, placement new operator may cause a syntax error. It's likely because of the code above, anotate it and try again!
At Release mode, it's OK, but throws exception which is not std::bad_alloc, can't figure out which it is. Maybe there is something with the way shared memory is built. My shared memory creation code is as bellow:
CBVHVertexBuffer::CBVHVertexBuffer(const CString &file, unsigned int size)
{
this->file = file;
this->size = size;
fileHandle = INVALID_HANDLE_VALUE;
mapHandle = INVALID_HANDLE_VALUE;
mapLength = 0;
start = NULL;
end = NULL;
head = NULL;
tail = NULL;
fileHandle = ::CreateFile(
file,
GENERIC_READ | GENERIC_WRITE,
0,
NULL,
CREATE_ALWAYS,
FILE_FLAG_SEQUENTIAL_SCAN | FILE_FLAG_DELETE_ON_CLOSE,
NULL);
if (fileHandle != INVALID_HANDLE_VALUE)
{
//::SetFilePointer(fileHandle, size * pickingVertexSize, NULL, FILE_BEGIN);
//::SetEndOfFile(fileHandle);
//mapLength = ::GetFileSize(fileHandle, NULL);
//mapLength = size * pickingVertexSize;
mapHandle = ::CreateFileMapping(fileHandle, NULL, PAGE_READWRITE, 0, size * pickingVertexSize, NULL);
mapLength = ::GetFileSize(fileHandle, NULL);
start = (Sphere*)::MapViewOfFile(mapHandle, FILE_MAP_WRITE, 0, 0, 0);
}
end = start + size;
Clear();
}
I would advice to only use Plain Old Data (POD) in your mapped file, because pointers can become invalid when mapping and unmapping files
In principle, you can use placement new operator (void* operator new (std::size_t size, void* ptr) throw();) to create an object in an already allocated memory buffer.
However, creating objects in file mapping regions should be applied for POD objects. Any raw pointer member (and virtual memory table if any) will not be recommended to be stored in mapped file, because they will be invalidated as soon as the mapping is done in a different address space. You should also take care to manage the memory allocation in this case.
I recommend to take a look at boost interprocess library (they have pretty good support for memory mapped files, including management of the memory buffers - allocation algorithms, object tracking and a pointer-like class for linking objects address-space independent (offset_ptr)).
I'm developing an app which will need to work with different types of archives. As many of the archive types as possible is good. I have choosen a 7zip.dll as an engine of archive-worker. But there is a problem, does anyone knows how to uncompress a file from archive to memory buffer? As I see, 7zip.dll supports only uncompressing to hard disk. Also, it would be nice to load archive from memory buffer. Has anyone tried to do something like that?
Not sure if I completely understand your needs (for example, don't you need the decompressed file on disk?).
I was looking at LZMA SDK 9.20 and its lzma.txt readme file, and there are plenty of hints that decompression to memory is possible - you may just need to use the C API rather than the C++ interface. Check out, for example, the section called Single-call Decompressing:
When to use: RAM->RAM decompressing
Compile files: LzmaDec.h + LzmaDec.c + Types.h
Compile defines: no defines
Memory Requirements:
- Input buffer: compressed size
- Output buffer: uncompressed size
- LZMA Internal Structures: state_size (16 KB for default settings)
Also, there is this function:
SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen,
const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status);
You can utilize these by memory-mapping the archive file. To the best of my knowledge, if your process creates a memory-mapped file with exclusive access (so no other process can access it) and does no explicit flushing, all changes to the file will be kept in memory until the mapping is destroyed or the file closed. Alternatively, you could just load the archive contents in memory.
For the sake of completeness, I hacked together several examples into a demo of using memory mapping in Windows.
#include <stdio.h>
#include <time.h>
#include <Windows.h>
#include <WinNT.h>
// This demo will limit the file to 4KiB
#define FILE_SIZE_MAX_LOWER_DW 4096
#define FILE_SIZE_MAX_UPPER_DW 0
#define MAP_OFFSET_LOWER_DW 0
#define MAP_OFFSET_UPPER_DW 0
#define TEST_ITERATIONS 1000
#define INT16_SIZE 2
typedef short int int16;
// NOTE: This will not work for Windows less than XP or 2003 Server!
int main()
{
HANDLE hFile, hFileMapping;
PBYTE mapViewStartAddress;
// Note: with no explicit security attributes, the process needs to have
// the necessary rights (e.g. read, write) to this location.
LPCSTR path = "C:\\Users\\mcmlxxxvi\\Desktop\\test.dat";
// First, open a file handle.
hFile = CreateFile(path,
GENERIC_READ | GENERIC_WRITE, // The file is created with Read/Write permissions
FILE_SHARE_READ, // Set this to 0 for exclusive access
NULL, // Optional security attributes
CREATE_ALWAYS, // File is created if not found, overwritten otherwise
FILE_ATTRIBUTE_TEMPORARY, // This affects the caching behaviour
0); // Attributes template, can be left NULL
if ((hFile) == INVALID_HANDLE_VALUE)
{
fprintf(stderr, "Unable to open file");
return 1;
}
// Then, create a memory mapping for the opened file.
hFileMapping = CreateFileMapping(hFile, // Handle for an opened file
NULL, // Optional security attributes
PAGE_READWRITE, // File can be mapped for Read/Write access
FILE_SIZE_MAX_UPPER_DW, // Maximum file size split in DWORDs.
FILE_SIZE_MAX_LOWER_DW, // NOTE: I may have these two mixed up!
NULL); // Optional name
if (hFileMapping == 0)
{
CloseHandle(hFile);
fprintf(stderr, "Unable to open file for mapping.");
return 1;
}
// Next, map a view (a continuous portion of the file) to a memory region
// The view must start and end at an offset that is a multiple of
// the allocation granularity (roughly speaking, the machine page size).
mapViewStartAddress = (PBYTE)MapViewOfFile(hFileMapping, // Handle to a memory-mapped file
FILE_MAP_READ | FILE_MAP_WRITE, // Maps the view for Read/Write access
MAP_OFFSET_UPPER_DW, // Offset in the file from which
MAP_OFFSET_LOWER_DW, // the view starts, split in DWORDs.
FILE_SIZE_MAX_LOWER_DW); // Size of the view (here, entire file)
if (mapViewStartAddress == 0)
{
CloseHandle(hFileMapping);
CloseHandle(hFile);
fprintf(stderr, "Couldn't map a view of the file.");
return 1;
}
// This is where actual business stuff belongs.
// This example application does iterations of reading and writing
// random numbers for the entire length of the file.
int16 value;
errno_t result = 0;
srand((int)time(NULL));
for (int i = 0; i < TEST_ITERATIONS; i++)
{
// Write
for (int j = 0; j < FILE_SIZE_MAX_LOWER_DW / INT16_SIZE; j++)
{
value = rand();
result = memcpy_s(mapViewStartAddress + j * INT16_SIZE, INT16_SIZE, &value, INT16_SIZE);
if (result != 0)
{
CloseHandle(hFileMapping);
CloseHandle(hFile);
fprintf(stderr, "File write error during iteration #%d, error %d", i, GetLastError());
return 1;
}
}
// Read
SetFilePointer(hFileMapping, 0, 0, FILE_BEGIN);
for (int j = 0; j < FILE_SIZE_MAX_LOWER_DW / sizeof(int); j++)
{
result = memcpy_s(&value, INT16_SIZE, mapViewStartAddress + j * INT16_SIZE, INT16_SIZE);
if (result != 0)
{
CloseHandle(hFileMapping);
CloseHandle(hFile);
fprintf(stderr, "File read error during iteration #%d, error %d", i, GetLastError());
return 1;
}
}
}
// End business stuff
CloseHandle(hFileMapping);
CloseHandle(hFile);
return 0;
}