Defining variable length structs and casting to them - d

In C, you sometimes see something like:
struct foobar
{
int size;
int data[1];
};
where the data member doesn't really have just one element; rather it's meant to be variable length.
If you do something like that in D, is it going to let you, for example, read myfoobar.data[4]?
I know D has variable length arrays, e.g. int[] myvarlenintarray;, but what if you're trying to interface with some code that already puts out a data structure in memory like the one above, and possibly much more complex than that? Let's say it's in the first portion of int[3000] buffer;. Is there an easy way to cast it to a usable struct without moving it in memory? If not, is there an easy way to get the data into a similar struct without having to manually parse out each member of the struct?
edit:
I think I need to give a practical example so you see where I'm coming from.
import std.c.windows.windows;
import std.utf;
import std.stdio;
public struct REPARSE_DATA_BUFFER
{
ULONG ReparseTag;
USHORT ReparseDataLength;
USHORT Reserved;
union
{
struct SymbolicLinkReparseBuffer
{
USHORT SubstituteNameOffset;
USHORT SubstituteNameLength;
USHORT PrintNameOffset;
USHORT PrintNameLength;
ULONG Flags;
WCHAR[1] PathBuffer;
}
SymbolicLinkReparseBuffer mySymbolicLinkReparseBuffer;
struct MountPointReparseBuffer
{
USHORT SubstituteNameOffset;
USHORT SubstituteNameLength;
USHORT PrintNameOffset;
USHORT PrintNameLength;
WCHAR[1] PathBuffer;
}
MountPointReparseBuffer myMountPointReparseBuffer;
struct GenericReparseBuffer
{
UCHAR[1] DataBuffer;
}
GenericReparseBuffer myGenericReparseBuffer;
}
}
alias REPARSE_DATA_BUFFER* PREPARSE_DATA_BUFFER;
enum MAXIMUM_REPARSE_DATA_BUFFER_SIZE = 16*1024;
// Values for 'ReparseTag' member of REPARSE_DATA_BUFFER:
enum : DWORD {
IO_REPARSE_TAG_SYMLINK = 0xA000000C,
IO_REPARSE_TAG_MOUNT_POINT = 0xA0000003 // which also defines a Junction Point
}
enum DWORD FSCTL_GET_REPARSE_POINT = 0x000900a8;
enum FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000;
public extern(Windows) BOOL function(HANDLE, DWORD, LPVOID, DWORD, LPVOID, DWORD, LPVOID, OVERLAPPED*) DeviceIoControl;
void main()
{
DeviceIoControl = cast(BOOL function(HANDLE, DWORD, LPVOID, DWORD, LPVOID, DWORD, LPVOID, OVERLAPPED*))GetProcAddress(LoadLibraryA("kernel32.dll"), "DeviceIoControl");
auto RPHandle = CreateFileW((r"J:\Documents and Settings").toUTF16z(), 0, FILE_SHARE_READ, null, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT + FILE_FLAG_BACKUP_SEMANTICS, null);
if (RPHandle == INVALID_HANDLE_VALUE)
{
printf("CreateFileW failed with error code %d.", GetLastError());
return;
}
BYTE[MAXIMUM_REPARSE_DATA_BUFFER_SIZE] reparsebuffer;
uint reparsedatasize;
auto getreparsepointresult = DeviceIoControl(RPHandle, FSCTL_GET_REPARSE_POINT, null, 0, cast(void*) reparsebuffer.ptr, MAXIMUM_REPARSE_DATA_BUFFER_SIZE, &reparsedatasize, null);
if (getreparsepointresult == 0)
{
printf("DeviceIoControl with FSCTL_GET_REPARSE_POINT failed with error code %d.", GetLastError());
return;
}
// Now what?
// If I do this:
auto ReparseDataPtr = cast(REPARSE_DATA_BUFFER*) reparsebuffer.ptr;
printf("%d == %d\n", reparsebuffer.ptr, ReparseDataPtr); // Alright, data hasn't been copied.
// But what good is a pointer? Can I use a pointer to a struct to access one of its members apart from dereferencing?
printf("%d == %d\n", &reparsebuffer[0], &(*ReparseDataPtr)); // Here, I dereference ReparseDataPtr, but nothing moves.
printf("%d == %d\n", &reparsebuffer[0], &((*ReparseDataPtr).ReparseTag)); // Same here, so I can access members in a roundabout way.
printf("%d == %d\n", &reparsebuffer[0], &(ReparseDataPtr.ReparseTag)); // And thanks to Jim's comment, here's a less roundabout way.
auto ReparseData = *ReparseDataPtr; // But if I assign a name to the dereferenced ReparseDataPtr,
printf("%d != %d\n", &reparsebuffer[0], &(ReparseData.ReparseTag)); // the data is copied to a new location, leaving most of PathBuffer behind.
REPARSE_DATA_BUFFER ReparseDataFn() {return *ReparseDataPtr;} // Similarly, this way
printf("%d != %d\n", &reparsebuffer[0], &(ReparseDataFn().ReparseTag)); // copies stuff to a new location.
}
Firstly, I don't understand why it's different for the case in which I don't give *ReparseDataPtr a name.
Secondly, is there no way to have a symbol whose type is REPARSE_DATA_BUFFER and whose data is located at reparsebuffer.ptr?

Have you tried doing the exact same thing in D as in C?
struct foobar { int size; int data[1]; };
It works... just use data.ptr instead of data to access the elements, because otherwise it will perform bounds checking with a length of 1.

You could access it via a helper method:
struct foobar
{
public:
int[] Data() { return data.ptr[0..size]; }
private:
int size;
int data[1];
}
You might also want to put int a static foreach over the members of foobar that uses static assert to make sure that the offset of each is less than the offset of data.

Related

DbgHelp: Wrong address of function parameter when it is a pass-by-value symbol on x64

I am using the Windows DbgHelp library to dump a callstack of my C++ application. The PDB is in the right place and I am successfully reporting the stack frames. I have an involved type traversal system that prints out all of the locals and this is mostly working. However, in some cases I am getting the wrong address and causing access violations. One case is when a function is passed an object by value:
struct Object { float x,y,z; }
void foo( Object objectByValue)
{
// dump callstack
}
In this case, the address calculated for objectByValue is wrong. It is near the correct place on the stack, but not exactly. I am having some difficulties finding information on the right address. I am doing the following:
Set the correct context with SymSetContext(...)
Invoke SymEnumSymbols with a callback
Inside the callback, check if SYMFLAG_REGREL is set
Assign Address = SymInfo->Address + context.rbp or context.rsp depending on
the SymInfo.Register value (CV_AMD64_RBP or CV_AMD64_RSP are only
ever present )
Then I use that address to access the variable.
For variables on the stack this address is correct, as it is for most other cases. However, it is not in some, including this case.
I have included a working example below with the following output:
main-> Address of on stack: 000000000020B0D0 = { 1.000000, 2.000000,
3.000000 }
foo-> Address of parameters: 000000000020D6F8 = { 1.000000, 2.000000, 3.000000 }
Print stack from bottom up:
Frame: foo Variable: objByValue offset=0xe0 address=0x20b090 size=8204
Frame: main Variable: objOnStack offset=0x10 address=0x20b0d0 size=8204
You can see in the example that the address calculate from the variable on the stack is correct, but for the pass by value it is wrong.
Does anyone have any insight on how I can correctly calculate this value?
#include "stdafx.h"
#include <windows.h>
#include <stdint.h>
#pragma comment(lib, "dbghelp.lib")
#pragma pack( push, before_imagehlp, 8 )
#include <imagehlp.h>
#pragma pack( pop, before_imagehlp )
// Normally it should be enough to use 'CONTEXT_FULL' (better would be 'CONTEXT_ALL')
#define USED_CONTEXT_FLAGS CONTEXT_FULL
#if defined(_M_AMD64)
const int ImageFileMachine = IMAGE_FILE_MACHINE_AMD64;
#else
const int ImageFileMachine = IMAGE_FILE_MACHINE_I386;
#endif
struct BaseAddresses
{
uint64_t Rsp;
uint64_t Rbp;
};
const int C_X64_REGISTER_RBP = 334; // Frame Base Pointer register
const int C_X64_REGISTER_RSP = 335; // Stack Pointer register (common in release builds with frame pointer removal)
BOOL EnumSymbolsCallback(PSYMBOL_INFO pSymInfo, ULONG SymbolSize, PVOID UserContext)
{
BaseAddresses * pBaseAddresses = (BaseAddresses*)UserContext;
ULONG64 base = 0;
if ((pSymInfo->Flags & SYMFLAG_REGREL) != 0)
{
switch (pSymInfo->Register)
{
case C_X64_REGISTER_RBP:
base = (ULONG64)pBaseAddresses->Rbp;
break;
case C_X64_REGISTER_RSP:
base = (ULONG64)pBaseAddresses->Rsp;
break;
default:
exit(0);
}
}
ULONG64 address = base + pSymInfo->Address;
printf("Variable: %s offset=0x%llx address=0x%llx size=%lu\n", pSymInfo->Name, pSymInfo->Address, address, pSymInfo->Size);
return TRUE;
}
DWORD DumpStackTrace()
{
HANDLE mProcess = GetCurrentProcess();
HANDLE mThread = GetCurrentThread();
if (!SymInitialize(mProcess, NULL, TRUE)) // load symbols, invasive
return 0;
CONTEXT c;
memset(&c, 0, sizeof(CONTEXT));
c.ContextFlags = USED_CONTEXT_FLAGS;
RtlCaptureContext(&c);
// SYMBOL_INFO & buffer storage
char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)];
PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)buffer;
STACKFRAME64 frame;
memset(&frame, 0, sizeof(STACKFRAME64));
DWORD64 displacement_from_symbol = 0;
printf("Print stack from bottom up:\n");
int framesToSkip = 1; // skip reporting this frame
do
{
// Get next stack frame
if (!StackWalk64(ImageFileMachine, mProcess, mThread, &frame, &c, nullptr, SymFunctionTableAccess64, SymGetModuleBase64, nullptr))
{
break;
}
// Lookup symbol name using the address
pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO);
pSymbol->MaxNameLen = MAX_SYM_NAME;
if (!SymFromAddr(mProcess, (ULONG64)frame.AddrPC.Offset, &displacement_from_symbol, pSymbol))
return false;
if (framesToSkip > 0)
{
framesToSkip--;
continue;
}
printf("Frame: %s\n", pSymbol->Name);
// Setup the context to get to the parameters
IMAGEHLP_STACK_FRAME imSFrame = { 0 };
imSFrame.InstructionOffset = frame.AddrPC.Offset;
if (!SymSetContext(mProcess, &imSFrame, NULL))
return false;
BaseAddresses addresses;
addresses.Rbp = c.Rbp;
addresses.Rsp = c.Rsp;
if (!SymEnumSymbols(mProcess, 0, 0, EnumSymbolsCallback, &addresses))
{
return false;
}
if (strcmp(pSymbol->Name, "main") == 0)
break;
} while (frame.AddrReturn.Offset != 0);
SymCleanup(mProcess);
return 0;
}
struct Structure
{
float x, y, z;
};
void foo(Structure objByValue)
{
printf("foo-> Address of parameters: %p = { %f, %f, %f }\n", &objByValue, objByValue.x, objByValue.y, objByValue.z);
DumpStackTrace();
}
int main()
{
Structure objOnStack = { 1, 2, 3 };
printf("main-> Address of on stack: %p = { %f, %f, %f }\n", &objOnStack, objOnStack.x, objOnStack.y, objOnStack.z);
foo(objOnStack);
return 0;
}
After reading the documentation on X64 calling convention I discovered the following sentence:
Any argument that doesn’t fit in 8 bytes, or isn't 1, 2, 4, or 8
bytes, must be passed by reference 1
So that explains the funny address - what the debug symbols are giving me is the address of the memory storing the reference to the full data. So my code flow essentially says:
if ( symbol is a parameter and the size > 8 )
address = *(uint64_t)address; // dereference
And then passing that address through my type system resolves correctly.
I thought other people might find this useful as it is not documented anywhere in the DbgHelp library, and while some people might understand the calling conventions, it didn't occur to me that the symbol data passed back wouldn't contain something helpful to indicate this.

Problem with SDK pointers - c++ to Delphi translating

I'm struggling with HikVision SDK, trying to get it work in Delphi. There's some samples in C++, but I've stuck with pointers issues while translating to Delphi. Till that moment code seems to work, as I get correct plate number.
Is this correct equivalent: BYTE *pBuffer -> PByte ?
How to extract picture from pBuffer?
C++ code:
typedef struct tagNET_ITS_PICTURE_INFO
{
DWORD dwDataLen; //Media data length
BYTE byType;
BYTE byAbsTime[32];
NET_VCA_RECT struPlateRect;
BYTE *pBuffer; //Data pointer
DWORD dwUTCTime;
//...
}NET_ITS_PICTURE_INFO, *LPNET_ITS_PICTURE_INFO;
typedef struct tagNET_ITS_PLATE_RESULT
{
//...
NET_DVR_PLATE_INFO struPlateInfo;
NET_ITS_PICTURE_INFO struPicInfo[6];
}NET_ITS_PLATE_RESULT, *LPNET_ITS_PLATE_RESULT;
void CALLBACK MSesGCallback(LONG lCommand, NET_DVR_ALARMER *pAlarmer, char *pAlarmInfo, DWORD dwBufLen, void* pUser)
{
char filename[100];
FILE *fSnapPic = NULL;
switch (lCommand) {
case COMM_ITS_PLATE_RESULT: {
NET_ITS_PLATE_RESULT struITSPlateResult = { 0 };
memcpy(&struITSPlateResult, pAlarmInfo, sizeof(struITSPlateResult));
printf("Plate: %s\n", struITSPlateResult.struPlateInfo.sLicense);
if (struITSPlateResult.struPicInfo[0].dwDataLen != 0)
{
sprintf(filename, "./pic/%d.jpg", 0);
fSnapPic = fopen(filename, "wb");
fwrite(struITSPlateResult.struPicInfo[0].pBuffer, struITSPlateResult.struPicInfo[0].dwDataLen, 1, fSnapPic);
fclose(fSnapPic);
}
}
}
return;
}
Delphi code:
LPNET_ITS_PICTURE_INFO = ^NET_ITS_PICTURE_INFO;
NET_ITS_PICTURE_INFO = record
dwDataLen: DWORD;
byType: BYTE;
byAbsTime: array [0..31] of BYTE;
struPlateRect: NET_VCA_RECT;
pBuffer: PByte; // ????????
dwUTCTime: DWORD;
//(...)
end;
LPNET_ITS_PLATE_RESULT = ^NET_ITS_PLATE_RESULT;
NET_ITS_PLATE_RESULT = record
//(...)
struPlateInfo: NET_DVR_PLATE_INFO;
struPicInfo: Array [0..5] of NET_ITS_PICTURE_INFO;
end;
procedure MessageCallBack(lCommand:longint; pAlarmer:LPNET_DVR_ALARMER; pAlarmInfo:PAnsiChar; dwBufLen:LongInt; pUser:pointer); stdcall;
var
struAlarmInfo: LPNET_DVR_ALARMINFO_V30;
struPlateResult: LPNET_ITS_PLATE_RESULT;
begin
case lCommand of
COMM_ITS_PLATE_RESULT:
begin
New(struPlateResult);
FillChar(struPlateResult^, SizeOf(NET_ITS_PLATE_RESULT), 0);
Move(pAlarmInfo^, struPlateResult^, Sizeof(NET_ITS_PLATE_RESULT));
Form1.memoOut.Lines.Add('sLicense: ' + struPlateResult.struPlateInfo.sLicense);
if (struPlateResult.struPicInfo[0].dwDataLen >0) then
??? how to get the picture from struPlateResult.struPicInfo[0].pBuffer ?
end;
end;
end;
Regards
Yes, PByte or PByteArray is correct translation.
If buffer contains valid file contents, you can save it into file:
FS := TFileStream.Create('test.jpg', fmCreate);
try
FS.Write(struITSPlateResult.struPicInfo[0].pBuffer^,
struITSPlateResult.struPicInfo[0].dwDataLen);
finally
FS.Free;
end;

How to tell if a heap is serialized or not by its HANDLE?

Say, if I call GetProcessHeaps in my process to get a list of heaps that it uses. Having a heap HANDLE how can I tell if such heap was created with the HEAP_NO_SERIALIZE flag or not?
While not a definite answer, you can call HeapQueryInformation(HeapCompatibilityInformation) and if it returns 2 then it is serialized because MSDN says this about HEAP_NO_SERIALIZE:
The low-fragmentation heap (LFH) cannot be enabled for a heap created with this option
I don't know if there is even a undocumented API to get the flags but for debugging purposes you can access the internal heap structure directly:
void DumpHeapType_Win8_x86(HANDLE hHeap)
{
typedef struct {
UINT32 Unknown1[2];
UINT32 Sig;
UINT32 Unknown2[1];
void*Unknown3[2]; //LIST_ENTRY?
void*Unknown4[1+1+1+1+2];
UINT32 Unknown5[1+1+1+1];
UINT32 Flags;
} HEAP_HDR;
typedef struct {
UINT32 Unknown1[2];
UINT32 Sig;
UINT32 Unknown2[1];
void*Unknown3[2]; //LIST_ENTRY?
HEAP_HDR*pHdr;
} HEAP_THING;
HEAP_THING *pThing = (HEAP_THING*) hHeap;
if (hHeap && pThing->Sig == 0xffeeffee)
{
HEAP_HDR *pHdr = (HEAP_HDR*) pThing->pHdr;
if (pHdr->Sig == 0xffeeffee)
{
printf("Flags=%#x Serialized=%d\n", pHdr->Flags, !(pHdr->Flags & HEAP_NO_SERIALIZE));
}
}
}
void playwithheaps()
{
HANDLE hHeap;
DumpHeapType_Win8_x86(hHeap = GetProcessHeap());
DumpHeapType_Win8_x86(hHeap = HeapCreate(0, 0, 0)); if (hHeap) HeapDestroy(hHeap);
DumpHeapType_Win8_x86(hHeap = HeapCreate(HEAP_NO_SERIALIZE, 0, 0)); if (hHeap) HeapDestroy(hHeap);
DumpHeapType_Win8_x86(hHeap = HeapCreate(HEAP_NO_SERIALIZE|HEAP_GENERATE_EXCEPTIONS, 0, 0)); if (hHeap) HeapDestroy(hHeap);
}
On my Windows 8 machine this gives me the following output:
Flags=0x2 Serialized=1
Flags=0x1002 Serialized=1
Flags=0x1003 Serialized=0
Flags=0x1007 Serialized=0
but the heap structure layout might be different on other versions so you would just have to test carefully...

Attributes of a structure receive multiple structures

Is it possible for an attribute of a 'struct' to host multiple structures?
Example, I need the attribute LPWFSPINFDK lppFDKs; which is part of the struct _wfs_pin_func_key_detail, receive multiple structs _wfs_pin_fdk.
I'm trying this way, compiles, but the final program does not recognize:
WFSPINFUNCKEYDETAIL PinFunKeyDetail;
WFSPINFDK ObjPinKey;
LPWFSPINFDK PinKey;
PinKey = &ObjPinKey;
PinKey->ulFDK = WFS_PIN_FK_FDK01;
PinKey->usXPosition = 5;
PinKey->usYPosition = 5;
PinFunKeyDetail.lppFDKs = &PinKey;
STRUCT: _wfs_pin_fdk
typedef struct _wfs_pin_fdk
{
ULONG ulFDK;
USHORT usXPosition;
USHORT usYPosition;
} WFSPINFDK, * LPWFSPINFDK;
STRUCT: _wfs_pin_func_key_detail
typedef struct _wfs_pin_func_key_detail
{
ULONG ulFuncMask;
USHORT usNumberFDKs;
LPWFSPINFDK * lppFDKs; //I want to receive the structs here
} WFSPINFUNCKEYDETAIL, * LPWFSPINFUNCKEYDETAIL;

Why I got crash when allocating buffer?

I am developing a driver with Filter. So when I write the SendNetBufferListsComplete function in filter.cpp I got a crash (bluescreen). WinDbug pointed to some buffer allocation. The code is here:
Edited:
sendNetBufferListsComplete(
IN PNET_BUFFER_LIST NetBufferLists,
IN ULONG SendCompleteFlags) {
PNET_BUFFER_LIST pNetBufferList = NetBufferLists;
PNET_BUFFER_LIST pNextNetBufferList = NULL;
while (pNetBufferList)
{
pNextNetBufferList = NET_BUFFER_LIST_NEXT_NBL(pNetBufferList);
NET_BUFFER_LIST_NEXT_NBL(pNetBufferList) = NULL;
PNET_BUFFER_LIST pParentNetBufferList = pNetBufferList->ParentNetBufferList;
if (pParentNetBufferList != NULL)
{
NDIS_STATUS status = NET_BUFFER_LIST_STATUS(pNetBufferList);
NdisFreeNetBufferList(pNetBufferList);
if (NdisInterlockedDecrement(&pParentNetBufferList->ChildRefCount) == 0) {
NET_BUFFER_LIST_STATUS(pParentNetBufferList) = status;
NdisFSendNetBufferListsComplete(m_hFilter, pParentNetBufferList, SendCompleteFlags);
}
}
else
{
if(pNetBufferList != NULL)
{
**---windbug pointed here---****
PVOID pBuffer = *(PVOID*) NET_BUFFER_LIST_CONTEXT_DATA_START(pNetBufferList);
PMDL pMdl = NET_BUFFER_FIRST_MDL(NET_BUFFER_LIST_FIRST_NB(pNetBufferList));
if(pMdl)
NdisFreeMdl(pMdl);
if(pBuffer)
delete[] (UCHAR*) pBuffer;
NdisFreeNetBufferList(pNetBufferList);
}
}
NdisInterlockedDecrement(&m_nSendNetBufferListCount);
pNetBufferList = pNextNetBufferList;
}
What is the actual problem? Is it overflow? Or NULL check problems?
In ndish.h
#define NET_BUFFER_LIST_CONTEXT_DATA_START(_NBL) ((PUCHAR)(((_NBL)->Context)+1)+(_NBL)->Context->Offset)
like this . and in Wdm.h
//
// I/O system definitions.
//
// Define a Memory Descriptor List (MDL)
//
// An MDL describes pages in a virtual buffer in terms of physical pages. The
// pages associated with the buffer are described in an array that is allocated
// just after the MDL header structure itself.
//
typedef
_Struct_size_bytes_(_Inexpressible_(sizeof(struct _MDL) + // 747934
(ByteOffset + ByteCount + PAGE_SIZE-1) / PAGE_SIZE * sizeof(PFN_NUMBER)))
struct _MDL {
struct _MDL *Next;
CSHORT Size;
CSHORT MdlFlags;
struct _EPROCESS *Process;
PVOID MappedSystemVa; /* see creators for field size annotations. */
PVOID StartVa; /* see creators for validity; could be address 0. */
ULONG ByteCount;
ULONG ByteOffset;
} MDL, *PMDL;