Why does SymInitialize() invoke CreateFile()? - c++

Question

Firstly, I want to hook CreateFile() and rewrite it. Then I want to recode the callstack of my new CreateFile() function. But when I use SymInitialize() to Initialize a handle, it falls into an endless loop. Through my debug, the reason is SymInitialize() invokes CreateFile(). So why does SymInitialize() involve CreateFile()? How to avoid this loop? Is there any alternative method to record callstack information to avoid this loop?
#include <windows.h>
#include <stdio.h>
#include "detours.h"
#include <fstream>
#include <io.h>
#pragma comment(lib, "detours.lib")
#include <DbgHelp.h> //SymInitialize
#pragma comment(lib,"dbghelp.lib")
#define STACK_INFO_LEN 200
struct stackInfo {
PDWORD hashValue; // hash value to identify same stack
char* szBriefInfo; // callstack info
};
stackInfo ShowTraceStack(char* szBriefInfo)
{
static const int MAX_STACK_FRAMES = 12;
void* pStack[MAX_STACK_FRAMES];
static char szStackInfo[STACK_INFO_LEN * MAX_STACK_FRAMES];
static char szFrameInfo[STACK_INFO_LEN];
HANDLE process = GetCurrentProcess(); // The handle used must be unique to avoid sharing a session with another component,
SymInitialize(process, NULL, TRUE);
PDWORD hashValue = (PDWORD)malloc(sizeof(DWORD)); // allow memory for hashVavlue, it will be rewrited in function CaptureStackBackTrace
WORD frames = CaptureStackBackTrace(0, MAX_STACK_FRAMES, pStack, hashValue);
//printf("hash value is: %ud \n", &hashValue);
if (szBriefInfo == NULL) {
strcpy_s(szStackInfo, "stack traceback:\n");
}
else {
strcpy_s(szStackInfo, szBriefInfo);
}
for (WORD i = 0; i < frames; ++i) {
DWORD64 address = (DWORD64)(pStack[i]);
DWORD64 displacementSym = 0;
char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)];
PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)buffer;
pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO);
pSymbol->MaxNameLen = MAX_SYM_NAME;
DWORD displacementLine = 0;
IMAGEHLP_LINE64 line;
line.SizeOfStruct = sizeof(IMAGEHLP_LINE64);
if (SymFromAddr(process, address, &displacementSym, pSymbol) &&
SymGetLineFromAddr64(process, address, &displacementLine, &line))
{
_snprintf_s(szFrameInfo, sizeof(szFrameInfo), "\t%s() at %s:%d(0x%x)\n",
pSymbol->Name, line.FileName, line.LineNumber, pSymbol->Address);
}
else
{
_snprintf_s(szFrameInfo, sizeof(szFrameInfo), "\terror: %d\n", GetLastError());
}
strcat_s(szStackInfo, szFrameInfo);
}
stackInfo traceStackInfo;
traceStackInfo.hashValue = hashValue;
traceStackInfo.szBriefInfo = szStackInfo;
return traceStackInfo;
//printf("%s", szStackInfo);
}
HANDLE (*__stdcall oldCreateFile)(LPCWSTR,
DWORD,
DWORD,
LPSECURITY_ATTRIBUTES,
DWORD,
DWORD,
HANDLE) = CreateFileW;
HANDLE WINAPI newCreateFile(
_In_ LPCWSTR lpFileName,
_In_ DWORD dwDesiredAccess,
_In_ DWORD dwShareMode,
_In_opt_ LPSECURITY_ATTRIBUTES lpSecurityAttributes,
_In_ DWORD dwCreationDisposition,
_In_ DWORD dwFlagsAndAttributes,
_In_opt_ HANDLE hTemplateFile
) {
ShowTraceStack((char*)"trace information.\n");
return oldCreateFile(
L".\\newFiles.txt", // L".\\NewFile.txt", // Filename
//lpFileName,
dwDesiredAccess, // Desired access
dwShareMode, // Share mode
lpSecurityAttributes, // Security attributes
dwCreationDisposition, // Creates a new file, only if it doesn't already exist
dwFlagsAndAttributes, // Flags and attributes
NULL);
}
void hook() {
DetourRestoreAfterWith();
DetourTransactionBegin();
DetourUpdateThread(GetCurrentThread());
DetourAttach(&(PVOID&)oldCreateFile, newCreateFile);
DetourTransactionCommit();
}
void unhook()
{
DetourTransactionBegin();
DetourUpdateThread(GetCurrentThread());
DetourDetach(&(PVOID&)oldCreateFile, newCreateFile);
DetourTransactionCommit();
}
void myProcess() {
HANDLE hFile = CreateFile(TEXT(".\\CreateFileDemo.txt"),
GENERIC_WRITE | GENERIC_READ,
0,
NULL,
CREATE_ALWAYS,
FILE_ATTRIBUTE_NORMAL, NULL);
if (hFile == INVALID_HANDLE_VALUE)
{
OutputDebugString(TEXT("CreateFile fail!\r\n"));
}
// write to file
const int BUFSIZE = 4096;
char chBuffer[BUFSIZE];
memcpy(chBuffer, "Test", 4);
DWORD dwWritenSize = 0;
BOOL bRet = WriteFile(hFile, chBuffer, 4, &dwWritenSize, NULL);
if (bRet) {
OutputDebugString(TEXT("WriteFile success!\r\n"));
}
}
int main(){
hook();
myProcess();
unhook();
}

Answer 1

The main problem is the call to SymInitialize where you pass through "TRUE" for fInvadeProcess parameter. This is causing it to SymLoadModuleEx to be called for each loaded module. This will cause a lot of file access to download / create / open PDB files for each loaded module. This is the reason for your infinite loop.
The "quick" fix for this sample is to move the call to SymInitialize into your main before the hook call as it only needs to be called once. This means all the PDB modules are loaded before the hooking / call to ShowTraceStack.
The other problems are:
dbghelp API is NOT thread safe - so this example will not work in a multi-threaded application
SymFromAddr may call CreateFile as well for the same reason to load a newly loaded module PDB information - so your hook not passing through the filename will cause PDB information to not work
If you are trying to make someone more useful I would:
Move SymInitialize to main before the hooking (called only once)
Only call CaptureStackBackTrace in the hook and queue the thread stack information to be processed at a later time
Create a separate thread the takes the CaptureStackBackTrace stack information output and convert it to a stack trace - this would is the only thread calling the dbghlp API making calls to dbghlp API thread safe
In your hook detect when being called from the dbghlp API usage thread and don't do the stack trace and don't modify the CreateFile parameters so you don't get into a infinite loop