Hooking/Detouring d3d9 (Present/EndScene) - Seems to call my function then crashes - c++

Question

As the title says, i'm trying to hook DirectX 9 V-Table and show some information on screen, I've been studying detours and hooking for a couple days now and i thought i understood it to a fair extent, but now i'm not too sure how to debug this issue.
My hook is calling another function ShowMsg(), which will eventually be a draw function, but for now it just shows a message-box.
My ShowMsg() function is called, but then the program is crashing.
I'm using the DirectX Feb 2010 "SimpleSample" application.
Here's my code, the commented parts are from a previous test at hooking a function from another test application.
// dllmain.cpp : Defines the entry point for the DLL application.
#include "stdafx.h"
#include <iostream>
#include <Windows.h>
#include "VirtualTable.h"
#include <d3d9.h>
#pragma comment(lib, "d3d9.lib")
#include <d3dx9.h>
#pragma comment(lib, "d3dx9.lib")
#include <detours.h>
#pragma comment(lib, "detours.lib")
using namespace std;
typedef void(__thiscall* Present)(IDirect3DDevice9* device);
Present g_org_Present;
void ShowMsg()
{
MessageBoxA(0, "This function was called.", "", 0);
}
void __fastcall hk_Present(IDirect3DDevice9* device)
{
ShowMsg();
//call the original function
g_org_Present(device);
}
void InitiateHooks()
{
HWND game_window = FindWindowA(NULL, "SimpleSample");
auto d3dpp = D3DPRESENT_PARAMETERS{};
auto d3d = Direct3DCreate9(D3D_SDK_VERSION);
IDirect3DDevice9* device;
d3dpp.BackBufferCount = 1;
d3dpp.MultiSampleType = D3DMULTISAMPLE_NONE;
d3dpp.SwapEffect = D3DSWAPEFFECT_DISCARD;
d3dpp.hDeviceWindow = game_window;
d3dpp.FullScreen_RefreshRateInHz = D3DPRESENT_RATE_DEFAULT;
d3dpp.PresentationInterval = D3DPRESENT_INTERVAL_IMMEDIATE;
d3dpp.BackBufferFormat = D3DFMT_R5G6B5;
d3dpp.Windowed = TRUE;
if (SUCCEEDED(d3d->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, game_window, D3DCREATE_HARDWARE_VERTEXPROCESSING, &d3dpp, &device)))
{
void** vmt = *(void***)device;
DWORD oldProtection;
VirtualProtect(&vmt[17], 4, PAGE_EXECUTE_READWRITE, &oldProtection);
g_org_Present = (Present)vmt[17];
vmt[17] = &hk_Present;
VirtualProtect(&vmt[17], 4, oldProtection, 0);
device->Present(NULL, NULL, NULL, NULL);
}
// VirtualTable* myTable = new VirtualTable();
//get the pointer to the actual virtual method table from our pointer to our class instance
// void** base = *(void***)myTable;
// DWORD oldProtection;
//one way to remove page protection(not the best but this is an example only)
// VirtualProtect(&base[1], 4, PAGE_EXECUTE_READWRITE, &oldProtection);
//save the original function
// g_org_VirtualFunction01 = (VirtualFunction01_t)base[1];
//overwrite
// base[1] = &hk_VirtualFunction01;
//restore page protection
// VirtualProtect(&base[1], 4, oldProtection, 0);
//call the virtual function (now hooked) from our class instance
// myTable->VirtualFunction01();
}
#pragma endregion
BOOL APIENTRY DllMain( HMODULE hModule,
DWORD ul_reason_for_call,
LPVOID lpReserved
)
{
switch (ul_reason_for_call)
{
case DLL_PROCESS_ATTACH:
CreateThread(0, 0x1000, (LPTHREAD_START_ROUTINE)InitiateHooks, 0, 0, NULL);
break;
case DLL_THREAD_ATTACH:
case DLL_THREAD_DETACH:
case DLL_PROCESS_DETACH:
break;
}
return TRUE;
}
Can someone please give me an idea of what i might be doing wrong here, so i can fix this, and for future reference.
Here's updated code, i realised i shouldn't be calling the function directly so i changed that, also changed it to try hook/detour EndScene, also using MS Detours instead of the other method which i think is V-Table patching, it seems like my EndScene hook is being called, as the MessageBox is continuously called. However, the program still crashes if i comment out the MessageBox.
#include "stdafx.h"
#include <iostream>
#include <Windows.h>
#include <intrin.h>
#include <tchar.h>
#include <tlhelp32.h>
#include <Psapi.h>
#include <winsock2.h>
#include <vector>
#include <ws2tcpip.h>
#pragma comment( lib, "Ws2_32.lib" )
#include <d3d9.h>
#pragma comment(lib, "d3d9.lib")
#include <d3dx9.h>
#pragma comment(lib, "d3dx9.lib")
#include <detours.h>
#pragma comment(lib, "detours.lib")
using namespace std;
D3DCOLOR RED = D3DCOLOR_ARGB(255, 255, 0, 0);
typedef HRESULT(__stdcall* EndScene) (IDirect3DDevice9*);
EndScene EndScene_orig;
HRESULT __stdcall EndScene_hook(IDirect3DDevice9* pDevice)
{
// D3DRECT rec = { 100,100,200,200 };
// pDevice->Clear(1, &rec, D3DCLEAR_TARGET, RED, 0, 0);
// MessageBoxA(0, "We made it here...2", "", 0); // <<<<----- This function is called over and over when not commented.
return EndScene_orig(pDevice);
}
void InitHook()
{
HWND game_window = FindWindow(NULL, _T("Skinned Mesh"));
auto d3dpp = D3DPRESENT_PARAMETERS{};
auto d3d = Direct3DCreate9(D3D_SDK_VERSION);
if (d3d)
{
d3dpp.BackBufferCount = 1;
d3dpp.MultiSampleType = D3DMULTISAMPLE_NONE;
d3dpp.SwapEffect = D3DSWAPEFFECT_DISCARD;
d3dpp.hDeviceWindow = game_window;
d3dpp.FullScreen_RefreshRateInHz = D3DPRESENT_RATE_DEFAULT;
d3dpp.PresentationInterval = D3DPRESENT_INTERVAL_IMMEDIATE;
d3dpp.BackBufferFormat = D3DFMT_R5G6B5;
d3dpp.Windowed = TRUE;
IDirect3DDevice9* Device;
if (SUCCEEDED(d3d->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, game_window, D3DCREATE_HARDWARE_VERTEXPROCESSING, &d3dpp, &Device)))
{
// MessageBoxA(0, "We made it here...", "", 0);
DWORD* pVTable = *reinterpret_cast<DWORD**>(Device);
DetourTransactionBegin();
DetourUpdateThread(GetCurrentThread());
EndScene_orig = (EndScene)pVTable[42];
DetourAttach(&(LPVOID&)pVTable[42], (PBYTE)EndScene_hook);
DetourTransactionCommit();
}
}
}
void SetupConsole()
{
AllocConsole();
freopen("CONOUT$", "wb", stdout);
freopen("CONOUT$", "wb", stderr);
freopen("CONIN$", "rb", stdin);
SetConsoleTitle("CSGOHAX");
}
BOOL APIENTRY DllMain(HMODULE hModule,
DWORD ul_reason_for_call,
LPVOID lpReserved
)
{
switch (ul_reason_for_call)
{
case DLL_PROCESS_ATTACH:
SetupConsole();
CreateThread(0, 0, (LPTHREAD_START_ROUTINE)InitHook, 0, 0, NULL);
break;
case DLL_THREAD_ATTACH:
case DLL_THREAD_DETACH:
case DLL_PROCESS_DETACH:
break;
}
return TRUE;
}
It has to be something simple that i just can't see as a problem..

Answer 1

I figured it out, reversed the binary in IDA, found the module address and the EndScene function aswell as its address, calculated the offset. Then used ollydbg and found the function again, made a signature from it, now i can find it dynamically using a signature scanning function.
So i can get the function address with this signature.
DWORD dwEndScene = FindPattern("d3d9.dll",
"\x6A\x18\xB8\x00\x00\x00\x00\xE8\x00\x00\x00\x00\x8B\x7D\x08\x8B\xDF\x8D\x47\x04\xF7\xDB\x1B\xDB\x23\xD8\x89\x5D\xE0\x33\xF6\x89\x75\xE4\x39\x73\x18\x75\x73",
"xxx????x????xxxxxxxxxxxxxxxxxxxxxxxxxxx");
Then i just detour the function
DetourTransactionBegin();
DetourUpdateThread(GetCurrentThread());
EndScene_orig = (oEndScene)(dwEndScene);
DetourAttach(&(LPVOID&)EndScene_orig, EndScene_hook);
This is much easier than trying to find the function in the V-Table using a dummy device as i was before.