D3D9 Hooking (EndScene + DrawIndexedPrimitive) - c++

Question

As none of you already know, I am quite fed up with DirectX at the moment. I have tried and tried, and no matter what I do, I cannot seem to get any hooks to work. I have scoured the web and studied hooks; I have combined bits of code together to scrap together my own hook; I have flat out plagiarized hooks to find out how they worked; I have even written a few from scratch. However, I cannot seem to get one to work. I'm trying to make a simple D3D mod-menu for CrossFire. What I have tried:
Hooking via VTable [had issues with getting a device pointer]
Hooking via pattern + mask scanning, etc and detours [unsure how to get a pattern, and cannot find one that works reliably for win10]
Creating a dummy device to get the addresses, etc, etc [caused an immediate shut-down of the game (detected)]
No matter what I do, the menu either flat out refuses to appear once programmed into the detoured EndScene, I get shut-down, I crash, or nothing happens.
Are there any good starter materials with sample code that I can learn from, as well as get this off the ground?
I already have the hack menu programmed, the variables set, the features programmed, the DllMain, the dependencies, you name it. All I need is to get a proper hook working - the only one I got to work had a weird bug where text drawing in EndScene & wall-hack in DrawIndexedPrimitive didn't work.

Answer 1

This is a large amount of code for a SO answer but here we go.
Get correct window
Create a dummy device
Get address of endscene from vtable
Do a standard x86 trampoline hook
Inside hook do whatever initialize stuff you need to
Get a copy of the correct device pointer
Draw stuff
call original function with correct device pointer
#pragma comment(lib, "d3d9.lib")
#pragma comment(lib, "d3dx9.lib")
#include <windows.h>
#include <d3d9.h>
#include <d3dx9.h>
bool Hook(char* src, char* dst, int len)
{
if (len < 5) return false;
DWORD curProtection;
VirtualProtect(src, len, PAGE_EXECUTE_READWRITE, &curProtection);
memset(src, 0x90, len);
uintptr_t relativeAddress = (uintptr_t)(dst - src - 5);
*src = (char)0xE9;
*(uintptr_t*)(src + 1) = (uintptr_t)relativeAddress;
DWORD temp;
VirtualProtect(src, len, curProtection, &temp);
return true;
}
char* TrampHook(char* src, char* dst, unsigned int len)
{
if (len < 5) return 0;
// Create the gateway (len + 5 for the overwritten bytes + the jmp)
char* gateway = (char*)VirtualAlloc(0, len + 5, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE);
// Put the bytes that will be overwritten in the gateway
memcpy(gateway, src, len);
// Get the gateway to destination addy
uintptr_t gateJmpAddy = (uintptr_t)(src - gateway - 5);
// Add the jmp opcode to the end of the gateway
*(gateway + len) = (char)0xE9;
// Add the address to the jmp
*(uintptr_t*)(gateway + len + 1) = gateJmpAddy;
// Place the hook at the destination
if (Hook(src, dst, len))
{
return gateway;
}
else return nullptr;
}
typedef HRESULT(APIENTRY* tEndScene)(LPDIRECT3DDEVICE9 pDevice);
static HWND window;
BOOL CALLBACK EnumWindowsCallback(HWND handle, LPARAM lParam)
{
DWORD wndProcId;
GetWindowThreadProcessId(handle, &wndProcId);
if (GetCurrentProcessId() != wndProcId)
return TRUE; // skip to next window
window = handle;
return FALSE; // window found abort search
}
HWND GetProcessWindow()
{
window = NULL;
EnumWindows(EnumWindowsCallback, NULL);
return window;
}
bool GetD3D9Device(void** pTable, size_t Size)
{
if (!pTable)
return false;
IDirect3D9* pD3D = Direct3DCreate9(D3D_SDK_VERSION);
if (!pD3D)
return false;
IDirect3DDevice9* pDummyDevice = NULL;
// options to create dummy device
D3DPRESENT_PARAMETERS d3dpp = {};
d3dpp.Windowed = false;
d3dpp.SwapEffect = D3DSWAPEFFECT_DISCARD;
d3dpp.hDeviceWindow = GetProcessWindow();
HRESULT dummyDeviceCreated = pD3D->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, d3dpp.hDeviceWindow, D3DCREATE_SOFTWARE_VERTEXPROCESSING, &d3dpp, &pDummyDevice);
if (dummyDeviceCreated != S_OK)
{
// may fail in windowed fullscreen mode, trying again with windowed mode
d3dpp.Windowed = !d3dpp.Windowed;
dummyDeviceCreated = pD3D->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, d3dpp.hDeviceWindow, D3DCREATE_SOFTWARE_VERTEXPROCESSING, &d3dpp, &pDummyDevice);
if (dummyDeviceCreated != S_OK)
{
pD3D->Release();
return false;
}
}
memcpy(pTable, *reinterpret_cast<void***>(pDummyDevice), Size);
pDummyDevice->Release();
pD3D->Release();
return true;
}
void DrawFilledRect(int x, int y, int w, int h, D3DCOLOR color, IDirect3DDevice9* dev)
{
D3DRECT BarRect = { x, y, x + w, y + h };
dev->Clear(1, &BarRect, D3DCLEAR_TARGET | D3DCLEAR_TARGET, color, 0, 0);
}
bool bInit = false;
tEndScene oEndScene = nullptr;
LPDIRECT3DDEVICE9 pD3DDevice = nullptr;
void* d3d9Device[119];
HRESULT APIENTRY hkEndScene(LPDIRECT3DDEVICE9 pDevice)
{
if (bInit == false)
{
pD3DDevice = pDevice;
bInit = true;
}
//draw stuff here like so:
DrawFilledRect(200, 200, 200, 200, D3DCOLOR_ARGB(255, 255, 0, 0), pDevice);
return oEndScene(pDevice);
}
DWORD WINAPI Init(HMODULE hModule)
{
if (GetD3D9Device(d3d9Device, sizeof(d3d9Device)))
{
oEndScene = (tEndScene)TrampHook((char*)d3d9Device[42], (char*)hkEndScene, 7);
}
return 0;
}
BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved)
{
switch (ul_reason_for_call)
{
case DLL_PROCESS_ATTACH:
{
CloseHandle(CreateThread(nullptr, 0, (LPTHREAD_START_ROUTINE)Init, hModule, 0, nullptr));
}
case DLL_THREAD_ATTACH:
case DLL_THREAD_DETACH:
case DLL_PROCESS_DETACH:
break;
}
return TRUE;
}
Credits to me, Broihon, Solaire & 0xDEC0DE for this barebones d3d9 endscene hook