Overriding Direct3D interfaces? - c++

Question

My project launches a target process, injecting a library into it which is meant to hook Direct3D functions. I have done this successfully in the past, but decided to rewrite the library to a closer specification of the Direct3D interfaces. The idea here was to create a set of my own 1:1 wrapper classes, each inheriting a DirectX interface (ie, class CD3D8 : IDirect3D8, class CD3DDevice8 : IDirect3DDevice8 and so forth). Since all members of each underlying DirectX COM interface are pure virtual methods, I thought I could easily override them...
So when my library hooks Direct3DCreate8, I return a pointer to my CDirect3D8 instead of the standard IDirect3D8. And then my library uses the virtual table of that pointer to hook method #15, which is IDirect3D8::CreateDevice. And once that is called, I return a pointer to CDirect3DDevice8 as opposed to IDirect3DDevice8. All appears to be well, except none of the overrided functions are being called by the injected application! Somehow, the app appears to be calling the original interface functions instead of my own. Am I missing some sort of concept here? Do I need to manually remap the virtual table pointers to the ones in my custom wrapper classes, for example?
Here's what I got so far (only showing d3d8):
D3D.h
#pragma once
#define STDM(method) COM_DECLSPEC_NOTHROW HRESULT STDMETHODCALLTYPE method
#define STDM_(type,method) COM_DECLSPEC_NOTHROW type STDMETHODCALLTYPE method
template<class IDirect3D8>
class CD3D : public IDirect3D8
{
public:
CD3D();
~CD3D();
/*** IUnknown methods ***/
STDM(QueryInterface)(THIS_ REFIID riid, void** ppvObj);
STDM_(ULONG,AddRef)(THIS);
STDM_(ULONG,Release)(THIS);
/*** IDirect3D8 methods ***/
STDM(RegisterSoftwareDevice)(THIS_ void* pInitializeFunction);
STDM_(UINT, GetAdapterCount)(THIS);
STDM(GetAdapterIdentifier)(THIS_ UINT Adapter,DWORD Flags,D3DADAPTER_IDENTIFIER8* pIdentifier);
STDM_(UINT, GetAdapterModeCount)(THIS_ UINT Adapter);
STDM(EnumAdapterModes)(THIS_ UINT Adapter,UINT Mode,D3DDISPLAYMODE* pMode);
STDM(GetAdapterDisplayMode)(THIS_ UINT Adapter,D3DDISPLAYMODE* pMode);
STDM(CheckDeviceType)(THIS_ UINT Adapter,D3DDEVTYPE CheckType,D3DFORMAT DisplayFormat,D3DFORMAT BackBufferFormat,BOOL Windowed);
STDM(CheckDeviceFormat)(THIS_ UINT Adapter,D3DDEVTYPE DeviceType,D3DFORMAT AdapterFormat,DWORD Usage,D3DRESOURCETYPE RType,D3DFORMAT CheckFormat);
STDM(CheckDeviceMultiSampleType)(THIS_ UINT Adapter,D3DDEVTYPE DeviceType,D3DFORMAT SurfaceFormat,BOOL Windowed,D3DMULTISAMPLE_TYPE MultiSampleType);
STDM(CheckDepthStencilMatch)(THIS_ UINT Adapter,D3DDEVTYPE DeviceType,D3DFORMAT AdapterFormat,D3DFORMAT RenderTargetFormat,D3DFORMAT DepthStencilFormat);
STDM(GetDeviceCaps)(THIS_ UINT Adapter,D3DDEVTYPE DeviceType,D3DCAPS8* pCaps);
STDM_(HMONITOR, GetAdapterMonitor)(THIS_ UINT Adapter);
STDM(CreateDevice)(THIS_ UINT Adapter,D3DDEVTYPE DeviceType,HWND hFocusWindow,DWORD BehaviorFlags,D3DPRESENT_PARAMETERS* pPresentationParameters,IDirect3DDevice8** ppReturnedDeviceInterface);
};
D3D.cpp
#include "stdafx.h"
#define STDIMP(iface, method, ...) COM_DECLSPEC_NOTHROW HRESULT STDMETHODCALLTYPE CD3D<iface>::method(__VA_ARGS__)
#define STDIMP_(iface, type, method, ...) COM_DECLSPEC_NOTHROW type STDMETHODCALLTYPE CD3D<iface>::method(__VA_ARGS__)
#define STDIMP8(method, ...) STDIMP(IDirect3D8, method, __VA_ARGS__)
#define STDIMP8_(type, method, ...) STDIMP_(IDirect3D8, type, method, __VA_ARGS__)
CD3D<IDirect3D8>::CD3D()
{
}
CD3D<IDirect3D8>::~CD3D()
{
}
STDIMP8(QueryInterface, THIS_ REFIID riid, void** ppvObj) {
return QueryInterface( riid, ppvObj );
}
STDIMP8_(ULONG, AddRef, THIS) {
return AddRef();
}
STDIMP8_(ULONG, Release, THIS) {
return Release();
}
STDIMP8(RegisterSoftwareDevice, THIS_ void* pInitializeFunction) {
return RegisterSoftwareDevice( pInitializeFunction );
}
STDIMP8_(UINT, GetAdapterCount, THIS) {
return GetAdapterCount();
}
STDIMP8(GetAdapterIdentifier, THIS_ UINT Adapter, DWORD Flags, D3DADAPTER_IDENTIFIER8* pIdentifier) {
return GetAdapterIdentifier( Adapter, Flags, pIdentifier );
}
STDIMP8_(UINT, GetAdapterModeCount, THIS_ UINT Adapter) {
return GetAdapterModeCount( Adapter );
}
STDIMP8(EnumAdapterModes, THIS_ UINT Adapter, UINT Mode, D3DDISPLAYMODE* pMode) {
return EnumAdapterModes( Adapter, Mode, pMode );
}
STDIMP8(GetAdapterDisplayMode, THIS_ UINT Adapter, D3DDISPLAYMODE* pMode) {
return GetAdapterDisplayMode( Adapter, pMode );
}
STDIMP8(CheckDeviceType, THIS_ UINT Adapter, D3DDEVTYPE CheckType, D3DFORMAT DisplayFormat, D3DFORMAT BackBufferFormat, BOOL Windowed) {
return CheckDeviceType( Adapter, CheckType, DisplayFormat, BackBufferFormat, Windowed );
}
STDIMP8(CheckDeviceFormat, THIS_ UINT Adapter, D3DDEVTYPE DeviceType, D3DFORMAT AdapterFormat, DWORD Usage, D3DRESOURCETYPE RType, D3DFORMAT CheckFormat) {
return CheckDeviceFormat( Adapter, DeviceType, AdapterFormat, Usage, RType, CheckFormat );
}
STDIMP8(CheckDeviceMultiSampleType, THIS_ UINT Adapter, D3DDEVTYPE DeviceType, D3DFORMAT SurfaceFormat, BOOL Windowed, D3DMULTISAMPLE_TYPE MultiSampleType) {
return CheckDeviceMultiSampleType( Adapter, DeviceType, SurfaceFormat, Windowed, MultiSampleType );
}
STDIMP8(CheckDepthStencilMatch, THIS_ UINT Adapter, D3DDEVTYPE DeviceType, D3DFORMAT AdapterFormat, D3DFORMAT RenderTargetFormat, D3DFORMAT DepthStencilFormat) {
return CheckDepthStencilMatch( Adapter, DeviceType, AdapterFormat, RenderTargetFormat, DepthStencilFormat );
}
STDIMP8(GetDeviceCaps, THIS_ UINT Adapter, D3DDEVTYPE DeviceType, D3DCAPS8* pCaps) {
return GetDeviceCaps( Adapter, DeviceType, pCaps );
}
STDIMP8_(HMONITOR, GetAdapterMonitor, THIS_ UINT Adapter) {
return GetAdapterMonitor( Adapter );
}
STDIMP8(CreateDevice, THIS_ UINT Adapter, D3DDEVTYPE DeviceType, HWND hFocusWindow, DWORD BehaviorFlags, D3DPRESENT_PARAMETERS* pPresentationParameters, IDirect3DDevice8** ppReturnedDeviceInterface) {
return CreateDevice( Adapter, DeviceType, hFocusWindow, BehaviorFlags, pPresentationParameters, ppReturnedDeviceInterface );
}
Main.cpp (Using Microsoft Detours 3.0):
#include <detours.h>
#include "D3D.h"
typedef HMODULE (WINAPI * HookLoadLibraryA)( LPCSTR lpFileName );
typedef IDirect3D8 *(WINAPI * HookDirect3DCreate8)( UINT SdkVersion );
typedef HRESULT (WINAPI * HookCreateDevice8)( IDirect3DDevice8* pInterface, UINT Adapter, D3DDEVTYPE DeviceType, HWND hFocusWindow, DWORD BehaviorFlags, D3DPRESENT_PARAMETERS* pPresentationParameters, IDirect3DDevice8** ppReturnedDeviceInterface );
HookLoadLibraryA RealLoadLibraryA;
HookDirect3DCreate8 RealDirect3DCreate8;
HookCreateDevice8 RealCreateDevice8;
//...
CD3D<IDirect3D8> *m_d3d8;
CD3DDevice<IDirect3D8> *m_d3dDev8;
//...
RealLoadLibraryA = (HookLoadLibraryA)GetProcAddress(GetModuleHandleA("kernel32.dll"), "LoadLibraryA");
DetourTransactionBegin();
DetourUpdateThread(GetCurrentThread());
DetourAttach(&(PVOID&)RealLoadLibraryA, FakeLoadLibraryA);
DetourTransactionCommit();
//...
VOID VirtualHook( PVOID pInterface, PVOID pHookProc, PVOID pOldProc, int iIndex )
{
// Hook a procedure within an interface's virtual table
PDWORD pVtable = (PDWORD)*((PDWORD)pInterface);
DWORD lpflOldProtect;
VirtualProtect( (PVOID)&pVtable[iIndex], sizeof(DWORD), PAGE_READWRITE, &lpflOldProtect );
if( pOldProc ) *(DWORD*)pOldProc = pVtable[iIndex];
pVtable[iIndex] = (DWORD)pHookProc;
VirtualProtect( pVtable, sizeof(DWORD), lpflOldProtect, &lpflOldProtect );
}
HRESULT WINAPI FakeCreateDevice8( IDirect3DDevice8* pInterface, UINT Adapter, D3DDEVTYPE DeviceType, HWND hFocusWindow, DWORD BehaviorFlags, D3DPRESENT_PARAMETERS* pPresentationParameters, IDirect3DDevice8** ppReturnedDeviceInterface )
{
HRESULT ret = RealCreateDevice8( pInterface, Adapter, DeviceType, hFocusWindow, BehaviorFlags, pPresentationParameters, ppReturnedDeviceInterface );
// Save the registers
__asm pushad
if(*ppReturnedDeviceInterface != NULL)
m_d3dDev8 = reinterpret_cast<CD3DDevice<IDirect3D8> *>(*ppReturnedDeviceInterface);
// Restore the registers
__asm popad
return ret;
}
IDirect3D8 *WINAPI FakeDirect3DCreate8( UINT SdkVersion )
{
m_d3d8 = reinterpret_cast<CD3D<IDirect3D8> *>(RealDirect3DCreate8( SdkVersion ));
if( m_d3d8 ) {
// Hook CreateDevice (vftable index #15)
VirtualHook( m_d3d8, &FakeCreateDevice8, &RealCreateDevice8, 15 );
}
return m_d3d8;
}
HMODULE WINAPI FakeLoadLibraryA( LPCSTR lpFileName )
{
CStringA strFileName( lpFileName );
int i = strFileName.ReverseFind('\\');
if(i != -1) strFileName = strFileName.Right(i + 1);
if( strFileName.CompareNoCase("d3d8.dll") == 0 )
{
// Hook Direct3DCreate8
HMODULE m_hD3D = RealLoadLibraryA( lpFileName );
RealDirect3DCreate8 = (HookDirect3DCreate8)GetProcAddress(m_hD3D, "Direct3DCreate8");
DetourTransactionBegin();
DetourUpdateThread( GetCurrentThread() );
DetourAttach(&(PVOID&)RealDirect3DCreate8, FakeDirect3DCreate8);
DetourTransactionCommit();
return m_hD3D;
}
}
... The hooked functions get called, but none of my wrapper functions do, and the injected application runs just as it normally would. Why is this? Of course, I could manually set hooks for each and every function found in each and every DirectX interface (for each version of Direct3D), but the point of this was to try and prevent having to do that and to keep it a bit cleaner. So is there a way to get this to work as I had intended? Thanks!

Answer 1

You're using reinterpret_cast, which won't do much since the underlying object will still have a pointer to the 'real' IDirect3D8 or IDirect3DDevice8 COM vtable, which will be used for the calls.
Instead, instantiate your custom CD3D or CD3DDevice by passing it the original object instance. Then modify all your calls to call back the correct method on the original object -- your class effectively acting as a transparent proxy.
I mean something such as:
STDIMP8_(ULONG, AddRef, THIS) {
return realObject->AddRef();
}
with realObject being the original IDirect3D8*.