Hooking DirectX EndScene from an injected DLL

Question

I want to detour EndScene from an arbitrary DirectX 9 application to create a small overlay. As an example, you could take the frame counter overlay of FRAPS, w

Answer 1

I know this question is old, but this should work for any program using DirectX9, You are creating your own instance basically, and then getting the pointer to the VTable, then you just hook it. You will need detours 3.X btw:

//Just some typedefs:
typedef HRESULT (WINAPI* oEndScene) (LPDIRECT3DDEVICE9 D3DDevice);
static oEndScene EndScene;

//Do this in a function or whatever
HMODULE hDLL=GetModuleHandleA("d3d9");
LPDIRECT3D9(__stdcall*pDirect3DCreate9)(UINT) = (LPDIRECT3D9(__stdcall*)(UINT))GetProcAddress( hDLL, "Direct3DCreate9");

LPDIRECT3D9 pD3D = pDirect3DCreate9(D3D_SDK_VERSION);

D3DDISPLAYMODE d3ddm;
HRESULT hRes = pD3D->GetAdapterDisplayMode(D3DADAPTER_DEFAULT, &d3ddm );
D3DPRESENT_PARAMETERS d3dpp; 
ZeroMemory( &d3dpp, sizeof(d3dpp));
d3dpp.Windowed = true;
d3dpp.SwapEffect = D3DSWAPEFFECT_DISCARD;
d3dpp.BackBufferFormat = d3ddm.Format;

WNDCLASSEX wc = { sizeof(WNDCLASSEX),CS_CLASSDC,TempWndProc,0L,0L,GetModuleHandle(NULL),NULL,NULL,NULL,NULL,("1"),NULL};
RegisterClassEx(&wc);
HWND hWnd = CreateWindow(("1"),NULL,WS_OVERLAPPEDWINDOW,100,100,300,300,GetDesktopWindow(),NULL,wc.hInstance,NULL);

hRes = pD3D->CreateDevice( 
    D3DADAPTER_DEFAULT,
    D3DDEVTYPE_HAL,
    hWnd,
    D3DCREATE_SOFTWARE_VERTEXPROCESSING | D3DCREATE_DISABLE_DRIVER_MANAGEMENT,
    &d3dpp, &ppReturnedDeviceInterface);

pD3D->Release();
DestroyWindow(hWnd);

if(pD3D == NULL){
    //printf ("WARNING: D3D FAILED");
    return false;
}
pInterface = (unsigned long*)*((unsigned long*)ppReturnedDeviceInterface);


EndScene = (oEndScene) (DWORD) pInterface[42];
DetourTransactionBegin();
DetourUpdateThread(GetCurrentThread());
DetourAttach(&(PVOID&)EndScene, newEndScene);
DetourTransactionCommit();

And then your function:

HRESULT WINAPI D3D9Hook::newEndScene(LPDIRECT3DDEVICE9 pDevice)
{   
    //Do your stuff here

    //Call the original (if you want)
    return EndScene(pDevice);
}