From 933ff38adb61f7c03920697f419fde45a77f8eb1 Mon Sep 17 00:00:00 2001 From: Alx Date: Wed, 11 Mar 2026 22:07:34 +0000 Subject: [PATCH] d3dhook rewrite This is a rewrite of d3dhook using minhook which was being used in other places in caster already, this is api compatible with there being a caveat of it returning different strings but those shouldn't affect it where it's being used. The rewrite was done so d3d9 hooking would work on linux and then allowing for some wine specific workarounds to be removed along with enabling the frame limiter in linux. The only problem with this is that it seems to crash on alt-enter on debug builds since dearimgui seems not to like it, unsure if this is linux only or a problem that was happening before. --- 3rdparty/d3dhook/D3DHook.cc | 380 ++++++++---------------------------- Makefile | 2 +- targets/DllHacks.cpp | 3 +- targets/DllMain.cpp | 7 +- targets/DllTrialManager.cpp | 2 - 5 files changed, 87 insertions(+), 307 deletions(-) diff --git a/3rdparty/d3dhook/D3DHook.cc b/3rdparty/d3dhook/D3DHook.cc index f9e2645c..c15bdb84 100644 --- a/3rdparty/d3dhook/D3DHook.cc +++ b/3rdparty/d3dhook/D3DHook.cc @@ -1,341 +1,123 @@ #include "D3DHook.h" -#include -#include -#include - #include #include -#include -#include - #include -#include -using namespace std; - -template -inline string toHexString ( const T& val ) -{ - stringstream ss; - ss << hex << val; - return ss.str(); -} +#include "../minhook/include/MinHook.h" -// DX interface entry offsets. -#define INTF_QueryInterface 0 -#define INTF_AddRef 1 -#define INTF_Release 2 -#define INTF_DX9_Reset 16 -#define INTF_DX9_Present 17 -#define INTF_DX9_EndScene 42 +#pragma comment(lib, "libMinHook.x86.lib") -typedef IDirect3D9 * ( __stdcall *DIRECT3DCREATE9 ) ( UINT ); -typedef ULONG ( __stdcall *PFN_DX9_ADDREF ) ( IDirect3DDevice9 *pDevice ); -typedef ULONG ( __stdcall *PFN_DX9_RELEASE ) ( IDirect3DDevice9 *pDevice ); -typedef HRESULT ( __stdcall *PFN_DX9_RESET ) ( IDirect3DDevice9 *pDevice, LPVOID ); -typedef HRESULT ( __stdcall *PFN_DX9_PRESENT ) ( IDirect3DDevice9 *pDevice, const RECT *, const RECT *, HWND, LPVOID ); -typedef HRESULT ( __stdcall *PFN_DX9_ENDSCENE ) ( IDirect3DDevice9 *pDevice ); +enum d3d_offsets { + D3D9_RESET = 16, + D3D9_PRESENT = 17, + D3D9_ENDSCENE = 42 +}; -CDllFile g_DX9; -IDirect3DDevice9 *g_pDevice = 0; -ULONG g_iRefCount = 1; -ULONG g_iRefCountMe = 0; -UINT_PTR m_nDX9_Present; -UINT_PTR m_nDX9_Reset; -UINT_PTR m_nDX9_EndScene; +typedef LPDIRECT3D9 (WINAPI* d3d9CreateFn)(UINT); +typedef HRESULT ( __stdcall *ResetFn ) ( IDirect3DDevice9 *pDevice, LPVOID ); +typedef HRESULT ( __stdcall *PresentFn ) ( IDirect3DDevice9 *pDevice, const RECT *, const RECT *, HWND, LPVOID ); +typedef HRESULT ( __stdcall *EndSceneFn ) ( IDirect3DDevice9 *pDevice); -CHookJump m_Hook_Present; -CHookJump m_Hook_Reset; -CHookJump m_Hook_EndScene; -UINT_PTR *m_Hook_AddRef = 0; -UINT_PTR *m_Hook_Release = 0; +ResetFn old_reset; +PresentFn old_present; +EndSceneFn old_endscene; -PFN_DX9_ADDREF s_D3D9_AddRef = 0; -PFN_DX9_RELEASE s_D3D9_Release = 0; -PFN_DX9_RESET s_D3D9_Reset = 0; -PFN_DX9_PRESENT s_D3D9_Present = 0; -PFN_DX9_ENDSCENE s_D3D9_EndScene = 0; +void **VTable; -EXTERN_C ULONG __declspec ( dllexport ) __stdcall DX9_AddRef ( IDirect3DDevice9 *pDevice ) -{ - // New AddRef function - g_iRefCount = s_D3D9_AddRef ( pDevice ); - // DEBUG_TRACE(("DX9_AddRef: called (m_iRefCount = %d)." LOG_CR, g_iRefCount)); - return g_iRefCount; +HRESULT __stdcall hook_reset(IDirect3DDevice9 *device, LPVOID params) { + InvalidateDeviceObjects(); + HRESULT res = old_reset(device, params); + return res; } -EXTERN_C ULONG __declspec ( dllexport ) __stdcall DX9_Release ( IDirect3DDevice9 *pDevice ) -{ - // New Release function - // a "fall-through" case - if ( ( g_iRefCount > g_iRefCountMe + 1 ) && s_D3D9_Release ) - { - g_iRefCount = s_D3D9_Release ( pDevice ); - // DEBUG_TRACE(("DX9_Release: called (m_iRefCount = %d)." LOG_CR, g_iRefCount)); - return g_iRefCount; - } - - /* - DEBUG_TRACE(("+++++++++++++++++++++++++++++++++++++" LOG_CR )); - DEBUG_MSG(("DX9_Release: called." LOG_CR)); - DEBUG_TRACE(("DX9_Release: pDevice = %08x" LOG_CR, (UINT_PTR)pDevice)); - DEBUG_TRACE(("DX9_Release: VTABLE[0] = %08x" LOG_CR, ((UINT_PTR*)(*((UINT_PTR*)pDevice)))[0])); - DEBUG_TRACE(("DX9_Release: VTABLE[1] = %08x" LOG_CR, ((UINT_PTR*)(*((UINT_PTR*)pDevice)))[1])); - DEBUG_TRACE(("DX9_Release: VTABLE[2] = %08x" LOG_CR, ((UINT_PTR*)(*((UINT_PTR*)pDevice)))[2])); - */ - - g_pDevice = pDevice; - - // unhook device methods - UnhookDirectX(); - - // reset the pointers - m_Hook_AddRef = 0; - m_Hook_Release = 0; - - // call the real Release() - // DEBUG_MSG(( "DX9_Release: about to call real Release." LOG_CR)); - - g_iRefCount = s_D3D9_Release ( pDevice ); - // DEBUG_MSG(( "DX9_Release: UNHOOK m_iRefCount = %d" LOG_CR, g_iRefCount)); - return g_iRefCount; +HRESULT __stdcall hook_present(IDirect3DDevice9 *device, const RECT *src, const RECT *dest, HWND window, LPVOID unused) { + PresentFrameBegin(device); + HRESULT res = old_present(device, src, dest, window, unused); + PresentFrameEnd(device); + return res; } -void DX9_HooksInit ( IDirect3DDevice9 *pDevice ) -{ - UINT_PTR *pVTable = ( UINT_PTR * ) ( * ( ( UINT_PTR * ) pDevice ) ); - assert ( pVTable ); - m_Hook_AddRef = pVTable + 1; - m_Hook_Release = pVTable + 2; - - // DEBUG_TRACE(("*m_Hook_AddRef = %08x" LOG_CR, *g_DX9.m_Hook_AddRef)); - // DEBUG_TRACE(("*m_Hook_Release = %08x" LOG_CR, *g_DX9.m_Hook_Release)); - - // hook AddRef method - s_D3D9_AddRef = ( PFN_DX9_ADDREF ) ( *m_Hook_AddRef ); - *m_Hook_AddRef = ( UINT_PTR ) DX9_AddRef; - - // hook Release method - s_D3D9_Release = ( PFN_DX9_RELEASE ) ( *m_Hook_Release ); - *m_Hook_Release = ( UINT_PTR ) DX9_Release; +HRESULT __stdcall hook_endscene(IDirect3DDevice9 *device) { + EndScene(device); + HRESULT res = old_endscene(device); + return res; } -void DX9_HooksVerify ( IDirect3DDevice9 *pDevice ) -{ - // It looks like at certain points, vtable entries get restored to its original values. - // If that happens, we need to re-assign them to our functions again. - // NOTE: we don't want blindly re-assign, because there can be other programs - // hooking on the same methods. Therefore, we only re-assign if we see that - // original addresses are restored by the system. - - UINT_PTR *pVTable = ( UINT_PTR * ) ( * ( ( UINT_PTR * ) pDevice ) ); - assert ( pVTable ); - if ( pVTable[INTF_AddRef] == ( UINT_PTR ) s_D3D9_AddRef ) - { - pVTable[INTF_AddRef] = ( UINT_PTR ) DX9_AddRef; - // DEBUG_MSG(( "DX9_HooksVerify: pDevice->AddRef() re-hooked." LOG_CR)); +std::string InitDirectX(void *hwnd) { + // Get VTable From Dummy Device + HMODULE d3d9_module = GetModuleHandleA("d3d9.dll"); + if (d3d9_module == NULL) { + return "failed to get the d3d9.dll module handle"; } - if ( pVTable[INTF_Release] == ( UINT_PTR ) s_D3D9_Release ) - { - pVTable[INTF_Release] = ( UINT_PTR ) DX9_Release; - // DEBUG_MSG(( "DX9_HooksVerify: pDevice->Release() re-hooked." LOG_CR)); - } -} - -EXTERN_C HRESULT __declspec ( dllexport ) __stdcall DX9_EndScene ( IDirect3DDevice9 *pDevice ) -{ - // New EndScene function - // put back saved code fragment - m_Hook_EndScene.SwapOld ( ( void * ) s_D3D9_EndScene ); - - // LOG(( "DX9_EndScene: called." LOG_CR)); - g_pDevice = pDevice; + d3d9CreateFn create_fn = (d3d9CreateFn)GetProcAddress(d3d9_module, "Direct3DCreate9"); - // remember IDirect3DDevice9::Release pointer so that we can clean-up properly. - if ( m_Hook_AddRef == 0 || m_Hook_Release == 0 ) - { - DX9_HooksInit ( pDevice ); + IDirect3D9 *direct3d = create_fn(D3D_SDK_VERSION); + if (direct3d == NULL) { + return "failed to create d3d9 object"; } - EndScene ( pDevice ); - - // call real EndScene() - HRESULT hRes = s_D3D9_EndScene ( pDevice ); - - DX9_HooksVerify ( pDevice ); - - // DEBUG_MSG(( "DX9_EndScene: done." LOG_CR)); - - // put JMP instruction again - m_Hook_EndScene.SwapReset ( ( void * ) s_D3D9_EndScene ); - return hRes; -} - -EXTERN_C HRESULT __declspec ( dllexport ) __stdcall DX9_Reset ( IDirect3DDevice9 *pDevice, LPVOID params ) -{ - // New Reset function - // put back saved code fragment - m_Hook_Reset.SwapOld ( ( void * ) s_D3D9_Reset ); - - // LOG(( "DX9_Reset: called." LOG_CR)); - - g_pDevice = pDevice; - - InvalidateDeviceObjects(); - - // call real Reset() - HRESULT hRes = s_D3D9_Reset ( pDevice, params ); - - DX9_HooksVerify ( pDevice ); - - // DEBUG_MSG(( "DX9_Reset: done." LOG_CR)); - - // put JMP instruction again - m_Hook_Reset.SwapReset ( ( void * ) s_D3D9_Reset ); - return hRes; -} - -EXTERN_C HRESULT __declspec ( dllexport ) __stdcall DX9_Present ( - IDirect3DDevice9 *pDevice, const RECT *src, const RECT *dest, HWND hwnd, LPVOID unused ) -{ - // New Present function - m_Hook_Present.SwapOld ( ( void * ) s_D3D9_Present ); - - // DEBUG_TRACE(( "--------------------------------" LOG_CR )); - // DEBUG_TRACE(( "DX9_Present: called." LOG_CR )); - - g_pDevice = pDevice; - - // remember IDirect3DDevice9::Release pointer so that we can clean-up properly. - if ( m_Hook_AddRef == 0 || m_Hook_Release == 0 ) - { - DX9_HooksInit ( pDevice ); + D3DPRESENT_PARAMETERS d3dparams = {0}; + d3dparams.Windowed = true; + d3dparams.SwapEffect = D3DSWAPEFFECT_DISCARD; + + IDirect3DDevice9 *device; + HRESULT res = direct3d->CreateDevice( + D3DADAPTER_DEFAULT, + D3DDEVTYPE_HAL, + (HWND)hwnd, + D3DCREATE_SOFTWARE_VERTEXPROCESSING, + &d3dparams, + &device + ); + if (res != D3D_OK) { + return "failed to create d3d9 device"; } - PresentFrameBegin ( pDevice ); + VTable = *(void ***)device; - // call real Present() - HRESULT hRes = s_D3D9_Present ( pDevice, src, dest, hwnd, unused ); + // Cleanup + device->Release(); + direct3d->Release(); - PresentFrameEnd ( pDevice ); - - DX9_HooksVerify ( pDevice ); - // DEBUG_TRACE(( "DX9_Present: done." LOG_CR )); - - m_Hook_Present.SwapReset ( ( void * ) s_D3D9_Present ); - return hRes; + return ""; } -string InitDirectX ( void *hwnd ) -{ - // get the offset from the start of the DLL to the interface element we want. - // step 1: Load d3d9.dll - HRESULT hRes = g_DX9.LoadDll ( TEXT ( "d3d9" ) ); - if ( IS_ERROR ( hRes ) ) - { - _com_error err ( hRes ); - return "Failed to load d3d9.dll: [" + toHexString ( hRes ) + "] " + err.ErrorMessage(); +std::string HookDirectX() { + if (MH_CreateHook(VTable[D3D9_RESET], (void*)&hook_reset, reinterpret_cast(&old_reset)) != MH_OK) { + return "failed to hook reset"; } - - // step 2: Get IDirect3D9 - DIRECT3DCREATE9 pDirect3DCreate9 = ( DIRECT3DCREATE9 ) g_DX9.GetProcAddress ( "Direct3DCreate9" ); - if ( pDirect3DCreate9 == 0 ) - { - return "Unable to find Direct3DCreate9"; + if (MH_CreateHook(VTable[D3D9_PRESENT], (void*)&hook_present, reinterpret_cast(&old_present)) != MH_OK) { + return "failed to hook present"; } - - IRefPtr pD3D = pDirect3DCreate9 ( D3D_SDK_VERSION ); - if ( !pD3D.IsValidRefObj() ) - { - return "Direct3DCreate9 failed"; + if (MH_CreateHook(VTable[D3D9_ENDSCENE], (void*)&hook_endscene, reinterpret_cast(&old_endscene)) != MH_OK) { + return "failed to hook endscene"; } - // step 3: Get IDirect3DDevice9 - D3DDISPLAYMODE d3ddm; - hRes = pD3D->GetAdapterDisplayMode ( D3DADAPTER_DEFAULT, &d3ddm ); - if ( FAILED ( hRes ) ) - { - _com_error err ( hRes ); - return "GetAdapterDisplayMode failed: [" + toHexString ( hRes ) + "] " + err.ErrorMessage(); + if (MH_EnableHook(VTable[D3D9_RESET]) != MH_OK) { + return "failed to enable reset hook"; } - - D3DPRESENT_PARAMETERS d3dpp; - ZeroMemory ( &d3dpp, sizeof ( d3dpp ) ); - d3dpp.Windowed = true; - d3dpp.SwapEffect = D3DSWAPEFFECT_DISCARD; - d3dpp.BackBufferFormat = d3ddm.Format; - - IRefPtr pD3DDevice; - hRes = pD3D->CreateDevice ( D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, ( HWND ) hwnd, - D3DCREATE_SOFTWARE_VERTEXPROCESSING, - &d3dpp, IREF_GETPPTR ( pD3DDevice, IDirect3DDevice9 ) ); - if ( FAILED ( hRes ) ) - { - _com_error err ( hRes ); - return "CreateDevice failed: [" + toHexString ( hRes ) + "] " + err.ErrorMessage(); + if (MH_EnableHook(VTable[D3D9_PRESENT]) != MH_OK) { + return "failed to enable present hook"; + } + if (MH_EnableHook(VTable[D3D9_ENDSCENE]) != MH_OK) { + return "failed to enable endscene hook"; } - - // step 4: store method addresses in out vars - UINT_PTR *pVTable = ( UINT_PTR * ) ( * ( ( UINT_PTR * ) pD3DDevice.get_RefObj() ) ); - assert ( pVTable ); - m_nDX9_Present = ( pVTable[INTF_DX9_Present] - g_DX9.get_DllInt() ); - m_nDX9_Reset = ( pVTable[INTF_DX9_Reset] - g_DX9.get_DllInt() ); - m_nDX9_EndScene = ( pVTable[INTF_DX9_EndScene] - g_DX9.get_DllInt() ); - - // LOG ( "InitDirectX: %08x, Present=0%x, Reset=0%x ", - // ( UINT_PTR ) pD3DDevice.get_RefObj(), m_nDX9_Present, m_nDX9_Reset ); - return ""; -} - -string HookDirectX() -{ - // This function hooks two IDirect3DDevice9 methods, using code overwriting technique. - // hook IDirect3DDevice9::Present(), using code modifications at run-time. - // ALGORITHM: we overwrite the beginning of real IDirect3DDevice9::Present - // with a JMP instruction to our routine (DX9_Present). - // When our routine gets called, first thing that it does - it restores - // the original bytes of IDirect3DDevice9::Present, then performs its pre-call tasks, - // then calls IDirect3DDevice9::Present, then does post-call tasks, then writes - // the JMP instruction back into the beginning of IDirect3DDevice9::Present, and - // returns. - - if ( !m_nDX9_Present || !m_nDX9_Reset || !m_nDX9_EndScene ) - return "No info on 'Present' and/or 'Reset'"; - - s_D3D9_Present = ( PFN_DX9_PRESENT ) ( g_DX9.get_DllInt() + m_nDX9_Present ); - s_D3D9_Reset = ( PFN_DX9_RESET ) ( g_DX9.get_DllInt() + m_nDX9_Reset ); - s_D3D9_EndScene = ( PFN_DX9_ENDSCENE ) ( g_DX9.get_DllInt() + m_nDX9_EndScene ); - - if ( !m_Hook_Present.InstallHook ( ( void * ) s_D3D9_Present, ( void * ) DX9_Present ) ) - return "m_Hook_Present failed"; - - if ( !m_Hook_Reset.InstallHook ( ( void * ) s_D3D9_Reset, ( void * ) DX9_Reset ) ) - return "m_Hook_Reset failed"; - - if ( !m_Hook_EndScene.InstallHook ( ( void * ) s_D3D9_EndScene, ( void * ) DX9_EndScene ) ) - return "m_Hook_EndScene failed"; return ""; } -void UnhookDirectX() -{ - // Restore original Reset() and Present() - if ( m_Hook_AddRef != 0 && s_D3D9_AddRef != 0 ) - { - *m_Hook_AddRef = ( UINT_PTR ) s_D3D9_AddRef; +void UnhookDirectX() { + // we assert false here as this shouldn't fail and I don't think there's any sane way to recover + if (MH_DisableHook(VTable[D3D9_RESET]) != MH_OK) { + assert(false); } - if ( m_Hook_Release != 0 && s_D3D9_Release != 0 ) - { - *m_Hook_Release = ( UINT_PTR ) s_D3D9_Release; + if (MH_DisableHook(VTable[D3D9_PRESENT]) != MH_OK) { + assert(false); + } + if (MH_DisableHook(VTable[D3D9_ENDSCENE]) != MH_OK) { + assert(false); } - - // restore IDirect3D9Device methods - m_Hook_Present.RemoveHook ( ( void * ) s_D3D9_Present ); - m_Hook_Reset.RemoveHook ( ( void * ) s_D3D9_Reset ); - m_Hook_EndScene.RemoveHook ( ( void * ) s_D3D9_EndScene ); - - InvalidateDeviceObjects(); } + diff --git a/Makefile b/Makefile index 50db09c6..866e8b67 100644 --- a/Makefile +++ b/Makefile @@ -208,7 +208,7 @@ res/rollback.bin: tools/$(GENERATOR) ifeq ($(UNAME),Darwin) wine tools/$(GENERATOR) $@ else ifeq ($(UNAME),Linux) - tools/$(GENERATOR) $@ + wine tools/$(GENERATOR) $@ else tools/$(GENERATOR) $@ endif diff --git a/targets/DllHacks.cpp b/targets/DllHacks.cpp index 8a86c34f..eb76b9af 100644 --- a/targets/DllHacks.cpp +++ b/targets/DllHacks.cpp @@ -207,8 +207,7 @@ void initializePostLoad() LOG ( "Enable hook failed: %s", MH_StatusString ( status ) ); bool loadFramestep = ( GetAsyncKeyState ( VK_F8 ) & 0x8000 ) == 0x8000; - // We can't hook DirectX calls on Wine (yet?). - if ( ProcessManager::isWine() || loadFramestep ) + if ( loadFramestep ) { return; } diff --git a/targets/DllMain.cpp b/targets/DllMain.cpp index 26fa753c..ffe727c2 100644 --- a/targets/DllMain.cpp +++ b/targets/DllMain.cpp @@ -1024,7 +1024,8 @@ struct DllMain #endif // RELEASE // Enable controllers now - if ( ! ProcessManager::isWine() ) + // TODO: I think this doesn't need to be behind an isWine flag + // if ( ! ProcessManager::isWine() ) ControllerManager::get().startHighFreqPolling(); // Initialize the overlay now @@ -1575,7 +1576,7 @@ struct DllMain } else { netMan.autoReplaySave = false; } - if ( ProcessManager::isWine() || options[Options::FrameLimiter] ) { + if ( options[Options::FrameLimiter] ) { } else { DllFrameRate::enable(); @@ -1865,7 +1866,7 @@ struct DllMain DllControllerManager::displayIPs = true; DllControllerManager::port = std::to_string(serverCtrlSocket->address.port); DllControllerManager::localIP = getInternalIpAddresses(); - + // Update the broadcast port and send over IPC netMan.config.broadcastPort = serverCtrlSocket->address.port; diff --git a/targets/DllTrialManager.cpp b/targets/DllTrialManager.cpp index 742fc961..188b4029 100644 --- a/targets/DllTrialManager.cpp +++ b/targets/DllTrialManager.cpp @@ -1566,8 +1566,6 @@ void DllTrialManager::render() //drawiidx(); if ( TrialManager::inputGuideEnabled ) drawInputGuide(); - if ( ProcessManager::isWine() ) - drawWineOverlay(); if ( TrialManager::inputEditorEnabled ) drawInputEditor(); }