Not able to hook COM functions properly?

[chyyran]

Trying the following trivial-ish example to test out hooking Direct3D11 Present but the hooked application just hangs on call. "present" is never printed to stdout but the hooked application will hang indefinitely until manually closed. SEV just gives me AppHangB1 with no useful output. Function pointers seem to be fetched properly and target process and hook DLL is x64.

I'm using dll-syringe to do my injection, which seemed to work for the MessageBox example. However unlike the MessageBox I get the function pointer to hook from the COM vtable as one would do in traditional hooking of COM interface methods.

use std::error::Error;
use std::mem::size_of;
use std::ptr::{null, null_mut};
use detour::static_detour;

use winapi::shared::dxgi::*;
use winapi::shared::dxgitype::*;
use winapi::shared::minwindef::*;
use winapi::shared::windef::*;
use winapi::um::d3d11::*;
use winapi::um::d3dcommon::*;
use winapi::shared::dxgiformat::*;

use winapi::um::libloaderapi::{DisableThreadLibraryCalls, GetModuleHandleW};
use winapi::um::unknwnbase::IUnknown;
use winapi::um::winnt::*;
use winapi::um::winuser::*;
use windy_macros::wstr;
use windy::WStr;

static_detour! {
    static PresentHook:  unsafe extern "system" fn(*mut IDXGISwapChain, UINT, UINT) -> HRESULT;
}

#[allow(non_snake_case)]
fn present(This: *mut IDXGISwapChain, SyncInterval: UINT, Flags: UINT) -> HRESULT {
    println!("present");
    unsafe { PresentHook.call(This, SyncInterval, Flags) }
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern "system" fn DllMain(
    module: HINSTANCE,
    call_reason: DWORD,
    _reserved: LPVOID,
) -> BOOL {
    unsafe { DisableThreadLibraryCalls(module); }

    if call_reason == DLL_PROCESS_ATTACH {
        unsafe { winapi::um::consoleapi::AllocConsole(); }

        std::thread::spawn(|| unsafe {
            match crate::main() {
                Ok(()) => 0 as u32,
                Err(e) => {
                    println!("Error occurred when injecting: {}", e);
                    1
                }
            }
        });
    }
    winapi::shared::minwindef::TRUE
}

unsafe fn main() -> Result<(), Box<dyn Error>> {
    let vtable = get_d3d11_vtables().as_ref().unwrap();
    println!("Found Present Pointer at {:p}", vtable.Present as *const ());
    PresentHook.initialize(vtable.Present, present)?;

    PresentHook.enable()?;

    // prints 'Hook activated' but target program hangs on `Present`
    println!("Hook activated");
    Ok(())
}

unsafe fn get_render_window() -> (WNDCLASSEXW, HWND) {
    let window_class_name = wstr!("DxHookWindowClass");
    let window_class = WNDCLASSEXW {
        cbSize: size_of::<WNDCLASSEXW>() as UINT,
        style: CS_HREDRAW | CS_VREDRAW,
        lpfnWndProc: Some(DefWindowProcW),
        cbClsExtra: 0,
        cbWndExtra: 0,
        hInstance: GetModuleHandleW(null()),
        hIcon: 0 as HICON,
        hCursor: 0 as HCURSOR,
        hbrBackground: 0 as HBRUSH,
        lpszMenuName: 0 as LPCWSTR,
        lpszClassName: window_class_name.as_ptr(),
        hIconSm: 0 as HICON
    };

    RegisterClassExW(&window_class);

    let hwnd = CreateWindowExW(0,
                               window_class.lpszClassName,
                               wstr!("DirectXWindow").as_ptr(),
        WS_OVERLAPPEDWINDOW,
        0, 0, 100, 100, 0 as HWND,
                               0 as HMENU, window_class.hInstance,
                               0 as LPVOID
    );

    (window_class, hwnd)
}

unsafe fn get_d3d11_vtables() -> *const IDXGISwapChainVtbl {
    let (window_class, hwnd) = get_render_window();
    println!("made new hwnd {:?}", hwnd);
    let swapchain_desc = DXGI_SWAP_CHAIN_DESC {
        BufferDesc: DXGI_MODE_DESC {
            Width: 100,
            Height: 100,
            RefreshRate: DXGI_RATIONAL {
                Numerator: 60,
                Denominator: 1
            },
            Format: DXGI_FORMAT_R8G8B8A8_UNORM,
            ScanlineOrdering: DXGI_MODE_SCANLINE_ORDER_UNSPECIFIED,
            Scaling: 0
        },
        SampleDesc: DXGI_SAMPLE_DESC {
            Count: 1,
            Quality: 0
        },
        BufferUsage: DXGI_USAGE_RENDER_TARGET_OUTPUT,
        BufferCount: 1,
        OutputWindow: hwnd,
        Windowed: 1,
        SwapEffect: DXGI_SWAP_EFFECT_DISCARD,
        Flags: DXGI_SWAP_CHAIN_FLAG_ALLOW_MODE_SWITCH
    };

    let feature_levels = [D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_11_1];
    let mut out_swapchain = null_mut::<IDXGISwapChain>();
    let mut out_device = null_mut::<ID3D11Device>();
    let mut out_context = null_mut::<ID3D11DeviceContext>();
    let mut out_feature_level : D3D_FEATURE_LEVEL = 0;
    //
    let result = D3D11CreateDeviceAndSwapChain(
        null_mut::<_>(),
        D3D_DRIVER_TYPE_HARDWARE,
        0 as HMODULE,
        0,
        feature_levels.as_ptr(),
        feature_levels.len() as UINT,
        D3D11_SDK_VERSION,
        &swapchain_desc,
        &mut out_swapchain,
        &mut out_device,
        &mut out_feature_level,
        &mut out_context
    );
    println!("d3dhresult {:x?}", 0);

    let swapchain_vtbl = (*out_swapchain).lpVtbl;

    out_swapchain.as_ref().map(|f| f.Release());
    out_device.as_ref().map(|f| f.Release());
    out_context.as_ref().map(|f| f.Release());

    CloseWindow(hwnd);
    UnregisterClassW(window_class.lpszClassName, window_class.hInstance);

    swapchain_vtbl
}

[SK83RJOSH]

This is most likely not an issue with detour-rs as I'm doing the same thing here with no issues. I'm using windows-rs but maybe you can find something useful there and compare approaches. 🙂

[chyyran]

I got the hook working with windows-rs but not with winapi. Might be a winapi issue then.

[lostmsu]

@chyyran can you share the working example with Present hooked? Do you mind if I take your Present hook code above and add it under examples to this library (e.g. detour-rs)?

[chyyran]

Well the present hook above doesn't work. I can share my working code but it's been abstracted under a couple layers of macros and not license compatible with this detour-rs (MPL2).

[lostmsu]

@chyyran I adapted the code above to work by using windows-rs as suggested. Can you give a permission to add it to examples?

[chyyran]

Yes, you can use the code snippet linked in https://github.com/darfink/detour-rs/issues/29#issue-1126667018 under the terms of the MIT License