obs-studio/plugins/win-capture/get-graphics-offsets/d3d9-offsets.cpp

#include <stdio.h>
#include <windows.h>
#include <d3d9.h>
#include "get-graphics-offsets.h"

typedef HRESULT(WINAPI *d3d9createex_t)(UINT, IDirect3D9Ex **);

struct d3d9_info {
	HMODULE module;
	HWND hwnd;
	IDirect3D9Ex *d3d9ex;
	IDirect3DDevice9Ex *device;
	IDirect3DSwapChain9 *swap;
};

static inline bool d3d9_init(d3d9_info &info)
{
	d3d9createex_t create;
	HRESULT hr;

	info.hwnd = CreateWindowExA(0, DUMMY_WNDCLASS, "d3d9 get-offset window",
				    WS_POPUP, 0, 0, 1, 1, nullptr, nullptr,
				    GetModuleHandleA(nullptr), nullptr);
	if (!info.hwnd) {
		return false;
	}

	info.module = LoadLibraryA("d3d9.dll");
	if (!info.module) {
		return false;
	}

	create = (d3d9createex_t)GetProcAddress(info.module,
						"Direct3DCreate9Ex");
	if (!create) {
		return false;
	}

	hr = create(D3D_SDK_VERSION, &info.d3d9ex);
	if (FAILED(hr)) {
		return false;
	}

	D3DPRESENT_PARAMETERS pp = {};
	pp.Windowed = true;
	pp.SwapEffect = D3DSWAPEFFECT_FLIP;
	pp.BackBufferFormat = D3DFMT_A8R8G8B8;
	pp.BackBufferWidth = 2;
	pp.BackBufferHeight = 2;
	pp.BackBufferCount = 1;
	pp.hDeviceWindow = info.hwnd;
	pp.PresentationInterval = D3DPRESENT_INTERVAL_IMMEDIATE;

	hr = info.d3d9ex->CreateDeviceEx(
		D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, info.hwnd,
		D3DCREATE_HARDWARE_VERTEXPROCESSING | D3DCREATE_NOWINDOWCHANGES,
		&pp, nullptr, &info.device);
	if (FAILED(hr)) {
		return false;
	}

	hr = info.device->GetSwapChain(0, &info.swap);
	if (FAILED(hr)) {
		return false;
	}

	return true;
}

static inline void d3d9_free(d3d9_info &info)
{
	if (info.swap)
		info.swap->Release();
	if (info.device)
		info.device->Release();
	if (info.d3d9ex)
		info.d3d9ex->Release();
	if (info.hwnd)
		DestroyWindow(info.hwnd);
}

#ifdef _WIN64

#define MAX_CMP_SIZE 22

static const uint8_t mask[][MAX_CMP_SIZE] = {
	{0xF8, 0xFF, 0xC0, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00,
	 0x00, 0x00, 0x00, 0xFF, 0x00, 0xF8, 0xF8, 0x00, 0x00, 0x00, 0x00},
	{0xF8, 0xFF, 0xC0, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xC0, 0x00, 0x00,
	 0x00, 0x00, 0xFF, 0x00, 0xF8, 0xF8, 0x00, 0x00, 0x00, 0x00},
	{0xF8, 0xFF, 0xC0, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xC0, 0x00, 0x00,
	 0x00, 0x00, 0xFF, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00},
};

static const uint8_t mask_cmp[][MAX_CMP_SIZE] = {
	/*
	* Windows 7
	* 48 8B 83 B8 3D 00 00                    mov     rax, [rbx+3DB8h]
	* 44 39 B8 68 50 00 00                    cmp     [rax+5068h], r15d
	* 75 12                                   jnz     short loc_7FF7AA90530
	* 41 B8 F9 19 00 00                       mov     r8d, 19F9h
	*/
	{0x48, 0x8B, 0x80, 0x00, 0x00, 0x00, 0x00, 0x44, 0x39, 0x00, 0x00,
	 0x00, 0x00, 0x00, 0x75, 0x00, 0x40, 0xB8, 0x00, 0x00, 0x00, 0x00},
	/*
	* Windows ???+
	* 49 8B 87 78 41 00 00                    mov     rax, [r15+4178h]
	* 39 98 E0 51 00 00                       cmp     [rax+51E0h], ebx
	* 75 12                                   jnz     short loc_1800AEC9C
	* 41 B9 C3 1A 00 00                       mov     r9d, 1AC3h
	*/
	{0x48, 0x8B, 0x80, 0x00, 0x00, 0x00, 0x00, 0x39, 0x80, 0x00, 0x00,
	 0x00, 0x00, 0x75, 0x00, 0x40, 0xB8, 0x00, 0x00, 0x00, 0x00},
	/*
	* Windows 10 April 2018
	* 49 8B 87 58 40 00 00                 mov     rax, [r15+4058h]
	* 39 98 C0 53 00 00                    cmp     [rax+53C0h], ebx
	* 75 12                                jnz     short loc_1800A7FEC
	* 48 8D 15 7F B6 09 00                 lea     rdx, addrErrorMsg
	*
	* Note: different instructions, last byte skipped due to MAX_CMP_SIZE
	*/
	{0x48, 0x8B, 0x80, 0x00, 0x00, 0x00, 0x00, 0x39, 0x80, 0x00, 0x00,
	 0x00, 0x00, 0x75, 0x00, 0x48, 0x8D, 0x00, 0x00, 0x00, 0x00}};

// Offset into the code for the numbers we're interested in
static const uint32_t code_offsets[][2] = {
	{3, 10},
	{3, 9},
	{3, 9},
};
#else

#define MAX_CMP_SIZE 20

static const uint8_t mask[][MAX_CMP_SIZE] = {
	{0xFF, 0xC0, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xC0, 0x00, 0x00, 0x00, 0x00,
	 0xFF, 0x00, 0xFF, 0x00, 0x00, 0x00, 0x00},
	{0xFF, 0xC0, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xC0, 0x00, 0x00,
	 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0x00, 0x00},
	{0xFF, 0xC0, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xC0, 0x00, 0x00,
	 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0x00, 0x00},
};

static const uint8_t mask_cmp[][MAX_CMP_SIZE] = {
	/*
	* Windows 7+
	* 8B 83 E8 29 00 00      mov     eax, [ebx+29E8h]
	* 39 B0 80 4B 00 00      cmp     [eax+4B80h], esi
	* 75 14                  jnz     short loc_754CD9E1
	* 68 F9 19 00 00         push    19F9h
	*/
	{0x8B, 0x80, 0x00, 0x00, 0x00, 0x00, 0x39, 0x80, 0x00, 0x00, 0x00, 0x00,
	 0x75, 0x00, 0x68, 0x00, 0x00, 0x00, 0x00},

	/* Windows 10 Creator's Update+
	* 8B 86 F8 2B 00 00      mov     eax, [esi+2BF8h]
	* 83 B8 00 4D 00 00 00   cmp     dword ptr [eax+4D00h], 0
	* 75 0F                  jnz     short loc_100D793C
	* 68 C3 1A 00 00         push    1AC3h
	*/
	{0x8B, 0x80, 0x00, 0x00, 0x00, 0x00, 0x83, 0x80, 0x00, 0x00,
	 0x00, 0x00, 0x00, 0x75, 0x00, 0x68, 0x00, 0x00, 0x00, 0x00},

	/*
	* Windows 10 April 2018 Update
	* 8B 86 68 2B 00 00         mov     eax, [esi+2B68h]
	* 83 B8 F4 4D 00 00 00      cmp     dword ptr [eax+4DF4h], 0
	* 75 0F                     jnz     short loc_100D9A9C
	* BA 08 71 01 10            mov     edx, offset errMsg
	*/
	{0x8B, 0x80, 0x00, 0x00, 0x00, 0x00, 0x83, 0x80, 0x00, 0x00,
	 0x00, 0x00, 0x00, 0x75, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00}};

// Offset into the code for the numbers we're interested in
static const uint32_t code_offsets[][2] = {
	{2, 8},
	{2, 8},
	{2, 8},
};
#endif

#define MAX_FUNC_SCAN_BYTES 200

static inline bool pattern_matches(uint8_t *byte, uint32_t *offset1,
				   uint32_t *offset2)
{
	for (size_t j = 0; j < sizeof(mask) / sizeof(mask[0]); j++) {
		for (size_t i = 0; i < MAX_CMP_SIZE; i++) {
			if ((byte[i] & mask[j][i]) != mask_cmp[j][i])
				goto next_signature;
		}

		*offset1 = code_offsets[j][0];
		*offset2 = code_offsets[j][1];

		return true;
	next_signature:;
	}

	return false;
}

void get_d3d9_offsets(struct d3d9_offsets *offsets)
{
	d3d9_info info = {};
	bool success = d3d9_init(info);

	if (success) {
		uint8_t **vt = *(uint8_t ***)info.device;
		uint8_t *crr = vt[125];

		offsets->present = vtable_offset(info.module, info.device, 17);
		offsets->present_ex =
			vtable_offset(info.module, info.device, 121);
		offsets->present_swap =
			vtable_offset(info.module, info.swap, 3);

		uint32_t offset1, offset2;
		for (size_t i = 0; i < MAX_FUNC_SCAN_BYTES; i++) {
			if (pattern_matches(&crr[i], &offset1, &offset2)) {
#define get_offset(x) *(uint32_t *)&crr[i + x]
				uint32_t off1 = get_offset(offset1);
				uint32_t off2 = get_offset(offset2);

				/* check to make sure offsets are within
				 * expected values */
				if (off1 > 0xFFFF || off2 > 0xFFFF)
					break;

					/* check to make sure offsets actually point
				 * toward expected data */
#ifdef _MSC_VER
				__try {
					uint8_t *ptr = (uint8_t *)(info.device);

					uint8_t *d3d9_ptr =
						*(uint8_t **)(ptr + off1);
					if (d3d9_ptr != (uint8_t *)info.d3d9ex)
						break;

					BOOL &is_d3d9ex =
						*(BOOL *)(d3d9_ptr + off2);
					if (is_d3d9ex != TRUE)
						break;

				} __except (EXCEPTION_EXECUTE_HANDLER) {
					break;
				}
#endif

				offsets->d3d9_clsoff = off1;
				offsets->is_d3d9ex_clsoff = off2;
				break;
			}
		}
	}

	d3d9_free(info);
}