win-capture: Add trick to ensure game cap. shared tex support

This new trick forgoes the use of patches and allows the ability to use
shared GPU resources despite being a Direct3D 9.0c context.

plugins/win-capture/get-graphics-offsets/d3d9-offsets.cpp

@@ -80,17 +80,107 @@ static inline void d3d9_free(d3d9_info &info)
 		DestroyWindow(info.hwnd);
 }
 
+#ifdef _WIN64
+
+#define CMP_SIZE 21
+
+static const uint8_t mask[CMP_SIZE] =
+{0xF8, 0xFF, 0xC0, 0x00, 0x00, 0x00, 0x00,
+ 0xFF, 0xC0, 0x00, 0x00, 0x00, 0x00,
+ 0xFF, 0x00,
+ 0xF8, 0xF8, 0x00, 0x00, 0x00, 0x00};
+
+static const uint8_t mask_cmp[CMP_SIZE] =
+{0x48, 0x8B, 0x80, 0x00, 0x00, 0x00, 0x00,
+ 0x39, 0x80, 0x00, 0x00, 0x00, 0x00,
+ 0x75, 0x00,
+ 0x40, 0xB8, 0x00, 0x00, 0x00, 0x00};
+#else
+
+#define CMP_SIZE 19
+
+static const uint8_t mask[CMP_SIZE] =
+{0xFF, 0xC0, 0x00, 0x00, 0x00, 0x00,
+ 0xFF, 0xC0, 0x00, 0x00, 0x00, 0x00,
+ 0xFF, 0x00,
+ 0xFF, 0x00, 0x00, 0x00, 0x00};
+
+static const uint8_t mask_cmp[CMP_SIZE] =
+{0x8B, 0x80, 0x00, 0x00, 0x00, 0x00,
+ 0x39, 0x80, 0x00, 0x00, 0x00, 0x00,
+ 0x75, 0x00,
+ 0x68, 0x00, 0x00, 0x00, 0x00};
+#endif
+
+#define MAX_FUNC_SCAN_BYTES 200
+
+static inline bool pattern_matches(uint8_t *byte)
+{
+	for (size_t i = 0; i < CMP_SIZE; i++) {
+		if ((byte[i] & mask[i]) != mask_cmp[i])
+			return false;
+	}
+
+	return true;
+}
+
 void get_d3d9_offsets(struct d3d9_offsets *offsets)
 {
 	d3d9_info info    = {};
 	bool      success = d3d9_init(info);
 
 	if (success) {
+		uint8_t **vt = *(uint8_t***)info.device;
+		uint8_t *crr = vt[125];
+
 		offsets->present = vtable_offset(info.module, info.device, 17);
 		offsets->present_ex = vtable_offset(info.module, info.device,
 				121);
 		offsets->present_swap = vtable_offset(info.module, info.swap,
 				3);
+
+		for (size_t i = 0; i < MAX_FUNC_SCAN_BYTES; i++) {
+			if (pattern_matches(&crr[i])) {
+#define get_offset(x) *(uint32_t*)&crr[i + x]
+#ifdef _WIN64
+				uint32_t off1 = get_offset(3);
+				uint32_t off2 = get_offset(9);
+#else
+				uint32_t off1 = get_offset(2);
+				uint32_t off2 = get_offset(8);
+#endif
+
+				/* check to make sure offsets are within
+				 * expected values */
+				if (off1 > 0xFFFF || off2 > 0xFFFF)
+					break;
+
+				/* check to make sure offsets actually point
+				 * toward expected data */
+#ifdef _MSC_VER
+				__try {
+					uint8_t *ptr = (uint8_t*)(info.device);
+
+					uint8_t *d3d9_ptr =
+						*(uint8_t**)(ptr + off1);
+					if (d3d9_ptr != (uint8_t*)info.d3d9ex)
+						break;
+
+					BOOL &is_d3d9ex =
+						*(BOOL*)(d3d9_ptr + off2);
+					if (is_d3d9ex != TRUE)
+						break;
+
+				} __except(EXCEPTION_EXECUTE_HANDLER) {
+					break;
+				}
+#endif
+
+				offsets->d3d9_clsoff = off1;
+				offsets->is_d3d9ex_clsoff = off2;
+				break;
+			}
+		}
 	}
 
 	d3d9_free(info);

plugins/win-capture/get-graphics-offsets/get-graphics-offsets.c

@@ -31,6 +31,8 @@ int main(int argc, char *argv[])
 	printf("present=0x%"PRIx32"\n", d3d9.present);
 	printf("present_ex=0x%"PRIx32"\n", d3d9.present_ex);
 	printf("present_swap=0x%"PRIx32"\n", d3d9.present_swap);
+	printf("d3d9_clsoff=0x%"PRIx32"\n", d3d9.d3d9_clsoff);
+	printf("is_d3d9ex_clsoff=0x%"PRIx32"\n", d3d9.is_d3d9ex_clsoff);
 	printf("[dxgi]\n");
 	printf("present=0x%"PRIx32"\n", dxgi.present);
 	printf("resize=0x%"PRIx32"\n", dxgi.resize);

plugins/win-capture/graphics-hook-info.h

@@ -33,6 +33,8 @@ struct d3d9_offsets {
 	uint32_t present;
 	uint32_t present_ex;
 	uint32_t present_swap;
+	uint32_t d3d9_clsoff;
+	uint32_t is_d3d9ex_clsoff;
 };
 
 struct dxgi_offsets {

plugins/win-capture/graphics-hook/d3d9-capture.cpp

@@ -223,14 +223,30 @@ static inline bool d3d9_shtex_init_shtex()
 
 static inline bool d3d9_shtex_init_copytex()
 {
-	uint8_t *patch_addr = get_d3d9_patch_addr(data.d3d9, data.patch);
+	struct d3d9_offsets offsets = global_hook_info->offsets.d3d9;
+	uint8_t *patch_addr = nullptr;
+	BOOL *p_is_d3d9 = nullptr;
 	uint8_t saved_data[MAX_PATCH_SIZE];
 	size_t patch_size = 0;
+	BOOL was_d3d9ex = false;
 	IDirect3DTexture9 *tex;
 	DWORD protect_val;
 	HRESULT hr;
 
-	if (patch_addr) {
+	if (offsets.d3d9_clsoff && offsets.is_d3d9ex_clsoff) {
+		uint8_t *device_ptr = (uint8_t*)(data.device);
+		uint8_t *d3d9_ptr =
+			*(uint8_t**)(device_ptr + offsets.d3d9_clsoff);
+		p_is_d3d9 = (BOOL*)(d3d9_ptr + offsets.is_d3d9ex_clsoff);
+	} else {
+		patch_addr = get_d3d9_patch_addr(data.d3d9, data.patch);
+	}
+
+	if (p_is_d3d9) {
+		was_d3d9ex = *p_is_d3d9;
+		*p_is_d3d9 = true;
+
+	} else if (patch_addr) {
 		patch_size = patch[data.patch].size;
 		VirtualProtect(patch_addr, patch_size, PAGE_EXECUTE_READWRITE,
 				&protect_val);
@@ -242,7 +258,10 @@ static inline bool d3d9_shtex_init_copytex()
 			D3DUSAGE_RENDERTARGET, data.d3d9_format,
 			D3DPOOL_DEFAULT, &tex, &data.handle);
 
-	if (patch_addr && patch_size) {
+	if (p_is_d3d9) {
+		*p_is_d3d9 = was_d3d9ex;
+
+	} else if (patch_addr && patch_size) {
 		memcpy(patch_addr, saved_data, patch_size);
 		VirtualProtect(patch_addr, patch_size, protect_val,
 				&protect_val);
@@ -449,6 +468,9 @@ static bool d3d9_init_format_swapchain(uint32_t &cx, uint32_t &cy, HWND &window)
 static void d3d9_init(IDirect3DDevice9 *device)
 {
 	IDirect3DDevice9Ex *d3d9ex = nullptr;
+	bool has_d3d9ex_bool_offset =
+		global_hook_info->offsets.d3d9.d3d9_clsoff &&
+		global_hook_info->offsets.d3d9.is_d3d9ex_clsoff;
 	bool success;
 	uint32_t cx = 0;
 	uint32_t cy = 0;
@@ -463,8 +485,10 @@ static void d3d9_init(IDirect3DDevice9 *device)
 	if (SUCCEEDED(hr)) {
 		d3d9ex->Release();
 		data.patch = -1;
-	} else {
+	} else if (!has_d3d9ex_bool_offset) {
 		data.patch = get_d3d9_patch(data.d3d9);
+	} else {
+		data.patch = -1;
 	}
 
 	if (!d3d9_init_format_backbuffer(cx, cy, window)) {
@@ -473,7 +497,8 @@ static void d3d9_init(IDirect3DDevice9 *device)
 		}
 	}
 
-	if (global_hook_info->force_shmem || (!d3d9ex && data.patch == -1)) {
+	if (global_hook_info->force_shmem ||
+	    (!d3d9ex && data.patch == -1 && !has_d3d9ex_bool_offset)) {
 		success = d3d9_shmem_init(cx, cy, window);
 	} else {
 		success = d3d9_shtex_init(cx, cy, window);

plugins/win-capture/load-graphics-offsets.c

@@ -30,6 +30,10 @@ static inline bool load_offsets_from_string(struct graphics_offsets *offsets,
 		(uint32_t)config_get_uint(config, "d3d9", "present_ex");
 	offsets->d3d9.present_swap =
 		(uint32_t)config_get_uint(config, "d3d9", "present_swap");
+	offsets->d3d9.d3d9_clsoff =
+		(uint32_t)config_get_uint(config, "d3d9", "d3d9_clsoff");
+	offsets->d3d9.is_d3d9ex_clsoff =
+		(uint32_t)config_get_uint(config, "d3d9", "is_d3d9ex_clsoff");
 
 	offsets->dxgi.present =
 		(uint32_t)config_get_uint(config, "dxgi", "present");