win-capture: D3D12 swap chain queue usage

Attempt to schedule shared texture copies against the command queue that
the game's swap chain uses to try to reduce artifacts. The heuristics
for obtaining the queue are not perfect, so provide a toggle to use the
previous behavior.

plugins/win-capture/data/locale/en-US.ini

@@ -34,4 +34,6 @@ GameCapture.HookRate.Slow="Slow"
 GameCapture.HookRate.Normal="Normal (recommended)"
 GameCapture.HookRate.Fast="Fast"
 GameCapture.HookRate.Fastest="Fastest"
+GameCapture.D3D12UseSwapQueue="Attempt to fix synchronization for Direct3D 12"
+GameCapture.D3D12UseSwapQueue.Long="Fix is best effort, and not perfect. Disable if game becomes unstable."
 Mode="Mode"

plugins/win-capture/game-capture.c

@@ -27,17 +27,18 @@
 
 /* clang-format off */
 
-#define SETTING_MODE             "capture_mode"
-#define SETTING_CAPTURE_WINDOW   "window"
-#define SETTING_ACTIVE_WINDOW    "active_window"
-#define SETTING_WINDOW_PRIORITY  "priority"
-#define SETTING_COMPATIBILITY    "sli_compatibility"
-#define SETTING_CURSOR           "capture_cursor"
-#define SETTING_TRANSPARENCY     "allow_transparency"
-#define SETTING_LIMIT_FRAMERATE  "limit_framerate"
-#define SETTING_CAPTURE_OVERLAYS "capture_overlays"
-#define SETTING_ANTI_CHEAT_HOOK  "anti_cheat_hook"
-#define SETTING_HOOK_RATE        "hook_rate"
+#define SETTING_MODE                 "capture_mode"
+#define SETTING_CAPTURE_WINDOW       "window"
+#define SETTING_ACTIVE_WINDOW        "active_window"
+#define SETTING_WINDOW_PRIORITY      "priority"
+#define SETTING_COMPATIBILITY        "sli_compatibility"
+#define SETTING_CURSOR               "capture_cursor"
+#define SETTING_TRANSPARENCY         "allow_transparency"
+#define SETTING_LIMIT_FRAMERATE      "limit_framerate"
+#define SETTING_CAPTURE_OVERLAYS     "capture_overlays"
+#define SETTING_ANTI_CHEAT_HOOK      "anti_cheat_hook"
+#define SETTING_D3D12_USE_SWAP_QUEUE "d3d12_use_swap_queue"
+#define SETTING_HOOK_RATE            "hook_rate"
 
 /* deprecated */
 #define SETTING_ANY_FULLSCREEN   "capture_any_fullscreen"
@@ -68,6 +69,8 @@
 #define TEXT_HOOK_RATE_NORMAL    obs_module_text("GameCapture.HookRate.Normal")
 #define TEXT_HOOK_RATE_FAST      obs_module_text("GameCapture.HookRate.Fast")
 #define TEXT_HOOK_RATE_FASTEST   obs_module_text("GameCapture.HookRate.Fastest")
+#define TEXT_D3D12_SWAP_QUEUE    obs_module_text("GameCapture.D3D12UseSwapQueue")
+#define TEXT_D3D12_SWAP_QUEUE_LONG obs_module_text("GameCapture.D3D12UseSwapQueue.Long")
 
 #define TEXT_MODE_ANY            TEXT_ANY_FULLSCREEN
 #define TEXT_MODE_WINDOW         obs_module_text("GameCapture.CaptureWindow")
@@ -106,6 +109,7 @@ struct game_capture_config {
 	bool limit_framerate;
 	bool capture_overlays;
 	bool anticheat_hook;
+	bool d3d12_use_swap_queue;
 	enum hook_rate hook_rate;
 };
 
@@ -418,6 +422,8 @@ static inline void get_config(struct game_capture_config *cfg,
 		obs_data_get_bool(settings, SETTING_CAPTURE_OVERLAYS);
 	cfg->anticheat_hook =
 		obs_data_get_bool(settings, SETTING_ANTI_CHEAT_HOOK);
+	cfg->d3d12_use_swap_queue =
+		obs_data_get_bool(settings, SETTING_D3D12_USE_SWAP_QUEUE);
 	cfg->hook_rate =
 		(enum hook_rate)obs_data_get_int(settings, SETTING_HOOK_RATE);
 }
@@ -451,6 +457,9 @@ static inline bool capture_needs_reset(struct game_capture_config *cfg1,
 
 	} else if (cfg1->capture_overlays != cfg2->capture_overlays) {
 		return true;
+
+	} else if (cfg1->d3d12_use_swap_queue != cfg2->d3d12_use_swap_queue) {
+		return true;
 	}
 
 	return false;
@@ -745,6 +754,8 @@ static inline bool init_hook_info(struct game_capture *gc)
 	gc->global_hook_info->force_shmem = gc->config.force_shmem;
 	gc->global_hook_info->UNUSED_use_scale = false;
 	gc->global_hook_info->allow_srgb_alias = true;
+	gc->global_hook_info->d3d12_use_swap_queue =
+		gc->config.d3d12_use_swap_queue;
 	reset_frame_interval(gc);
 
 	obs_enter_graphics();
@@ -1866,6 +1877,7 @@ static void game_capture_defaults(obs_data_t *settings)
 	obs_data_set_default_bool(settings, SETTING_LIMIT_FRAMERATE, false);
 	obs_data_set_default_bool(settings, SETTING_CAPTURE_OVERLAYS, false);
 	obs_data_set_default_bool(settings, SETTING_ANTI_CHEAT_HOOK, true);
+	obs_data_set_default_bool(settings, SETTING_D3D12_USE_SWAP_QUEUE, true);
 	obs_data_set_default_int(settings, SETTING_HOOK_RATE,
 				 (int)HOOK_RATE_NORMAL);
 }
@@ -2050,6 +2062,10 @@ static obs_properties_t *game_capture_properties(void *data)
 	obs_properties_add_bool(ppts, SETTING_CAPTURE_OVERLAYS,
 				TEXT_CAPTURE_OVERLAYS);
 
+	p = obs_properties_add_bool(ppts, SETTING_D3D12_USE_SWAP_QUEUE,
+				    TEXT_D3D12_SWAP_QUEUE);
+	obs_property_set_long_description(p, TEXT_D3D12_SWAP_QUEUE_LONG);
+
 	p = obs_properties_add_list(ppts, SETTING_HOOK_RATE, TEXT_HOOK_RATE,
 				    OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_INT);
 	obs_property_list_add_int(p, TEXT_HOOK_RATE_SLOW, HOOK_RATE_SLOW);

plugins/win-capture/graphics-hook-info.h

@@ -40,6 +40,10 @@ struct d3d9_offsets {
 	uint32_t is_d3d9ex_clsoff;
 };
 
+struct d3d12_offsets {
+	uint32_t execute_command_lists;
+};
+
 struct dxgi_offsets {
 	uint32_t present;
 	uint32_t resize;
@@ -83,6 +87,7 @@ struct graphics_offsets {
 	struct dxgi_offsets dxgi;
 	struct ddraw_offsets ddraw;
 	struct dxgi_offsets2 dxgi2;
+	struct d3d12_offsets d3d12;
 };
 
 struct hook_info {
@@ -104,6 +109,7 @@ struct hook_info {
 	bool flip;
 
 	/* additional options */
+	bool d3d12_use_swap_queue;
 	uint64_t frame_interval;
 	bool UNUSED_use_scale;
 	bool force_shmem;
@@ -113,7 +119,7 @@ struct hook_info {
 	/* hook addresses */
 	struct graphics_offsets offsets;
 
-	uint32_t reserved[127];
+	uint32_t reserved[126];
 };
 static_assert(sizeof(struct hook_info) == 648, "ABI compatibility");
 

plugins/win-capture/graphics-hook-ver.h

@@ -12,7 +12,7 @@
  * THIS IS YOUR ONLY WARNING. */
 
 #define HOOK_VER_MAJOR 1
-#define HOOK_VER_MINOR 3
+#define HOOK_VER_MINOR 4
 #define HOOK_VER_PATCH 0
 
 #define STRINGIFY(s) #s

plugins/win-capture/graphics-hook/d3d12-capture.cpp

@@ -12,6 +12,11 @@
 
 #define MAX_BACKBUFFERS 8
 
+typedef HRESULT(STDMETHODCALLTYPE *execute_command_lists_t)(
+	ID3D12CommandQueue *, UINT, ID3D12CommandList *const *);
+
+static struct func_hook execute_command_lists;
+
 struct d3d12_data {
 	ID3D12Device *device; /* do not release */
 	uint32_t cx;
@@ -39,6 +44,10 @@ struct d3d12_data {
 
 static struct d3d12_data data = {};
 
+extern thread_local bool dxgi_presenting;
+extern ID3D12CommandQueue *dxgi_possible_swap_queue;
+extern bool dxgi_present_attempted;
+
 void d3d12_free(void)
 {
 	if (data.copy_tex)
@@ -170,8 +179,21 @@ static bool d3d12_init_11on12(void)
 		return false;
 	}
 
-	hr = create_11_on_12(data.device, 0, nullptr, 0, nullptr, 0, 0,
+	IUnknown *queue = nullptr;
+	IUnknown *const *queues = nullptr;
+	UINT num_queues = 0;
+	if (global_hook_info->d3d12_use_swap_queue) {
+		hlog("d3d12_init_11on12: creating 11 device with swap queue");
+		queue = dxgi_possible_swap_queue;
+		queues = &queue;
+		num_queues = 1;
+	} else {
+		hlog("d3d12_init_11on12: creating 11 device without swap queue");
+	}
+
+	hr = create_11_on_12(data.device, 0, nullptr, 0, queues, num_queues, 0,
 			     &data.device11, &data.context11, nullptr);
+
 	if (FAILED(hr)) {
 		hlog_hr("d3d12_init_11on12: failed to create 11 device", hr);
 		return false;
@@ -342,4 +364,96 @@ void d3d12_capture(void *swap_ptr, void *, bool capture_overlay)
 	}
 }
 
+static HRESULT STDMETHODCALLTYPE
+hook_execute_command_lists(ID3D12CommandQueue *queue, UINT NumCommandLists,
+			   ID3D12CommandList *const *ppCommandLists)
+{
+	HRESULT hr;
+
+	if (!dxgi_possible_swap_queue) {
+		if (dxgi_presenting) {
+			hlog("D3D12 queue from present");
+			dxgi_possible_swap_queue = queue;
+		} else if (dxgi_present_attempted &&
+			   (queue->GetDesc().Type ==
+			    D3D12_COMMAND_LIST_TYPE_DIRECT)) {
+			hlog("D3D12 queue from first direct after present");
+			dxgi_possible_swap_queue = queue;
+		}
+	}
+
+	unhook(&execute_command_lists);
+	execute_command_lists_t call =
+		(execute_command_lists_t)execute_command_lists.call_addr;
+	hr = call(queue, NumCommandLists, ppCommandLists);
+	rehook(&execute_command_lists);
+
+	return hr;
+}
+
+static bool manually_get_d3d12_addrs(HMODULE d3d12_module,
+				     void **execute_command_lists_addr)
+{
+	PFN_D3D12_CREATE_DEVICE create =
+		(PFN_D3D12_CREATE_DEVICE)GetProcAddress(d3d12_module,
+							"D3D12CreateDevice");
+	if (!create) {
+		hlog("Failed to load D3D12CreateDevice");
+		return false;
+	}
+
+	bool success = false;
+	ID3D12Device *device;
+	if (SUCCEEDED(create(NULL, D3D_FEATURE_LEVEL_11_0,
+			     IID_PPV_ARGS(&device)))) {
+		D3D12_COMMAND_QUEUE_DESC desc{};
+		ID3D12CommandQueue *queue;
+		HRESULT hr =
+			device->CreateCommandQueue(&desc, IID_PPV_ARGS(&queue));
+		success = SUCCEEDED(hr);
+		if (success) {
+			void **queue_vtable = *(void ***)queue;
+			*execute_command_lists_addr = queue_vtable[10];
+
+			queue->Release();
+		} else {
+			hlog("Failed to create D3D12 command queue");
+		}
+
+		device->Release();
+	} else {
+		hlog("Failed to create D3D12 device");
+	}
+
+	return success;
+}
+
+bool hook_d3d12(void)
+{
+	HMODULE d3d12_module = get_system_module("d3d12.dll");
+	if (!d3d12_module) {
+		return false;
+	}
+
+	void *execute_command_lists_addr = nullptr;
+	if (!manually_get_d3d12_addrs(d3d12_module,
+				      &execute_command_lists_addr)) {
+		hlog("Failed to get D3D12 values");
+		return true;
+	}
+
+	if (!execute_command_lists_addr) {
+		hlog("Invalid D3D12 values");
+		return true;
+	}
+
+	hook_init(&execute_command_lists, execute_command_lists_addr,
+		  (void *)hook_execute_command_lists,
+		  "ID3D12CommandQueue::ExecuteCommandLists");
+	rehook(&execute_command_lists);
+
+	hlog("Hooked D3D12");
+	return true;
+}
+
 #endif

plugins/win-capture/graphics-hook/dxgi-capture.cpp

@@ -23,6 +23,10 @@ static struct func_hook resize_buffers;
 static struct func_hook present;
 static struct func_hook present1;
 
+thread_local bool dxgi_presenting = false;
+ID3D12CommandQueue *dxgi_possible_swap_queue = nullptr;
+bool dxgi_present_attempted = false;
+
 struct dxgi_swap_data {
 	IDXGISwapChain *swap;
 	void (*capture)(void *, void *, bool);
@@ -71,11 +75,14 @@ static bool setup_dxgi(IDXGISwapChain *swap)
 #if COMPILE_D3D12_HOOK
 	hr = swap->GetDevice(__uuidof(ID3D12Device), (void **)&device);
 	if (SUCCEEDED(hr)) {
-		data.swap = swap;
-		data.capture = d3d12_capture;
-		data.free = d3d12_free;
-		device->Release();
-		return true;
+		if (!global_hook_info->d3d12_use_swap_queue ||
+		    dxgi_possible_swap_queue) {
+			data.swap = swap;
+			data.capture = d3d12_capture;
+			data.free = d3d12_free;
+			device->Release();
+			return true;
+		}
 	}
 #endif
 
@@ -94,6 +101,8 @@ static ULONG STDMETHODCALLTYPE hook_release(IUnknown *unknown)
 		data.swap = nullptr;
 		data.free = nullptr;
 		data.capture = nullptr;
+		dxgi_possible_swap_queue = nullptr;
+		dxgi_present_attempted = false;
 	}
 
 	return refs;
@@ -115,6 +124,8 @@ static HRESULT STDMETHODCALLTYPE hook_resize_buffers(IDXGISwapChain *swap,
 	data.swap = nullptr;
 	data.free = nullptr;
 	data.capture = nullptr;
+	dxgi_possible_swap_queue = nullptr;
+	dxgi_present_attempted = false;
 
 	unhook(&resize_buffers);
 	resize_buffers_t call = (resize_buffers_t)resize_buffers.call_addr;
@@ -161,10 +172,13 @@ static HRESULT STDMETHODCALLTYPE hook_present(IDXGISwapChain *swap,
 		}
 	}
 
+	dxgi_presenting = true;
 	unhook(&present);
 	present_t call = (present_t)present.call_addr;
 	hr = call(swap, sync_interval, flags);
 	rehook(&present);
+	dxgi_presenting = false;
+	dxgi_present_attempted = true;
 
 	if (capture && capture_overlay) {
 		/*
@@ -215,10 +229,13 @@ hook_present1(IDXGISwapChain1 *swap, UINT sync_interval, UINT flags,
 		}
 	}
 
+	dxgi_presenting = true;
 	unhook(&present1);
 	present1_t call = (present1_t)present1.call_addr;
 	hr = call(swap, sync_interval, flags, params);
 	rehook(&present1);
+	dxgi_presenting = false;
+	dxgi_present_attempted = true;
 
 	if (capture && capture_overlay) {
 		if (resize_buffers_called) {

plugins/win-capture/graphics-hook/graphics-hook.c

@@ -317,6 +317,7 @@ static inline bool attempt_hook(void)
 	//static bool ddraw_hooked = false;
 	static bool d3d8_hooked = false;
 	static bool d3d9_hooked = false;
+	static bool d3d12_hooked = false;
 	static bool dxgi_hooked = false;
 	static bool gl_hooked = false;
 #if COMPILE_VULKAN_HOOK
@@ -329,6 +330,12 @@ static inline bool attempt_hook(void)
 	}
 #endif //COMPILE_VULKAN_HOOK
 
+#if COMPILE_D3D12_HOOK
+	if (!d3d12_hooked) {
+		d3d12_hooked = hook_d3d12();
+	}
+#endif
+
 	if (!d3d9_hooked) {
 		if (!d3d9_hookable()) {
 			DbgOut("[OBS] no D3D9 hook address found!\n");

plugins/win-capture/graphics-hook/graphics-hook.h

@@ -43,6 +43,7 @@ extern void shmem_texture_data_unlock(int idx);
 extern bool hook_ddraw(void);
 extern bool hook_d3d8(void);
 extern bool hook_d3d9(void);
+extern bool hook_d3d12(void);
 extern bool hook_dxgi(void);
 extern bool hook_gl(void);
 #if COMPILE_VULKAN_HOOK