libobs-d3d11: Set maximum GPU priority

(This commit also modifies the UI)

This solves the issue where OBS would be deprioritized by Windows over
fullscreen games, causing OBS to lag out whereas the games would still
run fine.

UI/obs-app.cpp

@@ -1911,6 +1911,18 @@ static void load_debug_privilege(void)
 				      NULL);
 	}
 
+	if (!!LookupPrivilegeValue(NULL, SE_INC_BASE_PRIORITY_NAME, &val)) {
+		tp.PrivilegeCount = 1;
+		tp.Privileges[0].Luid = val;
+		tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
+
+		if (!AdjustTokenPrivileges(token, false, &tp, sizeof(tp), NULL,
+					   NULL)) {
+			blog(LOG_INFO, "Could not set privilege to "
+				       "increase GPU priority");
+		}
+	}
+
 	CloseHandle(token);
 }
 #endif

libobs-d3d11/CMakeLists.txt

@@ -4,6 +4,20 @@ include_directories(SYSTEM "${CMAKE_SOURCE_DIR}/libobs")
 
 add_definitions(-DLIBOBS_EXPORTS)
 
+if(NOT DEFINED GPU_PRIORITY_VAL OR "${GPU_PRIORITY_VAL}" STREQUAL "" OR
+   "${GPU_PRIORITY_VAL}" STREQUAL "0")
+	set(USE_GPU_PRIORITY FALSE)
+	set(GPU_PRIORITY_VAL "0")
+else()
+	set(USE_GPU_PRIORITY TRUE)
+endif()
+
+configure_file(
+	"${CMAKE_CURRENT_SOURCE_DIR}/d3d11-config.h.in"
+	"${CMAKE_CURRENT_BINARY_DIR}/d3d11-config.h")
+
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+
 set(libobs-d3d11_SOURCES
 	d3d11-indexbuffer.cpp
 	d3d11-samplerstate.cpp
@@ -18,6 +32,7 @@ set(libobs-d3d11_SOURCES
 	d3d11-zstencilbuffer.cpp)
 
 set(libobs-d3d11_HEADERS
+	${CMAKE_CURRENT_BINARY_DIR}/d3d11-config.h
 	d3d11-shaderprocessor.hpp
 	d3d11-subsystem.hpp)
 

libobs-d3d11/d3d11-config.h.in

@@ -0,0 +1,20 @@
+#pragma once
+
+#ifndef TRUE
+#define TRUE 1
+#endif
+
+#ifndef ON
+#define ON 1
+#endif
+
+#ifndef FALSE
+#define FALSE 0
+#endif
+
+#ifndef OFF
+#define OFF 0
+#endif
+
+#define USE_GPU_PRIORITY @USE_GPU_PRIORITY@
+#define GPU_PRIORITY_VAL @GPU_PRIORITY_VAL@

libobs-d3d11/d3d11-subsystem.cpp

@@ -21,8 +21,10 @@
 #include <util/dstr.h>
 #include <util/util.hpp>
 #include <graphics/matrix3.h>
+#include <winternl.h>
 #include <d3d9.h>
 #include "d3d11-subsystem.hpp"
+#include "d3d11-config.h"
 
 struct UnsupportedHWError : HRError {
 	inline UnsupportedHWError(const char *str, HRESULT hr)
@@ -351,6 +353,58 @@ try {
 	return false;
 }
 
+#if USE_GPU_PRIORITY
+static bool set_priority(ID3D11Device *device)
+{
+	typedef enum _D3DKMT_SCHEDULINGPRIORITYCLASS {
+		D3DKMT_SCHEDULINGPRIORITYCLASS_IDLE,
+		D3DKMT_SCHEDULINGPRIORITYCLASS_BELOW_NORMAL,
+		D3DKMT_SCHEDULINGPRIORITYCLASS_NORMAL,
+		D3DKMT_SCHEDULINGPRIORITYCLASS_ABOVE_NORMAL,
+		D3DKMT_SCHEDULINGPRIORITYCLASS_HIGH,
+		D3DKMT_SCHEDULINGPRIORITYCLASS_REALTIME
+	} D3DKMT_SCHEDULINGPRIORITYCLASS;
+
+	ComQIPtr<IDXGIDevice> dxgiDevice(device);
+	if (!dxgiDevice) {
+		blog(LOG_DEBUG, "%s: Failed to get IDXGIDevice", __FUNCTION__);
+		return false;
+	}
+
+	HMODULE gdi32 = GetModuleHandleW(L"GDI32");
+	if (!gdi32) {
+		blog(LOG_DEBUG, "%s: Failed to get GDI32", __FUNCTION__);
+		return false;
+	}
+
+	NTSTATUS (*d3dkmt_spspc)(HANDLE, D3DKMT_SCHEDULINGPRIORITYCLASS);
+	d3dkmt_spspc = (decltype(d3dkmt_spspc))GetProcAddress(
+		gdi32, "D3DKMTSetProcessSchedulingPriorityClass");
+	if (!d3dkmt_spspc) {
+		blog(LOG_DEBUG, "%s: Failed to get d3dkmt_spspc", __FUNCTION__);
+		return false;
+	}
+
+	NTSTATUS status = d3dkmt_spspc(GetCurrentProcess(),
+				       D3DKMT_SCHEDULINGPRIORITYCLASS_REALTIME);
+	if (status != 0) {
+		blog(LOG_DEBUG, "%s: Failed to set process priority class: %d",
+		     __FUNCTION__, (int)status);
+		return false;
+	}
+
+	HRESULT hr = dxgiDevice->SetGPUThreadPriority(GPU_PRIORITY_VAL);
+	if (FAILED(hr)) {
+		blog(LOG_DEBUG, "%s: SetGPUThreadPriority failed",
+		     __FUNCTION__);
+		return false;
+	}
+
+	blog(LOG_INFO, "D3D11 GPU priority setup success");
+	return true;
+}
+#endif
+
 void gs_device::InitDevice(uint32_t adapterIdx)
 {
 	wstring adapterName;
@@ -385,6 +439,13 @@ void gs_device::InitDevice(uint32_t adapterIdx)
 	blog(LOG_INFO, "D3D11 loaded successfully, feature level used: %x",
 	     (unsigned int)levelUsed);
 
+	/* adjust gpu thread priority */
+#if USE_GPU_PRIORITY
+	if (!set_priority(device)) {
+		blog(LOG_INFO, "D3D11 GPU priority setup failed (not admin?)");
+	}
+#endif
+
 	/* ---------------------------------------- */
 	/* check for nv12 texture output support    */