win-capture: Implement D3D8 game capture support

plugins/win-capture/game-capture.c

@@ -86,6 +86,7 @@ struct game_capture {
 	bool                          error_acquiring : 1;
 	bool                          dwm_capture : 1;
 	bool                          initial_config : 1;
+	bool                          convert_16bit : 1;
 
 	struct game_capture_config    config;
 
@@ -838,6 +839,94 @@ static inline bool init_capture_data(struct game_capture *gc)
 	return true;
 }
 
+#define PIXEL_16BIT_SIZE 2
+#define PIXEL_32BIT_SIZE 4
+
+static inline uint32_t convert_5_to_8bit(uint16_t val)
+{
+	return (uint32_t)((double)(val & 0x1F) * (255.0/31.0));
+}
+
+static inline uint32_t convert_6_to_8bit(uint16_t val)
+{
+	return (uint32_t)((double)(val & 0x3F) * (255.0/63.0));
+}
+
+static void copy_b5g6r5_tex(struct game_capture *gc, int cur_texture,
+		uint8_t *data, uint32_t pitch)
+{
+	uint8_t *input = gc->texture_buffers[cur_texture];
+	uint32_t gc_cx = gc->cx;
+	uint32_t gc_cy = gc->cy;
+	uint32_t gc_pitch = gc->pitch;
+
+	for (uint32_t y = 0; y < gc_cy; y++) {
+		register uint8_t *in  = input + (gc_pitch * y);
+		register uint8_t *end = input + (gc_cx * PIXEL_16BIT_SIZE);
+		register uint8_t *out = data  + pitch;
+
+		while (in < end) {
+			register uint16_t in_pix = *(uint16_t*)in;
+			register uint32_t out_pix = 0xFF000000;
+
+			out_pix |= convert_5_to_8bit(in_pix);
+			in_pix >>= 5;
+			out_pix |= convert_6_to_8bit(in_pix) << 8;
+			in_pix >>= 6;
+			out_pix |= convert_5_to_8bit(in_pix) << 16;
+
+			*(uint32_t*)out = out_pix;
+
+			in  += PIXEL_16BIT_SIZE;
+			out += PIXEL_32BIT_SIZE;
+		}
+	}
+}
+
+static void copy_b5g5r5a1_tex(struct game_capture *gc, int cur_texture,
+		uint8_t *data, uint32_t pitch)
+{
+	uint8_t *input = gc->texture_buffers[cur_texture];
+	uint32_t gc_cx = gc->cx;
+	uint32_t gc_cy = gc->cy;
+	uint32_t gc_pitch = gc->pitch;
+
+	for (uint32_t y = 0; y < gc_cy; y++) {
+		register uint8_t *in  = input + (gc_pitch * y);
+		register uint8_t *end = input + (gc_cx * PIXEL_16BIT_SIZE);
+		register uint8_t *out = data  + pitch;
+
+		while (in < end) {
+			register uint16_t in_pix = *(uint16_t*)in;
+			register uint32_t out_pix = 0;
+
+			out_pix |= convert_5_to_8bit(in_pix);
+			in_pix >>= 5;
+			out_pix |= convert_5_to_8bit(in_pix) << 8;
+			in_pix >>= 5;
+			out_pix |= convert_5_to_8bit(in_pix) << 16;
+			in_pix >>= 5;
+			out_pix |= (in_pix * 255) << 24;
+
+			*(uint32_t*)out = out_pix;
+
+			in  += PIXEL_16BIT_SIZE;
+			out += PIXEL_32BIT_SIZE;
+		}
+	}
+}
+
+static inline void copy_16bit_tex(struct game_capture *gc, int cur_texture,
+		uint8_t *data, uint32_t pitch)
+{
+	if (gc->global_hook_info->format == DXGI_FORMAT_B5G5R5A1_UNORM) {
+		copy_b5g5r5a1_tex(gc, cur_texture, data, pitch);
+
+	} else if (gc->global_hook_info->format == DXGI_FORMAT_B5G6R5_UNORM) {
+		copy_b5g6r5_tex(gc, cur_texture, data, pitch);
+	}
+}
+
 static void copy_shmem_tex(struct game_capture *gc)
 {
 	int cur_texture = gc->shmem_data->last_tex;
@@ -863,7 +952,10 @@ static void copy_shmem_tex(struct game_capture *gc)
 	}
 
 	if (gs_texture_map(gc->texture, &data, &pitch)) {
-		if (pitch == gc->pitch) {
+		if (gc->convert_16bit) {
+			copy_16bit_tex(gc, cur_texture, data, pitch);
+
+		} else if (pitch == gc->pitch) {
 			memcpy(data, gc->texture_buffers[cur_texture],
 					pitch * gc->cy);
 		} else {
@@ -884,18 +976,29 @@ static void copy_shmem_tex(struct game_capture *gc)
 	ReleaseMutex(mutex);
 }
 
+static inline bool is_16bit_format(uint32_t format)
+{
+	return format == DXGI_FORMAT_B5G5R5A1_UNORM ||
+	       format == DXGI_FORMAT_B5G6R5_UNORM;
+}
+
 static inline bool init_shmem_capture(struct game_capture *gc)
 {
+	enum gs_color_format format;
+
 	gc->texture_buffers[0] =
 		(uint8_t*)gc->data + gc->shmem_data->tex1_offset;
 	gc->texture_buffers[1] =
 		(uint8_t*)gc->data + gc->shmem_data->tex2_offset;
 
+	gc->convert_16bit = is_16bit_format(gc->global_hook_info->format);
+	format = gc->convert_16bit ?
+		GS_BGRA : convert_format(gc->global_hook_info->format);
+
 	obs_enter_graphics();
 	gs_texture_destroy(gc->texture);
-	gc->texture = gs_texture_create(gc->cx, gc->cy,
-			convert_format(gc->global_hook_info->format),
-			1, NULL, GS_DYNAMIC);
+	gc->texture = gs_texture_create(gc->cx, gc->cy, format, 1, NULL,
+			GS_DYNAMIC);
 	obs_leave_graphics();
 
 	if (!gc->texture) {

plugins/win-capture/graphics-hook/CMakeLists.txt

@@ -14,6 +14,7 @@ set(graphics-hook_SOURCES
 	../funchook.c
 	../obfuscate.c
 	gl-capture.c
+	d3d8-capture.cpp
 	d3d9-capture.cpp
 	dxgi-capture.cpp
 	d3d10-capture.cpp

plugins/win-capture/graphics-hook/d3d8-capture.cpp

@@ -0,0 +1,302 @@
+#define _CRT_SECURE_NO_WARNINGS
+#include <dxgi.h>
+
+#include "../d3d8-api/d3d8.h"
+#include "graphics-hook.h"
+#include "../funchook.h"
+
+typedef HRESULT(STDMETHODCALLTYPE *reset_t)(IDirect3DDevice8*,
+		D3DPRESENT_PARAMETERS*);
+typedef HRESULT(STDMETHODCALLTYPE *present_t)(IDirect3DDevice8*,
+		CONST RECT*, CONST RECT*, HWND, CONST RGNDATA*);
+
+static struct func_hook present;
+static struct func_hook reset;
+
+struct d3d8_data {
+	HMODULE                        d3d8;
+	uint32_t                       cx;
+	uint32_t                       cy;
+	D3DFORMAT                      d3d8_format;
+	DXGI_FORMAT                    dxgi_format;
+
+	struct shmem_data              *shmem_info;
+	HWND                           window;
+	uint32_t                       pitch;
+	IDirect3DSurface8              *copy_surfaces[NUM_BUFFERS];
+	bool                           surface_locked[NUM_BUFFERS];
+	int                            cur_surface;
+	int                            copy_wait;
+};
+
+static d3d8_data data = {};
+
+static DXGI_FORMAT d3d8_to_dxgi_format(D3DFORMAT format)
+{
+	switch ((unsigned long)format) {
+	case D3DFMT_X1R5G5B5:
+	case D3DFMT_A1R5G5B5: return DXGI_FORMAT_B5G5R5A1_UNORM;
+	case D3DFMT_R5G6B5:   return DXGI_FORMAT_B5G6R5_UNORM;
+	case D3DFMT_A8R8G8B8: return DXGI_FORMAT_B8G8R8A8_UNORM;
+	case D3DFMT_X8R8G8B8: return DXGI_FORMAT_B8G8R8X8_UNORM;
+	}
+
+	return DXGI_FORMAT_UNKNOWN;
+}
+
+static IDirect3DSurface8 *d3d8_get_backbuffer(IDirect3DDevice8 *device)
+{
+	IDirect3DSurface8 *backbuffer;
+	HRESULT hr;
+
+	hr = device->GetRenderTarget(&backbuffer);
+	if (FAILED(hr)) {
+		hlog_hr("d3d8_get_backbuffer: Failed to get backbuffer", hr);
+		backbuffer = nullptr;
+	}
+
+	return backbuffer;
+}
+
+static bool d3d8_get_window_handle(IDirect3DDevice8 *device)
+{
+	D3DDEVICE_CREATION_PARAMETERS parameters;
+	HRESULT hr;
+	hr = device->GetCreationParameters(&parameters);
+	if (FAILED(hr)) {
+		hlog_hr("d3d8_get_window_handle: Failed to get "
+				"device creation parameters", hr);
+		return false;
+	}
+
+	data.window = parameters.hFocusWindow;
+
+	return true;
+}
+
+static bool d3d8_init_format_backbuffer(IDirect3DDevice8 *device)
+{
+	IDirect3DSurface8 *backbuffer;
+	D3DSURFACE_DESC desc;
+	HRESULT hr;
+
+	if (!d3d8_get_window_handle(device))
+		return false;
+
+	backbuffer = d3d8_get_backbuffer(device);
+	if (!backbuffer)
+		return false;
+
+	hr = backbuffer->GetDesc(&desc);
+	backbuffer->Release();
+	if (FAILED(hr)) {
+		hlog_hr("d3d8_init_format_backbuffer: Failed to get "
+				"backbuffer descriptor", hr);
+		return false;
+	}
+
+	data.d3d8_format = desc.Format;
+	data.dxgi_format = d3d8_to_dxgi_format(desc.Format);
+	data.cx = desc.Width;
+	data.cy = desc.Height;
+
+	return true;
+}
+
+static bool d3d8_shmem_init_buffer(IDirect3DDevice8 *device, int idx)
+{
+	HRESULT hr;
+
+	hr = device->CreateImageSurface(data.cx, data.cy,
+			data.d3d8_format, &data.copy_surfaces[idx]);
+	if (FAILED(hr)) {
+		hlog_hr("d3d8_shmem_init_buffer: Failed to create surface", hr);
+		return false;
+	}
+
+	if (idx == 0) {
+		D3DLOCKED_RECT rect;
+		hr = data.copy_surfaces[0]->LockRect(&rect, nullptr,
+				D3DLOCK_READONLY);
+		if (FAILED(hr)) {
+			hlog_hr("d3d8_shmem_init_buffer: Failed to lock buffer", hr);
+			return false;
+		}
+
+		data.pitch = rect.Pitch;
+		data.copy_surfaces[0]->UnlockRect();
+	}
+
+	return true;
+}
+
+static bool d3d8_shmem_init(IDirect3DDevice8 *device)
+{
+	for (int i = 0; i < NUM_BUFFERS; i++) {
+		if (!d3d8_shmem_init_buffer(device, i)) {
+			return false;
+		}
+	}
+	if (!capture_init_shmem(&data.shmem_info, data.window, data.cx, data.cy,
+				data.cx, data.cy, data.pitch, data.dxgi_format,
+				false)) {
+		return false;
+	}
+
+	hlog("d3d8 memory capture successfull");
+	return true;
+}
+
+static void d3d8_free()
+{
+	capture_free();
+
+	for (size_t i = 0; i < NUM_BUFFERS; i++) {
+		if (data.copy_surfaces[i]) {
+			if (data.surface_locked[i])
+				data.copy_surfaces[i]->UnlockRect();
+			data.copy_surfaces[i]->Release();
+		}
+	}
+
+	memset(&data, 0, sizeof(data));
+
+	hlog("----------------- d3d8 capture freed -----------------");
+}
+
+static void d3d8_init(IDirect3DDevice8 *device)
+{
+	data.d3d8 = get_system_module("d3d8.dll");
+
+	if (!d3d8_init_format_backbuffer(device))
+		return;
+
+	if (!d3d8_shmem_init(device))
+		d3d8_free();
+}
+
+static void d3d8_shmem_capture_copy(int idx)
+{
+	IDirect3DSurface8 *target = data.copy_surfaces[idx];
+	D3DLOCKED_RECT rect;
+	HRESULT hr;
+
+	if (data.surface_locked[idx])
+		return;
+
+	hr = target->LockRect(&rect, nullptr, D3DLOCK_READONLY);
+	if (SUCCEEDED(hr)) {
+		shmem_copy_data(idx, rect.pBits);
+	}
+}
+
+static void d3d8_shmem_capture(IDirect3DDevice8 *device,
+		IDirect3DSurface8 *backbuffer)
+{
+	int cur_surface;
+	int next_surface;
+	HRESULT hr;
+
+	cur_surface = data.cur_surface;
+	next_surface = (cur_surface == NUM_BUFFERS - 1) ? 0 : cur_surface + 1;
+
+	if (data.copy_wait < NUM_BUFFERS - 1) {
+		data.copy_wait++;
+	} else {
+		IDirect3DSurface8 *src = backbuffer;
+		IDirect3DSurface8 *dst = data.copy_surfaces[cur_surface];
+
+		if (shmem_texture_data_lock(next_surface)) {
+			dst->UnlockRect();
+			data.surface_locked[next_surface] = false;
+			shmem_texture_data_unlock(next_surface);
+		}
+
+		hr = device->CopyRects(src, nullptr, 0, dst, nullptr);
+		if (SUCCEEDED(hr)) {
+			d3d8_shmem_capture_copy(cur_surface);
+		}
+	}
+
+	data.cur_surface = next_surface;
+}
+
+static void d3d8_capture(IDirect3DDevice8 *device,
+		IDirect3DSurface8 *backbuffer)
+{
+	if (capture_should_stop()) {
+		d3d8_free();
+	}
+	if (capture_should_init()) {
+		d3d8_init(device);
+	}
+	if (capture_ready()) {
+		d3d8_shmem_capture(device, backbuffer);
+	}
+}
+
+
+static HRESULT STDMETHODCALLTYPE hook_reset(IDirect3DDevice8 *device,
+		D3DPRESENT_PARAMETERS *parameters)
+{
+	HRESULT hr;
+
+	if (capture_active())
+		d3d8_free();
+
+	unhook(&reset);
+	reset_t call = (reset_t)reset.call_addr;
+	hr = call(device, parameters);
+	rehook(&reset);
+
+	return hr;
+}
+
+static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice8 *device,
+		CONST RECT *src_rect, CONST RECT *dst_rect,
+		HWND override_window, CONST RGNDATA *dirty_region)
+{
+	IDirect3DSurface8 *backbuffer;
+	HRESULT hr;
+
+	backbuffer = d3d8_get_backbuffer(device);
+	if (backbuffer) {
+		d3d8_capture(device, backbuffer);
+		backbuffer->Release();
+	}
+
+	unhook(&present);
+	present_t call = (present_t)present.call_addr;
+	hr = call(device, src_rect, dst_rect, override_window, dirty_region);
+	rehook(&present);
+
+	return hr;
+}
+
+bool hook_d3d8(void)
+{
+	HMODULE d3d8_module = get_system_module("d3d8.dll");
+	void *present_addr;
+	void *reset_addr;
+
+	if (!d3d8_module) {
+		return false;
+	}
+
+	present_addr = get_offset_addr(d3d8_module,
+			global_hook_info->offsets.d3d8.present);
+	reset_addr = get_offset_addr(d3d8_module,
+			global_hook_info->offsets.d3d8.reset);
+
+	hook_init(&present, present_addr, (void*)hook_present,
+			"IDirect3DDevice8::Present");
+	hook_init(&reset, reset_addr, (void*)hook_reset,
+			"IDirect3DDevice8::Reset");
+
+	rehook(&present);
+	rehook(&reset);
+
+	hlog("Hooked D3D8");
+
+	return true;
+}

plugins/win-capture/graphics-hook/graphics-hook.c

@@ -274,7 +274,7 @@ static inline bool dxgi_hookable(void)
 static inline bool attempt_hook(void)
 {
 	//static bool ddraw_hooked = false;
-	//static bool d3d8_hooked  = false;
+	static bool d3d8_hooked  = false;
 	static bool d3d9_hooked  = false;
 	static bool dxgi_hooked  = false;
 	static bool gl_hooked    = false;
@@ -310,7 +310,7 @@ static inline bool attempt_hook(void)
 		rehook_gl();*/
 	}
 
-	/*if (!d3d8_hooked) {
+	if (!d3d8_hooked) {
 		if (!d3d8_hookable()) {
 			d3d8_hooked = true;
 		} else {
@@ -321,7 +321,7 @@ static inline bool attempt_hook(void)
 		}
 	}
 
-	if (!ddraw_hooked) {
+	/*if (!ddraw_hooked) {
 		if (!ddraw_hookable()) {
 			ddraw_hooked = true;
 		} else {

plugins/win-capture/load-graphics-offsets.c

@@ -20,6 +20,11 @@ static inline bool load_offsets_from_string(struct graphics_offsets *offsets,
 		return false;
 	}
 
+	offsets->d3d8.present =
+		(uint32_t)config_get_uint(config, "d3d8", "present");
+	offsets->d3d8.reset =
+		(uint32_t)config_get_uint(config, "d3d8", "reset");
+
 	offsets->d3d9.present =
 		(uint32_t)config_get_uint(config, "d3d9", "present");
 	offsets->d3d9.present_ex =