win-dshow: Use a single thread per device

This prevents certain issues I've encountered with devices where they
expect to shut down in a specific thread they started up in, as well as
a number of other issues, such as the configuration dialogs.

The configuration dialogs require that a message loop be present, and
this was not the case previously because everything was in the video
thread, which has no windows-specific code.

Configuration/crossbar/etc dialogs will now execute correctly.

plugins/win-dshow/win-dshow.cpp

@@ -5,6 +5,7 @@
 #include <util/dstr.hpp>
 #include <util/util.hpp>
 #include <util/platform.h>
+#include <util/windows/WinHandle.hpp>
 #include "libdshowcapture/dshowcapture.hpp"
 #include "ffmpeg-decode.h"
 
@@ -87,10 +88,49 @@ public:
 	inline operator ffmpeg_decode*() {return &decode;}
 };
 
+class CriticalSection {
+	CRITICAL_SECTION mutex;
+
+public:
+	inline CriticalSection()  {InitializeCriticalSection(&mutex);}
+	inline ~CriticalSection() {DeleteCriticalSection(&mutex);}
+
+	inline operator CRITICAL_SECTION*() {return &mutex;}
+};
+
+class CriticalScope {
+	CriticalSection &mutex;
+
+	CriticalScope() = delete;
+	CriticalScope& operator=(CriticalScope &cs) = delete;
+
+public:
+	inline CriticalScope(CriticalSection &mutex_) : mutex(mutex_)
+	{
+		EnterCriticalSection(mutex);
+	}
+
+	inline ~CriticalScope()
+	{
+		LeaveCriticalSection(mutex);
+	}
+};
+
+enum class Action {
+	None,
+	Update,
+	Shutdown,
+	ConfigVideo,
+	ConfigAudio,
+	ConfigCrossbar1,
+	ConfigCrossbar2,
+};
+
+static DWORD CALLBACK DShowThread(LPVOID ptr);
+
 struct DShowInput {
 	obs_source_t source;
 	Device       device;
-	bool         comInitialized;
 	bool         deviceHasAudio;
 
 	Decoder      audio_decoder;
@@ -102,16 +142,51 @@ struct DShowInput {
 	obs_source_frame frame;
 	obs_source_audio audio;
 
+	WinHandle semaphore;
+	WinHandle thread;
+	CriticalSection mutex;
+	vector<Action> actions;
+
+	inline void QueueAction(Action action)
+	{
+		CriticalScope scope(mutex);
+		actions.push_back(action);
+		ReleaseSemaphore(semaphore, 1, nullptr);
+	}
+
 	inline DShowInput(obs_source_t source_)
 		: source         (source_),
-		  device         (InitGraph::False),
-		  comInitialized (false)
+		  device         (InitGraph::False)
 	{
 		memset(&audio, 0, sizeof(audio));
 		memset(&frame, 0, sizeof(frame));
 
 		av_log_set_level(AV_LOG_WARNING);
 		av_log_set_callback(ffmpeg_log);
+
+		semaphore = CreateSemaphore(nullptr, 0, 0x7FFFFFFF, nullptr);
+		if (!semaphore)
+			throw "Failed to create semaphore";
+
+		thread = CreateThread(nullptr, 0, DShowThread, this, 0,
+				nullptr);
+		if (!thread)
+			throw "Failed to create thread";
+
+		QueueAction(Action::Update);
+	}
+
+	inline ~DShowInput()
+	{
+		{
+			CriticalScope scope(mutex);
+			actions.resize(1);
+			actions[0] = Action::Shutdown;
+		}
+
+		ReleaseSemaphore(semaphore, 1, nullptr);
+
+		WaitForSingleObject(thread, INFINITE);
 	}
 
 	void OnEncodedVideoData(enum AVCodecID id,
@@ -129,8 +204,86 @@ struct DShowInput {
 	bool UpdateVideoConfig(obs_data_t settings);
 	bool UpdateAudioConfig(obs_data_t settings);
 	void Update(obs_data_t settings);
+
+	void DShowLoop();
 };
 
+static DWORD CALLBACK DShowThread(LPVOID ptr)
+{
+	DShowInput *dshowInput = (DShowInput*)ptr;
+
+	CoInitialize(nullptr);
+	dshowInput->DShowLoop();
+	CoUninitialize();
+	return 0;
+}
+
+static inline void ProcessMessages()
+{
+	MSG msg;
+	while (PeekMessage(&msg, nullptr, 0, 0, PM_REMOVE)) {
+		TranslateMessage(&msg);
+		DispatchMessage(&msg);
+	}
+}
+
+/* Always keep directshow in a single thread for a given device */
+void DShowInput::DShowLoop()
+{
+	while (true) {
+		DWORD ret = MsgWaitForMultipleObjects(1, &semaphore, false,
+				INFINITE, QS_ALLINPUT);
+		if (ret == (WAIT_OBJECT_0 + 1)) {
+			ProcessMessages();
+			continue;
+		} else if (ret != WAIT_OBJECT_0) {
+			break;
+		}
+
+		Action action = Action::None;
+		{
+			CriticalScope scope(mutex);
+			if (actions.size()) {
+				action = actions.front();
+				actions.erase(actions.begin());
+			}
+		}
+
+		switch (action) {
+		case Action::Update:
+			{
+				obs_data_t settings;
+				settings = obs_source_get_settings(source);
+				Update(settings);
+				obs_data_release(settings);
+				break;
+			}
+
+		case Action::Shutdown:
+			device.ShutdownGraph();
+			return;
+
+		case Action::ConfigVideo:
+			device.OpenDialog(nullptr, DialogType::ConfigVideo);
+			break;
+
+		case Action::ConfigAudio:
+			device.OpenDialog(nullptr, DialogType::ConfigAudio);
+			break;
+
+		case Action::ConfigCrossbar1:
+			device.OpenDialog(nullptr, DialogType::ConfigCrossbar);
+			break;
+
+		case Action::ConfigCrossbar2:
+			device.OpenDialog(nullptr, DialogType::ConfigCrossbar2);
+			break;
+
+		case Action::None:;
+		}
+	}
+}
+
 #define FPS_HIGHEST   0LL
 #define FPS_MATCHING -1LL
 
@@ -640,11 +793,6 @@ bool DShowInput::UpdateAudioConfig(obs_data_t settings)
 
 void DShowInput::Update(obs_data_t settings)
 {
-	if (!comInitialized) {
-		CoInitialize(nullptr);
-		comInitialized = true;
-	}
-
 	if (!device.ResetGraph())
 		return;
 
@@ -685,10 +833,14 @@ static const char *GetDShowInputName(void)
 
 static void *CreateDShowInput(obs_data_t settings, obs_source_t source)
 {
-	DShowInput *dshow = new DShowInput(source);
+	DShowInput *dshow = nullptr;
 
-	/* causes a deferred update in the video thread */
-	obs_source_update(source, nullptr);
+	try {
+		dshow = new DShowInput(source);
+	} catch (const char *error) {
+		blog(LOG_ERROR, "Could not create device '%s': %s",
+				obs_source_get_name(source), error);
+	}
 
 	UNUSED_PARAMETER(settings);
 	return dshow;
@@ -701,7 +853,8 @@ static void DestroyDShowInput(void *data)
 
 static void UpdateDShowInput(void *data, obs_data_t settings)
 {
-	reinterpret_cast<DShowInput*>(data)->Update(settings);
+	UNUSED_PARAMETER(settings);
+	reinterpret_cast<DShowInput*>(data)->QueueAction(Action::Update);
 }
 
 static void GetDShowDefaults(obs_data_t settings)
@@ -959,7 +1112,7 @@ static bool VideoConfigClicked(obs_properties_t props, obs_property_t p,
 		void *data)
 {
 	DShowInput *input = reinterpret_cast<DShowInput*>(data);
-	input->device.OpenDialog(nullptr, DialogType::ConfigVideo);
+	input->QueueAction(Action::ConfigVideo);
 
 	UNUSED_PARAMETER(props);
 	UNUSED_PARAMETER(p);
@@ -970,7 +1123,7 @@ static bool VideoConfigClicked(obs_properties_t props, obs_property_t p,
 		void *data)
 {
 	DShowInput *input = reinterpret_cast<DShowInput*>(data);
-	input->device.OpenDialog(nullptr, DialogType::ConfigAudio);
+	input->QueueAction(Action::ConfigAudio);
 
 	UNUSED_PARAMETER(props);
 	UNUSED_PARAMETER(p);
@@ -981,7 +1134,7 @@ static bool CrossbarConfigClicked(obs_properties_t props, obs_property_t p,
 		void *data)
 {
 	DShowInput *input = reinterpret_cast<DShowInput*>(data);
-	input->device.OpenDialog(nullptr, DialogType::ConfigCrossbar);
+	input->QueueAction(Action::ConfigCrossbar1);
 
 	UNUSED_PARAMETER(props);
 	UNUSED_PARAMETER(p);
@@ -992,7 +1145,7 @@ static bool CrossbarConfigClicked(obs_properties_t props, obs_property_t p,
 		void *data)
 {
 	DShowInput *input = reinterpret_cast<DShowInput*>(data);
-	input->device.OpenDialog(nullptr, DialogType::ConfigCrossbar2);
+	input->QueueAction(Action::ConfigCrossbar2);
 
 	UNUSED_PARAMETER(props);
 	UNUSED_PARAMETER(p);