Use recursive mutex for scene mutex

Fixes a deadlock when trying to remove a source from the GUI. The scene
item signal handlers would mark the source as removed which results in
the video thread also trying to run obs_sceneitem_destroy thereby
deadlocking the video thread (and the GUI thread)

libobs/obs-scene.c

@@ -36,17 +36,26 @@ static const char *scene_getname(const char *locale)
 
 static void *scene_create(obs_data_t settings, struct obs_source *source)
 {
+	pthread_mutexattr_t attr;
 	struct obs_scene *scene = bmalloc(sizeof(struct obs_scene));
 	scene->source     = source;
 	scene->first_item = NULL;
 
-	if (pthread_mutex_init(&scene->mutex, NULL) != 0) {
+	if (pthread_mutexattr_init(&attr) != 0)
+		goto fail;
+	if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) != 0)
+		goto fail;
+	if (pthread_mutex_init(&scene->mutex, &attr) != 0) {
 		blog(LOG_ERROR, "scene_create: Couldn't initialize mutex");
-		bfree(scene);
-		return NULL;
+		goto fail;
 	}
 
 	return scene;
+
+fail:
+	pthread_mutexattr_destroy(&attr);
+	bfree(scene);
+	return NULL;
 }
 
 static void scene_destroy(void *data)
@@ -115,7 +124,7 @@ static void scene_video_render(void *data)
 			struct obs_scene_item *del_item = item;
 			item = item->next;
 
-			obs_sceneitem_destroy(del_item);
+			obs_sceneitem_destroy(scene, del_item);
 			continue;
 		}
 
@@ -285,9 +294,13 @@ obs_sceneitem_t obs_scene_add(obs_scene_t scene, obs_source_t source)
 	return item;
 }
 
-int obs_sceneitem_destroy(obs_sceneitem_t item)
+int obs_sceneitem_destroy(obs_scene_t scene, obs_sceneitem_t item)
 {
 	int ref = 0;
+	if (!scene || !item)
+		return ref;
+
+	pthread_mutex_lock(&scene->mutex);
 
 	if (item) {
 		struct calldata params = {0};
@@ -303,6 +316,8 @@ int obs_sceneitem_destroy(obs_sceneitem_t item)
 		bfree(item);
 	}
 
+	pthread_mutex_unlock(&scene->mutex);
+
 	return ref;
 }
 

libobs/obs.h

@@ -503,7 +503,7 @@ EXPORT obs_sceneitem_t obs_scene_add(obs_scene_t scene, obs_source_t source);
 
 /** Removes/destroys a scene item.  Returns the source reference counter
  * (if any) */
-EXPORT int obs_sceneitem_destroy(obs_sceneitem_t item);
+EXPORT int obs_sceneitem_destroy(obs_scene_t scene, obs_sceneitem_t item);
 
 /** Gets the scene parent associated with the scene item */
 EXPORT obs_scene_t obs_sceneitem_getscene(obs_sceneitem_t item);

obs/window-basic-main.cpp

@@ -515,10 +515,17 @@ void OBSBasic::on_actionRemoveSource_triggered()
 	if (!sel)
 		return;
 
+	obs_scene_t scene = GetCurrentScene();
+	if (!scene)
+		return;
+
+	obs_scene_addref(scene);
+
 	QVariant userData = sel->data(Qt::UserRole);
 	obs_sceneitem_t item = VariantPtr<obs_sceneitem_t>(userData);
-	obs_source_t source = obs_sceneitem_getsource(item);
-	obs_sceneitem_destroy(item);
+	obs_sceneitem_destroy(scene, item);
+
+	obs_scene_release(scene);
 }
 
 void OBSBasic::on_actionSourceProperties_triggered()