libobs: Prevent infinite source recursion

Changed the design from using obs_source::enum_refs to just simply
preventing infinite source recursion in general, rather than allowing it
through the enum_refs variable.  obs_source_add_child has been changed
so that it now returns a boolean, and if the function fails, it means
that the child cannot be added due to that potential recursion.

libobs/obs-internal.h

@@ -288,9 +288,6 @@ struct obs_source {
 	/* ensures activate/deactivate are only called once */
 	volatile long                   activate_refs;
 
-	/* prevents infinite recursion when enumerating sources */
-	volatile long                   enum_refs;
-
 	/* used to indicate that the source has been removed and all
 	 * references to it should be released (not exactly how I would prefer
 	 * to handle things but it's the best option) */

libobs/obs-scene.c

@@ -552,6 +552,12 @@ obs_sceneitem_t *obs_scene_add(obs_scene_t *scene, obs_source_t *source)
 		return NULL;
 	}
 
+	if (!obs_source_add_child(scene->source, source)) {
+		blog(LOG_WARNING, "Failed to add source to scene due to "
+		                  "infinite source recursion");
+		return NULL;
+	}
+
 	item = bzalloc(sizeof(struct obs_scene_item));
 	item->source  = source;
 	item->visible = true;
@@ -563,7 +569,6 @@ obs_sceneitem_t *obs_scene_add(obs_scene_t *scene, obs_source_t *source)
 	matrix4_identity(&item->box_transform);
 
 	obs_source_addref(source);
-	obs_source_add_child(scene->source, source);
 
 	pthread_mutex_lock(&scene->mutex);
 

libobs/obs-source.c

@@ -1900,14 +1900,11 @@ static void enum_source_tree_callback(obs_source_t *parent, obs_source_t *child,
 {
 	struct source_enum_data *data = param;
 
-	if (child->info.enum_sources && !child->enum_refs) {
-		os_atomic_inc_long(&child->enum_refs);
-
-		if (child->context.data)
+	if (child->info.enum_sources) {
+		if (child->context.data) {
 			child->info.enum_sources(child->context.data,
 					enum_source_tree_callback, data);
-
-		os_atomic_dec_long(&child->enum_refs);
+		}
 	}
 
 	data->enum_callback(parent, child, data->param);
@@ -1917,16 +1914,12 @@ void obs_source_enum_sources(obs_source_t *source,
 		obs_source_enum_proc_t enum_callback,
 		void *param)
 {
-	if (!source_valid(source)      ||
-	    !source->info.enum_sources ||
-	    source->enum_refs)
+	if (!source_valid(source) || !source->info.enum_sources)
 		return;
 
 	obs_source_addref(source);
 
-	os_atomic_inc_long(&source->enum_refs);
 	source->info.enum_sources(source->context.data, enum_callback, param);
-	os_atomic_dec_long(&source->enum_refs);
 
 	obs_source_release(source);
 }
@@ -1937,31 +1930,47 @@ void obs_source_enum_tree(obs_source_t *source,
 {
 	struct source_enum_data data = {enum_callback, param};
 
-	if (!source_valid(source)      ||
-	    !source->info.enum_sources ||
-	    source->enum_refs)
+	if (!source_valid(source) || !source->info.enum_sources)
 		return;
 
 	obs_source_addref(source);
 
-	os_atomic_inc_long(&source->enum_refs);
 	source->info.enum_sources(source->context.data,
 			enum_source_tree_callback,
 			&data);
-	os_atomic_dec_long(&source->enum_refs);
 
 	obs_source_release(source);
 }
 
-void obs_source_add_child(obs_source_t *parent, obs_source_t *child)
+struct descendant_info {
+	bool exists;
+	obs_source_t *target;
+};
+
+static void check_descendant(obs_source_t *parent, obs_source_t *child,
+		void *param)
 {
-	if (!parent || !child) return;
+	struct descendant_info *info = param;
+	if (child == info->target || parent == info->target)
+		info->exists = true;
+}
+
+bool obs_source_add_child(obs_source_t *parent, obs_source_t *child)
+{
+	struct descendant_info info = {false, child};
+	if (!parent || !child) return false;
+
+	obs_source_enum_tree(parent, check_descendant, &info);
+	if (info.exists)
+		return false;
 
 	for (int i = 0; i < parent->show_refs; i++) {
 		enum view_type type;
 		type = (i < parent->activate_refs) ? MAIN_VIEW : AUX_VIEW;
 		obs_source_activate(child, type);
 	}
+
+	return true;
 }
 
 void obs_source_remove_child(obs_source_t *parent, obs_source_t *child)

libobs/obs.h

@@ -822,8 +822,10 @@ EXPORT void obs_source_process_filter(obs_source_t *filter, gs_effect_t *effect,
  * Adds a child source.  Must be called by parent sources on child sources
  * when the child is added.  This ensures that the source is properly activated
  * if the parent is active.
+ *
+ * @returns true if source can be added, false if it causes recursion
  */
-EXPORT void obs_source_add_child(obs_source_t *parent, obs_source_t *child);
+EXPORT bool obs_source_add_child(obs_source_t *parent, obs_source_t *child);
 
 /**
  * Removes a child source.  Must be called by parent sources on child sources