(API Change) Fix filter rendering design flaw

obs_source_process_filter tried to do everything in a single function,
but the problem is that effect parameters would not properly be
accounted for due to the way it internally draws, therefore it was
necessary to split the functions in to two, you first call
obs_source_process_filter_begin, then you set your effect parameters,
then you finally call obs_source_process_filter_end.  This ensures that
when the filter is drawn, that the effect parameters are set.

libobs/obs-internal.h

@@ -358,6 +358,7 @@ struct obs_source {
 	DARRAY(struct obs_source*)      filters;
 	pthread_mutex_t                 filter_mutex;
 	gs_texrender_t                  *filter_texrender;
+	enum obs_allow_direct_render    allow_direct;
 	bool                            rendering_filter;
 };
 

libobs/obs-source.c

@@ -2025,8 +2025,8 @@ static inline bool can_bypass(obs_source_t *target, obs_source_t *parent,
 		((parent_flags & OBS_SOURCE_ASYNC) == 0);
 }
 
-void obs_source_process_filter(obs_source_t *filter, gs_effect_t *effect,
-		uint32_t width, uint32_t height, enum gs_color_format format,
+void obs_source_process_filter_begin(obs_source_t *filter,
+		enum gs_color_format format,
 		enum obs_allow_direct_render allow_direct)
 {
 	obs_source_t *target, *parent;
@@ -2044,12 +2044,13 @@ void obs_source_process_filter(obs_source_t *filter, gs_effect_t *effect,
 	cy           = get_base_height(target);
 	use_matrix   = !!(target_flags & OBS_SOURCE_COLOR_MATRIX);
 
+	filter->allow_direct = allow_direct;
+
 	/* if the parent does not use any custom effects, and this is the last
 	 * filter in the chain for the parent, then render the parent directly
 	 * using the filter effect instead of rendering to texture to reduce
 	 * the total number of passes */
 	if (can_bypass(target, parent, parent_flags, allow_direct)) {
-		render_filter_bypass(target, effect, use_matrix);
 		return;
 	}
 
@@ -2073,11 +2074,30 @@ void obs_source_process_filter(obs_source_t *filter, gs_effect_t *effect,
 
 		gs_texrender_end(filter->filter_texrender);
 	}
+}
+
+void obs_source_process_filter_end(obs_source_t *filter, gs_effect_t *effect,
+		uint32_t width, uint32_t height)
+{
+	obs_source_t *target, *parent;
+	gs_texture_t *texture;
+	uint32_t     target_flags, parent_flags;
+	bool         use_matrix;
+
+	if (!filter) return;
 
-	/* --------------------------- */
+	target       = obs_filter_get_target(filter);
+	parent       = obs_filter_get_parent(filter);
+	target_flags = target->info.output_flags;
+	parent_flags = parent->info.output_flags;
+	use_matrix   = !!(target_flags & OBS_SOURCE_COLOR_MATRIX);
 
-	render_filter_tex(gs_texrender_get_texture(filter->filter_texrender),
-			effect, width, height, use_matrix);
+	if (can_bypass(target, parent, parent_flags, filter->allow_direct)) {
+		render_filter_bypass(target, effect, use_matrix);
+	} else {
+		texture = gs_texrender_get_texture(filter->filter_texrender);
+		render_filter_tex(texture, effect, width, height, use_matrix);
+	}
 }
 
 void obs_source_skip_video_filter(obs_source_t *filter)

libobs/obs.h

@@ -877,11 +877,28 @@ EXPORT struct obs_source_frame *obs_source_get_frame(obs_source_t *source);
 EXPORT void obs_source_release_frame(obs_source_t *source,
 		struct obs_source_frame *frame);
 
-/** Default RGB filter handler for generic effect filters */
-EXPORT void obs_source_process_filter(obs_source_t *filter, gs_effect_t *effect,
-		uint32_t width, uint32_t height, enum gs_color_format format,
+/**
+ * Default RGB filter handler for generic effect filters.  Processes the
+ * filter chain and renders them to texture if needed, then the filter is
+ * drawn with
+ *
+ * After calling this, set your parameters for the effect, then call
+ * obs_source_process_filter_end to draw the filter.
+ */
+EXPORT void obs_source_process_filter_begin(obs_source_t *filter,
+		enum gs_color_format format,
 		enum obs_allow_direct_render allow_direct);
 
+/**
+ * Draws the filter.
+ *
+ * Before calling this function, first call obs_source_process_filter_begin and
+ * then set the effect parameters, and then call this function to finalize the
+ * filter.
+ */
+EXPORT void obs_source_process_filter_end(obs_source_t *filter,
+		gs_effect_t *effect, uint32_t width, uint32_t height);
+
 /** Skips the filter if the filter is invalid and cannot be rendered */
 EXPORT void obs_source_skip_video_filter(obs_source_t *filter);
 

test/test-input/test-filter.c

@@ -50,8 +50,10 @@ static void *filter_create(obs_data_t *settings, obs_source_t *source)
 static void filter_render(void *data, gs_effect_t *effect)
 {
 	struct test_filter *tf = data;
-	obs_source_process_filter(tf->source, tf->whatever, 0, 0, GS_RGBA,
+
+	obs_source_process_filter_begin(tf->source, GS_RGBA,
 			OBS_ALLOW_DIRECT_RENDERING);
+	obs_source_process_filter_end(tf->source, tf->whatever, 0, 0);
 
 	UNUSED_PARAMETER(effect);
 }