libobs: Add ability to reroute encoders

Allows the ability for one encoder to defer to another in case of
failure or unsupported feature.  Okay, fine, it's mostly a hack so the
new NVENC encoder can fall back to the FFmpeg encoder if NV12 textures
aren't in use, that way it does not have to implement raw fallback
support itself.  The settings and properties are pretty much the same,
so there's no reason not to utilize it in order to save time that could
otherwise be spent more productively.

libobs/obs-encoder.c

@@ -64,8 +64,8 @@ static bool init_encoder(struct obs_encoder *encoder, const char *name,
 	if (pthread_mutex_init(&encoder->outputs_mutex, NULL) != 0)
 		return false;
 
-	if (encoder->info.get_defaults)
-		encoder->info.get_defaults(encoder->context.settings);
+	if (encoder->orig_info.get_defaults)
+		encoder->orig_info.get_defaults(encoder->context.settings);
 
 	return true;
 }
@@ -90,8 +90,10 @@ static struct obs_encoder *create_encoder(const char *id,
 		encoder->info.id      = bstrdup(id);
 		encoder->info.type    = type;
 		encoder->owns_info_id = true;
+		encoder->orig_info    = encoder->info;
 	} else {
 		encoder->info = *ei;
+		encoder->orig_info = *ei;
 	}
 
 	success = init_encoder(encoder, name, settings, hotkey_data);
@@ -329,16 +331,17 @@ obs_properties_t *obs_encoder_properties(const obs_encoder_t *encoder)
 	if (!obs_encoder_valid(encoder, "obs_encoder_properties"))
 		return NULL;
 
-	if (encoder->info.get_properties2) {
+	if (encoder->orig_info.get_properties2) {
 		obs_properties_t *props;
-		props = encoder->info.get_properties2(encoder->context.data,
-				encoder->info.type_data);
+		props = encoder->orig_info.get_properties2(
+				encoder->context.data,
+				encoder->orig_info.type_data);
 		obs_properties_apply_settings(props, encoder->context.settings);
 		return props;
 
-	} else if (encoder->info.get_properties) {
+	} else if (encoder->orig_info.get_properties) {
 		obs_properties_t *props;
-		props = encoder->info.get_properties(encoder->context.data);
+		props = encoder->orig_info.get_properties(encoder->context.data);
 		obs_properties_apply_settings(props, encoder->context.settings);
 		return props;
 	}
@@ -404,6 +407,8 @@ static void intitialize_audio_encoder(struct obs_encoder *encoder)
 	reset_audio_buffers(encoder);
 }
 
+static THREAD_LOCAL bool can_reroute = false;
+
 static inline bool obs_encoder_initialize_internal(obs_encoder_t *encoder)
 {
 	if (encoder_active(encoder))
@@ -413,19 +418,45 @@ static inline bool obs_encoder_initialize_internal(obs_encoder_t *encoder)
 
 	obs_encoder_shutdown(encoder);
 
-	if (encoder->info.create)
-		encoder->context.data = encoder->info.create(
+	if (encoder->orig_info.create) {
+		can_reroute = true;
+		encoder->info = encoder->orig_info;
+		encoder->context.data = encoder->orig_info.create(
 				encoder->context.settings, encoder);
+		can_reroute = false;
+	}
 	if (!encoder->context.data)
 		return false;
 
-	if (encoder->info.type == OBS_ENCODER_AUDIO)
+	if (encoder->orig_info.type == OBS_ENCODER_AUDIO)
 		intitialize_audio_encoder(encoder);
 
 	encoder->initialized = true;
 	return true;
 }
 
+void *obs_encoder_create_rerouted(obs_encoder_t *encoder, const char *reroute_id)
+{
+	if (!obs_ptr_valid(encoder, "obs_encoder_reroute"))
+		return NULL;
+	if (!obs_ptr_valid(reroute_id, "obs_encoder_reroute"))
+		return NULL;
+	if (!can_reroute)
+		return NULL;
+
+	const struct obs_encoder_info *ei = find_encoder(reroute_id);
+	if (ei) {
+		if (ei->type != encoder->orig_info.type ||
+		    astrcmpi(ei->codec, encoder->orig_info.codec) != 0) {
+			return NULL;
+		}
+		encoder->info = *ei;
+		return encoder->info.create(encoder->context.settings, encoder);
+	}
+
+	return NULL;
+}
+
 bool obs_encoder_initialize(obs_encoder_t *encoder)
 {
 	bool success;
@@ -1202,13 +1233,13 @@ bool obs_weak_encoder_references_encoder(obs_weak_encoder_t *weak,
 void *obs_encoder_get_type_data(obs_encoder_t *encoder)
 {
 	return obs_encoder_valid(encoder, "obs_encoder_get_type_data")
-		? encoder->info.type_data : NULL;
+		? encoder->orig_info.type_data : NULL;
 }
 
 const char *obs_encoder_get_id(const obs_encoder_t *encoder)
 {
 	return obs_encoder_valid(encoder, "obs_encoder_get_id")
-		? encoder->info.id : NULL;
+		? encoder->orig_info.id : NULL;
 }
 
 uint32_t obs_get_encoder_caps(const char *encoder_id)

libobs/obs-internal.h

@@ -942,6 +942,9 @@ struct obs_encoder {
 	struct obs_encoder_info         info;
 	struct obs_weak_encoder         *control;
 
+	/* allows re-routing to another encoder */
+	struct obs_encoder_info         orig_info;
+
 	pthread_mutex_t                 init_mutex;
 
 	uint32_t                        samplerate;

libobs/obs.h

@@ -1935,6 +1935,9 @@ EXPORT void obs_encoder_packet_ref(struct encoder_packet *dst,
 		struct encoder_packet *src);
 EXPORT void obs_encoder_packet_release(struct encoder_packet *packet);
 
+EXPORT void *obs_encoder_create_rerouted(obs_encoder_t *encoder,
+		const char *reroute_id);
+
 
 /* ------------------------------------------------------------------------- */
 /* Stream Services */