libobs: Defer reconfiguring encoders to the encode threads

Fixes a long-standing issue with Dynamic Bitrate where the RTMP output
thread could try to reconfigure an encoder while the encoder is in the
middle of encoding. x264 seems to handle multithreaded calls well, but
NVENC would deadlock in this situation with no error visible to the
user.

libobs/obs-encoder.c

@@ -397,9 +397,12 @@ void obs_encoder_update(obs_encoder_t *encoder, obs_data_t *settings)
 
 	obs_data_apply(encoder->context.settings, settings);
 
-	if (encoder->info.update && encoder->context.data)
-		encoder->info.update(encoder->context.data,
-				     encoder->context.settings);
+	// Note, we don't actually apply the changes to the encoder here
+	// as it may be active in another thread. Setting this to true
+	// makes the changes apply at the next possible moment in the
+	// encoder / GPU encoder thread.
+	if (encoder->info.update)
+		encoder->reconfigure_requested = true;
 }
 
 bool obs_encoder_get_extra_data(const obs_encoder_t *encoder,
@@ -970,6 +973,12 @@ bool do_encode(struct obs_encoder *encoder, struct encoder_frame *frame)
 	bool received = false;
 	bool success;
 
+	if (encoder->reconfigure_requested) {
+		encoder->reconfigure_requested = false;
+		encoder->info.update(encoder->context.data,
+				     encoder->context.settings);
+	}
+
 	pkt.timebase_num = encoder->timebase_num;
 	pkt.timebase_den = encoder->timebase_den;
 	pkt.encoder = encoder;

libobs/obs-internal.h

@@ -1120,6 +1120,9 @@ struct obs_encoder {
 
 	const char *profile_encoder_encode_name;
 	char *last_error_message;
+
+	/* reconfigure encoder at next possible opportunity */
+	bool reconfigure_requested;
 };
 
 extern struct obs_encoder_info *find_encoder(const char *id);

libobs/obs-video-gpu-encode.c

@@ -90,6 +90,12 @@ static void *gpu_encode_thread(void *unused)
 			if (video_pause_check(&encoder->pause, timestamp))
 				continue;
 
+			if (encoder->reconfigure_requested) {
+				encoder->reconfigure_requested = false;
+				encoder->info.update(encoder->context.data,
+						     encoder->context.settings);
+			}
+
 			if (!encoder->start_ts)
 				encoder->start_ts = timestamp;