libobs, plugins: Deprecate obs_output_t functions with flag parameters

This deprecates the following functions, replacing them with new
versions:
- `obs_output_can_begin_data_capture()` - now `*capture2()`
- `obs_output_initialize_encoders()` - now `*encoders2()`
- `obs_output_begin_data_capture()` - now `*capture2()`

The flags parameter was initially designed to support audio-only or
video-only operation of an output which had the `OBS_OUTPUT_AV` flag,
however, full support for that was never implemented, and there are
likely fundamental issues with an implementation, mainly that most
outputs are programmed assuming that there will always be at least one
audio and one video track. This requires new flags specifying support
for optional audio/video, among other things.

An implementation to allow audio/video to be optional is best done
using the flag technique above, with audio/video enablement specified
by whether media (raw, `video_t/audio_t`) or encoder (`obs_encoder_t`)
objects are specified.

Since every implementation I could find always specifies `flags` as 0,
I was able to safely conclude that immediately removing the parameter's
functionality is safe to do.

docs/sphinx/reference-outputs.rst

@@ -882,44 +882,35 @@ Functions used by outputs
 
 ---------------------
 
-.. function:: bool obs_output_can_begin_data_capture(const obs_output_t *output, uint32_t flags)
+.. function:: bool obs_output_can_begin_data_capture2(const obs_output_t *output)
 
    Determines whether video/audio capture (encoded or raw) is able to
-   start.  Call this before initializing any output data to ensure that
+   start.  Call this before initializing any output state to ensure that
    the output can start.
 
-   :param flags: Set to 0 to initialize both audio/video, otherwise a
-                 bitwise OR combination of OBS_OUTPUT_VIDEO and/or
-                 OBS_OUTPUT_AUDIO
    :return:      *true* if data capture can begin
 
 ---------------------
 
-.. function:: bool obs_output_initialize_encoders(obs_output_t *output, uint32_t flags)
+.. function:: bool obs_output_initialize_encoders2(obs_output_t *output)
 
    Initializes any encoders/services associated with the output.  This
    must be called for encoded outputs before calling
-   :c:func:`obs_output_begin_data_capture()`.
+   :c:func:`obs_output_begin_data_capture2()`.
 
-   :param flags: Set to 0 to initialize both audio/video, otherwise a
-                 bitwise OR combination of OBS_OUTPUT_VIDEO and/or
-                 OBS_OUTPUT_AUDIO
    :return:      *true* if successful, *false* otherwise
 
 ---------------------
 
-.. function:: bool obs_output_begin_data_capture(obs_output_t *output, uint32_t flags)
+.. function:: bool obs_output_begin_data_capture2(obs_output_t *output)
 
    Begins data capture from raw media or encoders.  This is typically
    when the output actually activates (starts) internally.  Video/audio
    data will start being sent to the callbacks of the output.
 
-   :param flags: Set to 0 to initialize both audio/video, otherwise a
-                 bitwise OR combination of OBS_OUTPUT_VIDEO and/or
-                 OBS_OUTPUT_AUDIO
    :return:      *true* if successful, *false* otherwise.  Typically the
                  return value does not need to be checked if
-                 :c:func:`obs_output_can_begin_data_capture()` was
+                 :c:func:`obs_output_can_begin_data_capture2()` was
                  called
 
 ---------------------

libobs/obs-output-delay.c

@@ -157,10 +157,10 @@ bool obs_output_delay_start(obs_output_t *output)
 	};
 
 	if (!delay_active(output)) {
-		bool can_begin = obs_output_can_begin_data_capture(output, 0);
+		bool can_begin = obs_output_can_begin_data_capture2(output);
 		if (!can_begin)
 			return false;
-		if (!obs_output_initialize_encoders(output, 0))
+		if (!obs_output_initialize_encoders2(output))
 			return false;
 	}
 
@@ -175,7 +175,7 @@ bool obs_output_delay_start(obs_output_t *output)
 		return true;
 	}
 
-	if (!obs_output_begin_data_capture(output, 0)) {
+	if (!obs_output_begin_data_capture2(output)) {
 		obs_output_cleanup_delay(output);
 		return false;
 	}

libobs/obs-output.c

@@ -1128,9 +1128,9 @@ void obs_output_set_audio_conversion(
 	output->audio_conversion_set = true;
 }
 
-static inline bool audio_valid(const struct obs_output *output, bool encoded)
+static inline bool audio_valid(const struct obs_output *output)
 {
-	if (encoded) {
+	if (flag_encoded(output)) {
 		for (size_t i = 0; i < MAX_OUTPUT_AUDIO_ENCODERS; i++) {
 			if (output->audio_encoders[i]) {
 				return true;
@@ -1142,27 +1142,19 @@ static inline bool audio_valid(const struct obs_output *output, bool encoded)
 	return output->audio != NULL;
 }
 
-static bool can_begin_data_capture(const struct obs_output *output,
-				   bool encoded, bool has_video, bool has_audio,
-				   bool has_service)
+static bool can_begin_data_capture(const struct obs_output *output)
 {
-	if (has_video) {
-		if (encoded) {
-			if (!output->video_encoder)
-				return false;
-		} else {
-			if (!output->video)
-				return false;
-		}
-	}
-
-	if (has_audio) {
-		if (!audio_valid(output, encoded)) {
+	if (flag_video(output)) {
+		if (flag_encoded(output) && !output->video_encoder)
+			return false;
+		else if (!output->video)
 			return false;
-		}
 	}
 
-	if (has_service && !output->service)
+	if (flag_audio(output) && !audio_valid(output))
+		return false;
+
+	if (flag_service(output) && !output->service)
 		return false;
 
 	return true;
@@ -2033,12 +2025,13 @@ static inline bool preserve_active(struct obs_output *output)
 	return (output->delay_flags & OBS_OUTPUT_DELAY_PRESERVE) != 0;
 }
 
-static void hook_data_capture(struct obs_output *output, bool encoded,
-			      bool has_video, bool has_audio)
+static void hook_data_capture(struct obs_output *output)
 {
 	encoded_callback_t encoded_callback;
+	bool has_video = flag_video(output);
+	bool has_audio = flag_audio(output);
 
-	if (encoded) {
+	if (flag_encoded(output)) {
 		pthread_mutex_lock(&output->interleaved_mutex);
 		reset_packet_data(output);
 		pthread_mutex_unlock(&output->interleaved_mutex);
@@ -2114,26 +2107,8 @@ static inline void signal_stop(struct obs_output *output)
 	calldata_free(&params);
 }
 
-static inline void convert_flags(const struct obs_output *output,
-				 uint32_t flags, bool *encoded, bool *has_video,
-				 bool *has_audio, bool *has_service)
+bool obs_output_can_begin_data_capture2(const obs_output_t *output)
 {
-	*encoded = (output->info.flags & OBS_OUTPUT_ENCODED) != 0;
-	if (!flags)
-		flags = output->info.flags;
-	else
-		flags &= output->info.flags;
-
-	*has_video = (flags & OBS_OUTPUT_VIDEO) != 0;
-	*has_audio = (flags & OBS_OUTPUT_AUDIO) != 0;
-	*has_service = (flags & OBS_OUTPUT_SERVICE) != 0;
-}
-
-bool obs_output_can_begin_data_capture(const obs_output_t *output,
-				       uint32_t flags)
-{
-	bool encoded, has_video, has_audio, has_service;
-
 	if (!obs_output_valid(output, "obs_output_can_begin_data_capture"))
 		return false;
 
@@ -2145,11 +2120,14 @@ bool obs_output_can_begin_data_capture(const obs_output_t *output,
 	if (data_capture_ending(output))
 		pthread_join(output->end_data_capture_thread, NULL);
 
-	convert_flags(output, flags, &encoded, &has_video, &has_audio,
-		      &has_service);
+	return can_begin_data_capture(output);
+}
 
-	return can_begin_data_capture(output, encoded, has_video, has_audio,
-				      has_service);
+bool obs_output_can_begin_data_capture(const obs_output_t *output,
+				       uint32_t flags)
+{
+	UNUSED_PARAMETER(flags);
+	return obs_output_can_begin_data_capture2(output);
 }
 
 static inline bool initialize_audio_encoders(obs_output_t *output)
@@ -2201,33 +2179,34 @@ static inline void pair_encoders(obs_output_t *output)
 	}
 }
 
-bool obs_output_initialize_encoders(obs_output_t *output, uint32_t flags)
+bool obs_output_initialize_encoders2(obs_output_t *output)
 {
-	bool encoded, has_video, has_audio, has_service;
-
 	if (!obs_output_valid(output, "obs_output_initialize_encoders"))
 		return false;
-
+	if (!log_flag_encoded(output, __FUNCTION__, false))
+		return false;
 	if (active(output))
 		return delay_active(output);
 
-	convert_flags(output, flags, &encoded, &has_video, &has_audio,
-		      &has_service);
-
-	if (!encoded)
-		return false;
-	if (has_video && !obs_encoder_initialize(output->video_encoder)) {
+	if (flag_video(output) &&
+	    !obs_encoder_initialize(output->video_encoder)) {
 		obs_output_set_last_error(
 			output,
 			obs_encoder_get_last_error(output->video_encoder));
 		return false;
 	}
-	if (has_audio && !initialize_audio_encoders(output))
+	if (flag_audio(output) && !initialize_audio_encoders(output))
 		return false;
 
 	return true;
 }
 
+bool obs_output_initialize_encoders(obs_output_t *output, uint32_t flags)
+{
+	UNUSED_PARAMETER(flags);
+	return obs_output_initialize_encoders2(output);
+}
+
 static bool begin_delayed_capture(obs_output_t *output)
 {
 	if (delay_capturing(output))
@@ -2284,10 +2263,8 @@ static void reset_raw_output(obs_output_t *output)
 	pause_reset(&output->pause);
 }
 
-bool obs_output_begin_data_capture(obs_output_t *output, uint32_t flags)
+bool obs_output_begin_data_capture2(obs_output_t *output)
 {
-	bool encoded, has_video, has_audio, has_service;
-
 	if (!obs_output_valid(output, "obs_output_begin_data_capture"))
 		return false;
 
@@ -2298,23 +2275,19 @@ bool obs_output_begin_data_capture(obs_output_t *output, uint32_t flags)
 
 	output->total_frames = 0;
 
-	convert_flags(output, flags, &encoded, &has_video, &has_audio,
-		      &has_service);
-
-	if (!encoded)
+	if (!log_flag_encoded(output, __FUNCTION__, false))
 		reset_raw_output(output);
 
-	if (!can_begin_data_capture(output, encoded, has_video, has_audio,
-				    has_service))
+	if (!can_begin_data_capture(output))
 		return false;
 
-	if (has_video && has_audio)
+	if (flag_video(output) && flag_audio(output))
 		pair_encoders(output);
 
 	os_atomic_set_bool(&output->data_active, true);
-	hook_data_capture(output, encoded, has_video, has_audio);
+	hook_data_capture(output);
 
-	if (has_service)
+	if (flag_service(output))
 		obs_service_activate(output->service);
 
 	do_output_signal(output, "activate");
@@ -2334,6 +2307,12 @@ bool obs_output_begin_data_capture(obs_output_t *output, uint32_t flags)
 	return true;
 }
 
+bool obs_output_begin_data_capture(obs_output_t *output, uint32_t flags)
+{
+	UNUSED_PARAMETER(flags);
+	return obs_output_begin_data_capture2(output);
+}
+
 static inline void stop_audio_encoders(obs_output_t *output,
 				       encoded_callback_t encoded_callback)
 {
@@ -2362,14 +2341,12 @@ static inline void stop_raw_audio(obs_output_t *output)
 
 static void *end_data_capture_thread(void *data)
 {
-	bool encoded, has_video, has_audio, has_service;
 	encoded_callback_t encoded_callback;
 	obs_output_t *output = data;
+	bool has_video = flag_video(output);
+	bool has_audio = flag_audio(output);
 
-	convert_flags(output, 0, &encoded, &has_video, &has_audio,
-		      &has_service);
-
-	if (encoded) {
+	if (flag_encoded(output)) {
 		if (output->active_delay_ns)
 			encoded_callback = process_delay;
 		else
@@ -2390,7 +2367,7 @@ static void *end_data_capture_thread(void *data)
 			stop_raw_audio(output);
 	}
 
-	if (has_service)
+	if (flag_service(output))
 		obs_service_deactivate(output->service, false);
 
 	if (output->active_delay_ns)

libobs/obs.h

@@ -2250,26 +2250,24 @@ obs_output_set_audio_conversion(obs_output_t *output,
 				const struct audio_convert_info *conversion);
 
 /** Returns whether data capture can begin with the specified flags */
-EXPORT bool obs_output_can_begin_data_capture(const obs_output_t *output,
-					      uint32_t flags);
+EXPORT bool obs_output_can_begin_data_capture2(const obs_output_t *output);
+OBS_DEPRECATED EXPORT bool
+obs_output_can_begin_data_capture(const obs_output_t *output, uint32_t flags);
 
 /** Initializes encoders (if any) */
-EXPORT bool obs_output_initialize_encoders(obs_output_t *output,
-					   uint32_t flags);
+EXPORT bool obs_output_initialize_encoders2(obs_output_t *output);
+OBS_DEPRECATED EXPORT bool obs_output_initialize_encoders(obs_output_t *output,
+							  uint32_t flags);
 
 /**
  * Begins data capture from media/encoders.
  *
  * @param  output  Output context
- * @param  flags   Set this to 0 to use default output flags set in the
- *                 obs_output_info structure, otherwise set to a either
- *                 OBS_OUTPUT_VIDEO or OBS_OUTPUT_AUDIO to specify whether to
- *                 connect audio or video.  This is useful for things like
- *                 ffmpeg which may or may not always want to use both audio
- *                 and video.
  * @return         true if successful, false otherwise.
  */
-EXPORT bool obs_output_begin_data_capture(obs_output_t *output, uint32_t flags);
+EXPORT bool obs_output_begin_data_capture2(obs_output_t *output);
+OBS_DEPRECATED EXPORT bool obs_output_begin_data_capture(obs_output_t *output,
+							 uint32_t flags);
 
 /** Ends data capture from media/encoders */
 EXPORT void obs_output_end_data_capture(obs_output_t *output);

plugins/aja/aja-output.cpp

@@ -1229,7 +1229,7 @@ static bool aja_output_start(void *data)
 
 	obs_output_set_audio_conversion(ajaOutput->GetOBSOutput(), &conversion);
 
-	if (!obs_output_begin_data_capture(ajaOutput->GetOBSOutput(), 0)) {
+	if (!obs_output_begin_data_capture2(ajaOutput->GetOBSOutput())) {
 		blog(LOG_ERROR,
 		     "aja_output_start: Begin OBS data capture failed!");
 		return false;

plugins/decklink/decklink-output.cpp

@@ -110,7 +110,7 @@ static bool decklink_output_start(void *data)
 
 	obs_output_set_audio_conversion(decklink->GetOutput(), &conversion);
 
-	if (!obs_output_begin_data_capture(decklink->GetOutput(), 0))
+	if (!obs_output_begin_data_capture2(decklink->GetOutput()))
 		return false;
 
 	return true;

plugins/linux-v4l2/v4l2-output.c

@@ -173,7 +173,7 @@ static bool try_connect(void *data, const char *device)
 	}
 
 	blog(LOG_INFO, "Virtual camera started");
-	obs_output_begin_data_capture(vcam->output, 0);
+	obs_output_begin_data_capture2(vcam->output);
 
 	return true;
 

plugins/mac-virtualcam/src/obs-plugin/plugin-main.mm

@@ -523,7 +523,7 @@ static bool virtualcam_output_start(void *data)
 		[vcam->machServer run];
 	}
 
-	if (!obs_output_begin_data_capture(vcam->output, 0)) {
+	if (!obs_output_begin_data_capture2(vcam->output)) {
 		return false;
 	}
 

plugins/obs-ffmpeg/obs-ffmpeg-hls-mux.c

@@ -118,9 +118,9 @@ bool ffmpeg_hls_mux_start(void *data)
 	obs_data_t *settings;
 	int keyint_sec;
 
-	if (!obs_output_can_begin_data_capture(stream->output, 0))
+	if (!obs_output_can_begin_data_capture2(stream->output))
 		return false;
-	if (!obs_output_initialize_encoders(stream->output, 0))
+	if (!obs_output_initialize_encoders2(stream->output))
 		return false;
 
 	service = obs_output_get_service(stream->output);
@@ -170,7 +170,7 @@ bool ffmpeg_hls_mux_start(void *data)
 	stream->dropped_frames = 0;
 	stream->min_priority = 0;
 
-	obs_output_begin_data_capture(stream->output, 0);
+	obs_output_begin_data_capture2(stream->output);
 
 	dstr_copy(&stream->printable_path, path_str);
 	info("Writing to path '%s'...", stream->printable_path.array);

plugins/obs-ffmpeg/obs-ffmpeg-mpegts.c

@@ -1025,9 +1025,9 @@ static bool set_config(struct ffmpeg_output *stream)
 		}
 		av_dump_format(ff_data->output, 0, NULL, 1);
 	}
-	if (!obs_output_can_begin_data_capture(stream->output, 0))
+	if (!obs_output_can_begin_data_capture2(stream->output))
 		return false;
-	if (!obs_output_initialize_encoders(stream->output, 0))
+	if (!obs_output_initialize_encoders2(stream->output))
 		return false;
 
 	ret = pthread_create(&stream->write_thread, NULL, write_thread, stream);
@@ -1042,7 +1042,7 @@ static bool set_config(struct ffmpeg_output *stream)
 	os_atomic_set_bool(&stream->active, true);
 	stream->write_thread_active = true;
 	stream->total_bytes = 0;
-	obs_output_begin_data_capture(stream->output, 0);
+	obs_output_begin_data_capture2(stream->output);
 
 	return true;
 fail:

plugins/obs-ffmpeg/obs-ffmpeg-mux.c

@@ -411,9 +411,9 @@ static inline bool ffmpeg_mux_start_internal(struct ffmpeg_muxer *stream,
 
 	update_encoder_settings(stream, path);
 
-	if (!obs_output_can_begin_data_capture(stream->output, 0))
+	if (!obs_output_can_begin_data_capture2(stream->output))
 		return false;
-	if (!obs_output_initialize_encoders(stream->output, 0))
+	if (!obs_output_initialize_encoders2(stream->output))
 		return false;
 
 	if (stream->is_network) {
@@ -468,7 +468,7 @@ static inline bool ffmpeg_mux_start_internal(struct ffmpeg_muxer *stream,
 	os_atomic_set_bool(&stream->active, true);
 	os_atomic_set_bool(&stream->capturing, true);
 	stream->total_bytes = 0;
-	obs_output_begin_data_capture(stream->output, 0);
+	obs_output_begin_data_capture2(stream->output);
 
 	info("Writing file '%s'...", stream->path.array);
 	return true;
@@ -1039,9 +1039,9 @@ static bool replay_buffer_start(void *data)
 {
 	struct ffmpeg_muxer *stream = data;
 
-	if (!obs_output_can_begin_data_capture(stream->output, 0))
+	if (!obs_output_can_begin_data_capture2(stream->output))
 		return false;
-	if (!obs_output_initialize_encoders(stream->output, 0))
+	if (!obs_output_initialize_encoders2(stream->output))
 		return false;
 
 	obs_data_t *s = obs_output_get_settings(stream->output);
@@ -1052,7 +1052,7 @@ static bool replay_buffer_start(void *data)
 	os_atomic_set_bool(&stream->active, true);
 	os_atomic_set_bool(&stream->capturing, true);
 	stream->total_bytes = 0;
-	obs_output_begin_data_capture(stream->output, 0);
+	obs_output_begin_data_capture2(stream->output);
 
 	return true;
 }

plugins/obs-ffmpeg/obs-ffmpeg-output.c

@@ -1209,7 +1209,7 @@ static bool try_connect(struct ffmpeg_output *output)
 
 	output->active = true;
 
-	if (!obs_output_can_begin_data_capture(output->output, 0))
+	if (!obs_output_can_begin_data_capture2(output->output))
 		return false;
 
 	ret = pthread_create(&output->write_thread, NULL, write_thread, output);
@@ -1223,7 +1223,7 @@ static bool try_connect(struct ffmpeg_output *output)
 
 	obs_output_set_video_conversion(output->output, NULL);
 	obs_output_set_audio_conversion(output->output, &aci);
-	obs_output_begin_data_capture(output->output, 0);
+	obs_output_begin_data_capture2(output->output);
 	output->write_thread_active = true;
 	return true;
 }

plugins/obs-outputs/flv-output.c

@@ -154,9 +154,9 @@ static bool flv_output_start(void *data)
 	obs_data_t *settings;
 	const char *path;
 
-	if (!obs_output_can_begin_data_capture(stream->output, 0))
+	if (!obs_output_can_begin_data_capture2(stream->output))
 		return false;
-	if (!obs_output_initialize_encoders(stream->output, 0))
+	if (!obs_output_initialize_encoders2(stream->output))
 		return false;
 
 	stream->got_first_video = false;
@@ -177,7 +177,7 @@ static bool flv_output_start(void *data)
 
 	/* write headers and start capture */
 	os_atomic_set_bool(&stream->active, true);
-	obs_output_begin_data_capture(stream->output, 0);
+	obs_output_begin_data_capture2(stream->output);
 
 	info("Writing FLV file '%s'...", stream->path.array);
 	return true;

plugins/obs-outputs/ftl-stream.c

@@ -593,7 +593,7 @@ static int init_send(struct ftl_stream *stream)
 
 	os_atomic_set_bool(&stream->active, true);
 
-	obs_output_begin_data_capture(stream->output, 0);
+	obs_output_begin_data_capture2(stream->output);
 
 	return OBS_OUTPUT_SUCCESS;
 }
@@ -649,10 +649,10 @@ static bool ftl_stream_start(void *data)
 	obs_data_set_int(video_settings, "bf", 0);
 	obs_data_release(video_settings);
 
-	if (!obs_output_can_begin_data_capture(stream->output, 0)) {
+	if (!obs_output_can_begin_data_capture2(stream->output)) {
 		return false;
 	}
-	if (!obs_output_initialize_encoders(stream->output, 0)) {
+	if (!obs_output_initialize_encoders2(stream->output)) {
 		return false;
 	}
 

plugins/obs-outputs/null-output.c

@@ -51,15 +51,15 @@ static bool null_output_start(void *data)
 {
 	struct null_output *context = data;
 
-	if (!obs_output_can_begin_data_capture(context->output, 0))
+	if (!obs_output_can_begin_data_capture2(context->output))
 		return false;
-	if (!obs_output_initialize_encoders(context->output, 0))
+	if (!obs_output_initialize_encoders2(context->output))
 		return false;
 
 	if (context->stop_thread_active)
 		pthread_join(context->stop_thread, NULL);
 
-	obs_output_begin_data_capture(context->output, 0);
+	obs_output_begin_data_capture2(context->output);
 	return true;
 }
 

plugins/obs-outputs/rtmp-stream.c

@@ -1137,7 +1137,7 @@ static int init_send(struct rtmp_stream *stream)
 	}
 
 	if (!silently_reconnecting(stream))
-		obs_output_begin_data_capture(stream->output, 0);
+		obs_output_begin_data_capture2(stream->output);
 
 	return OBS_OUTPUT_SUCCESS;
 }
@@ -1456,9 +1456,9 @@ static bool rtmp_stream_start(void *data)
 	struct rtmp_stream *stream = data;
 
 	if (!silently_reconnecting(stream)) {
-		if (!obs_output_can_begin_data_capture(stream->output, 0))
+		if (!obs_output_can_begin_data_capture2(stream->output))
 			return false;
-		if (!obs_output_initialize_encoders(stream->output, 0))
+		if (!obs_output_initialize_encoders2(stream->output))
 			return false;
 	}
 

plugins/win-dshow/virtualcam.c

@@ -68,7 +68,7 @@ static bool virtualcam_start(void *data)
 	os_atomic_set_bool(&vcam->active, true);
 	os_atomic_set_bool(&vcam->stopping, false);
 	blog(LOG_INFO, "Virtual output started");
-	obs_output_begin_data_capture(vcam->output, 0);
+	obs_output_begin_data_capture2(vcam->output);
 	return true;
 }