UI: Select streaming output based on the protocol

UI/data/locale/en-US.ini

@@ -218,6 +218,7 @@ Basic.AutoConfig.TestPage="Final Results"
 Basic.AutoConfig.TestPage.SubTitle.Testing="The program is now executing a set of tests to estimate the ideal settings"
 Basic.AutoConfig.TestPage.SubTitle.Complete="Testing complete"
 Basic.AutoConfig.TestPage.TestingBandwidth="Performing bandwidth test, this may take a few minutes..."
+Basic.AutoConfig.TestPage.TestingBandwidth.NoOutput="No output for the protocol of this service was found"
 Basic.AutoConfig.TestPage.TestingBandwidth.Connecting="Connecting to: %1..."
 Basic.AutoConfig.TestPage.TestingBandwidth.ConnectFailed="Failed to connect to any servers, please check your internet connection and try again."
 Basic.AutoConfig.TestPage.TestingBandwidth.Server="Testing bandwidth for: %1"

UI/window-basic-auto-config-test.cpp

@@ -88,6 +88,7 @@ public:
 #define SUBTITLE_TESTING TEST_STR("Subtitle.Testing")
 #define SUBTITLE_COMPLETE TEST_STR("Subtitle.Complete")
 #define TEST_BW TEST_STR("TestingBandwidth")
+#define TEST_BW_NO_OUTPUT TEST_STR("TestingBandwidth.NoOutput")
 #define TEST_BW_CONNECTING TEST_STR("TestingBandwidth.Connecting")
 #define TEST_BW_CONNECT_FAIL TEST_STR("TestingBandwidth.ConnectFailed")
 #define TEST_BW_SERVER TEST_STR("TestingBandwidth.Server")
@@ -155,6 +156,23 @@ static inline void string_depad_key(string &key)
 
 const char *FindAudioEncoderFromCodec(const char *type);
 
+static inline bool can_use_output(const char *prot, const char *output,
+				  const char *prot_test1,
+				  const char *prot_test2 = nullptr)
+{
+	return (strcmp(prot, prot_test1) == 0 ||
+		(prot_test2 && strcmp(prot, prot_test2) == 0)) &&
+	       (obs_get_output_flags(output) & OBS_OUTPUT_SERVICE) != 0;
+}
+
+static bool return_first_id(void *data, const char *id)
+{
+	const char **output = (const char **)data;
+
+	*output = id;
+	return false;
+}
+
 void AutoConfigTestPage::TestBandwidthThread()
 {
 	bool connected = false;
@@ -268,9 +286,37 @@ void AutoConfigTestPage::TestBandwidthThread()
 	/* -----------------------------------*/
 	/* create output                      */
 
-	const char *output_type = obs_service_get_output_type(service);
-	if (!output_type)
-		output_type = "rtmp_output";
+	/* Check if the service has a preferred output type */
+	const char *output_type =
+		obs_service_get_preferred_output_type(service);
+	if (!output_type ||
+	    (obs_get_output_flags(output_type) & OBS_OUTPUT_SERVICE) == 0) {
+		/* Otherwise, prefer first-party output types */
+		const char *protocol = obs_service_get_protocol(service);
+
+		if (can_use_output(protocol, "rtmp_output", "RTMP", "RTMPS")) {
+			output_type = "rtmp_output";
+		} else if (can_use_output(protocol, "ffmpeg_hls_muxer",
+					  "HLS")) {
+			output_type = "ffmpeg_hls_muxer";
+		} else if (can_use_output(protocol, "ffmpeg_mpegts_muxer",
+					  "SRT", "RIST")) {
+			output_type = "ffmpeg_mpegts_muxer";
+		}
+
+		/* If third-party protocol, use the first enumerated type */
+		if (!output_type)
+			obs_enum_output_types_with_protocol(
+				protocol, &output_type, return_first_id);
+
+		/* If none, fail */
+		if (!output_type) {
+			QMetaObject::invokeMethod(
+				this, "Failure",
+				Q_ARG(QString, QTStr(TEST_BW_NO_OUTPUT)));
+			return;
+		}
+	}
 
 	OBSOutputAutoRelease output =
 		obs_output_create(output_type, "test_stream", nullptr, nullptr);

UI/window-basic-main-outputs.cpp

@@ -210,6 +210,73 @@ static bool CreateAACEncoder(OBSEncoder &res, string &id, int bitrate,
 	return false;
 }
 
+static inline bool can_use_output(const char *prot, const char *output,
+				  const char *prot_test1,
+				  const char *prot_test2 = nullptr)
+{
+	return (strcmp(prot, prot_test1) == 0 ||
+		(prot_test2 && strcmp(prot, prot_test2) == 0)) &&
+	       (obs_get_output_flags(output) & OBS_OUTPUT_SERVICE) != 0;
+}
+
+static bool return_first_id(void *data, const char *id)
+{
+	const char **output = (const char **)data;
+
+	*output = id;
+	return false;
+}
+
+static const char *GetStreamOutputType(const obs_service_t *service)
+{
+	const char *protocol = obs_service_get_protocol(service);
+	const char *output = nullptr;
+
+	if (!protocol) {
+		blog(LOG_WARNING, "The service '%s' has no protocol set",
+		     obs_service_get_id(service));
+		return nullptr;
+	}
+
+	if (!obs_is_output_protocol_registered(protocol)) {
+		blog(LOG_WARNING, "The protocol '%s' is not registered",
+		     protocol);
+		return nullptr;
+	}
+
+	/* Check if the service has a preferred output type */
+	output = obs_service_get_preferred_output_type(service);
+	if (output) {
+		if ((obs_get_output_flags(output) & OBS_OUTPUT_SERVICE) != 0)
+			return output;
+
+		blog(LOG_WARNING,
+		     "The output '%s' is not registered, fallback to another one",
+		     output);
+	}
+
+	/* Otherwise, prefer first-party output types */
+	if (can_use_output(protocol, "rtmp_output", "RTMP", "RTMPS")) {
+		return "rtmp_output";
+	} else if (can_use_output(protocol, "ffmpeg_hls_muxer", "HLS")) {
+		return "ffmpeg_hls_muxer";
+	} else if (can_use_output(protocol, "ffmpeg_mpegts_muxer", "SRT",
+				  "RIST")) {
+		return "ffmpeg_mpegts_muxer";
+	}
+
+	/* If third-party protocol, use the first enumerated type */
+	obs_enum_output_types_with_protocol(protocol, &output, return_first_id);
+	if (output)
+		return output;
+
+	blog(LOG_WARNING,
+	     "No output compatible with the service '%s' is registered",
+	     obs_service_get_id(service));
+
+	return nullptr;
+}
+
 /* ------------------------------------------------------------------------ */
 
 inline BasicOutputHandler::BasicOutputHandler(OBSBasic *main_) : main(main_)
@@ -913,18 +980,9 @@ bool SimpleOutput::SetupStreaming(obs_service_t *service)
 
 	/* --------------------- */
 
-	const char *type = obs_service_get_output_type(service);
-	if (!type) {
-		type = "rtmp_output";
-		const char *url = obs_service_get_url(service);
-		if (url != NULL &&
-		    strncmp(url, FTL_PROTOCOL, strlen(FTL_PROTOCOL)) == 0) {
-			type = "ftl_output";
-		} else if (url != NULL && strncmp(url, RTMP_PROTOCOL,
-						  strlen(RTMP_PROTOCOL)) != 0) {
-			type = "ffmpeg_mpegts_muxer";
-		}
-	}
+	const char *type = GetStreamOutputType(service);
+	if (!type)
+		return false;
 
 	/* XXX: this is messy and disgusting and should be refactored */
 	if (outputType != type) {
@@ -1898,18 +1956,9 @@ bool AdvancedOutput::SetupStreaming(obs_service_t *service)
 
 	/* --------------------- */
 
-	const char *type = obs_service_get_output_type(service);
-	if (!type) {
-		type = "rtmp_output";
-		const char *url = obs_service_get_url(service);
-		if (url != NULL &&
-		    strncmp(url, FTL_PROTOCOL, strlen(FTL_PROTOCOL)) == 0) {
-			type = "ftl_output";
-		} else if (url != NULL && strncmp(url, RTMP_PROTOCOL,
-						  strlen(RTMP_PROTOCOL)) != 0) {
-			type = "ffmpeg_mpegts_muxer";
-		}
-	}
+	const char *type = GetStreamOutputType(service);
+	if (!type)
+		return false;
 
 	/* XXX: this is messy and disgusting and should be refactored */
 	if (outputType != type) {