UI: Omit stream codecs the service doesn't support

This change makes it so that if you select a service, it will check to
see what codecs that service supports, and only list encoders of those
codecs.

If the service doesn't support a codec and you currently have an
unsupported codec selected in output settings, then it'll prompt you
with a dialog telling the user it will switch to a supported codec, and
if they click yes, then it'll change the codec for the user. If they
click no, then it'll switch back to the previously selected service.

UI/data/locale/en-US.ini

@@ -921,6 +921,8 @@ Basic.Settings.Output.Warn.EnforceResolutionFPS.Title="Incompatible Resolution/F
 Basic.Settings.Output.Warn.EnforceResolutionFPS.Msg="This streaming service does not support your current output resolution and/or framerate. They will be changed to the closest compatible value:\n\n%1\n\nDo you want to continue?"
 Basic.Settings.Output.Warn.EnforceResolutionFPS.Resolution="Resolution: %1"
 Basic.Settings.Output.Warn.EnforceResolutionFPS.FPS="FPS: %1"
+Basic.Settings.Output.Warn.ServiceCodecCompatibility.Title="Incompatible Encoder"
+Basic.Settings.Output.Warn.ServiceCodecCompatibility.Msg="The streaming service \"%1\" does not support the encoder \"%2\". The encoder will be changed to \"%3\".\n\nDo you want to continue?"
 Basic.Settings.Output.VideoBitrate="Video Bitrate"
 Basic.Settings.Output.AudioBitrate="Audio Bitrate"
 Basic.Settings.Output.Reconnect="Automatically Reconnect"

UI/window-basic-settings-stream.cpp

@@ -122,6 +122,7 @@ void OBSBasicSettings::LoadStream1Settings()
 	if (strcmp(type, "rtmp_custom") == 0) {
 		ui->service->setCurrentIndex(0);
 		ui->customServer->setText(server);
+		lastServiceIdx = 0;
 
 		bool use_auth = obs_data_get_bool(settings, "use_auth");
 		const char *username =
@@ -139,6 +140,7 @@ void OBSBasicSettings::LoadStream1Settings()
 			idx = 1;
 		}
 		ui->service->setCurrentIndex(idx);
+		lastServiceIdx = idx;
 
 		bool bw_test = obs_data_get_bool(settings, "bwtest");
 		ui->bandwidthTestEnable->setChecked(bw_test);
@@ -964,6 +966,9 @@ void OBSBasicSettings::UpdateResFPSLimits()
 	if (loading)
 		return;
 
+	if (!ServiceSupportsCodecCheck())
+		return;
+
 	int idx = ui->service->currentIndex();
 	if (idx == -1)
 		return;
@@ -1177,3 +1182,266 @@ bool OBSBasicSettings::IsServiceOutputHasNetworkFeatures()
 
 	return false;
 }
+
+static bool service_supports_codec(const char **codecs, const char *codec)
+{
+	if (!codecs)
+		return true;
+
+	while (*codecs) {
+		if (strcmp(*codecs, codec) == 0)
+			return true;
+		codecs++;
+	}
+
+	return false;
+}
+
+extern bool EncoderAvailable(const char *encoder);
+extern const char *get_simple_output_encoder(const char *name);
+
+static inline bool service_supports_encoder(const char **codecs,
+					    const char *encoder)
+{
+	if (!EncoderAvailable(encoder))
+		return false;
+
+	const char *codec = obs_get_encoder_codec(encoder);
+	return service_supports_codec(codecs, codec);
+}
+
+bool OBSBasicSettings::ServiceAndCodecCompatible()
+{
+	if (IsCustomService())
+		return true;
+	if (ui->service->currentData().toInt() == (int)ListOpt::ShowAll)
+		return true;
+
+	bool simple = (ui->outputMode->currentIndex() == 0);
+
+	OBSService service = SpawnTempService();
+	const char **codecs = obs_service_get_supported_video_codecs(service);
+	const char *codec;
+
+	if (simple) {
+		QString encoder =
+			ui->simpleOutStrEncoder->currentData().toString();
+		const char *id = get_simple_output_encoder(QT_TO_UTF8(encoder));
+		codec = obs_get_encoder_codec(id);
+	} else {
+		QString encoder = ui->advOutEncoder->currentData().toString();
+		codec = obs_get_encoder_codec(QT_TO_UTF8(encoder));
+	}
+
+	return service_supports_codec(codecs, codec);
+}
+
+/* we really need a way to find fallbacks in a less hardcoded way. maybe. */
+static QString get_adv_fallback(const QString &enc)
+{
+	if (enc == "jim_hevc_nvenc")
+		return "jim_nvenc";
+	if (enc == "h265_texture_amf")
+		return "h264_texture_amf";
+	return "obs_x264";
+}
+
+static QString get_simple_fallback(const QString &enc)
+{
+	if (enc == SIMPLE_ENCODER_NVENC_HEVC)
+		return SIMPLE_ENCODER_NVENC;
+	if (enc == SIMPLE_ENCODER_AMD_HEVC)
+		return SIMPLE_ENCODER_AMD;
+	return SIMPLE_ENCODER_X264;
+}
+
+bool OBSBasicSettings::ServiceSupportsCodecCheck()
+{
+	if (ServiceAndCodecCompatible()) {
+		if (lastServiceIdx != ui->service->currentIndex())
+			ResetEncoders(true);
+		return true;
+	}
+
+	QString service = ui->service->currentText();
+	QString cur_name;
+	QString fb_name;
+	bool simple = (ui->outputMode->currentIndex() == 0);
+
+	/* ------------------------------------------------- */
+	/* get current codec                                 */
+
+	if (simple) {
+		QString cur_enc =
+			ui->simpleOutStrEncoder->currentData().toString();
+		QString fb_enc = get_simple_fallback(cur_enc);
+
+		int cur_idx = ui->simpleOutStrEncoder->findData(cur_enc);
+		int fb_idx = ui->simpleOutStrEncoder->findData(fb_enc);
+
+		cur_name = ui->simpleOutStrEncoder->itemText(cur_idx);
+		fb_name = ui->simpleOutStrEncoder->itemText(fb_idx);
+	} else {
+		QString cur_enc = ui->advOutEncoder->currentData().toString();
+		QString fb_enc = get_adv_fallback(cur_enc);
+
+		cur_name = obs_encoder_get_display_name(QT_TO_UTF8(cur_enc));
+		fb_name = obs_encoder_get_display_name(QT_TO_UTF8(fb_enc));
+	}
+
+#define WARNING_VAL(x) \
+	QTStr("Basic.Settings.Output.Warn.ServiceCodecCompatibility." x)
+
+	QString msg = WARNING_VAL("Msg").arg(service, cur_name, fb_name);
+	auto button = OBSMessageBox::question(this, WARNING_VAL("Title"), msg);
+#undef WARNING_VAL
+
+	if (button == QMessageBox::No) {
+		QMetaObject::invokeMethod(ui->service, "setCurrentIndex",
+					  Qt::QueuedConnection,
+					  Q_ARG(int, lastServiceIdx));
+		return false;
+	}
+
+	ResetEncoders(true);
+	return true;
+}
+
+#define TEXT_USE_STREAM_ENC \
+	QTStr("Basic.Settings.Output.Adv.Recording.UseStreamEncoder")
+
+void OBSBasicSettings::ResetEncoders(bool streamOnly)
+{
+	QString lastAdvEnc = ui->advOutRecEncoder->currentData().toString();
+	QString lastEnc = ui->simpleOutStrEncoder->currentData().toString();
+	OBSService service = SpawnTempService();
+	const char **codecs = obs_service_get_supported_video_codecs(service);
+	const char *type;
+	size_t idx = 0;
+
+	QSignalBlocker s1(ui->simpleOutStrEncoder);
+	QSignalBlocker s2(ui->advOutEncoder);
+
+	/* ------------------------------------------------- */
+	/* clear encoder lists                               */
+
+	ui->simpleOutStrEncoder->clear();
+	ui->advOutEncoder->clear();
+
+	if (!streamOnly) {
+		ui->advOutRecEncoder->clear();
+		ui->advOutRecEncoder->addItem(TEXT_USE_STREAM_ENC, "none");
+	}
+
+	/* ------------------------------------------------- */
+	/* load advanced stream/recording encoders           */
+
+	while (obs_enum_encoder_types(idx++, &type)) {
+		const char *name = obs_encoder_get_display_name(type);
+		const char *codec = obs_get_encoder_codec(type);
+		uint32_t caps = obs_get_encoder_caps(type);
+
+		if (obs_get_encoder_type(type) != OBS_ENCODER_VIDEO)
+			continue;
+
+		const char *streaming_codecs[] = {
+			"h264",
+#ifdef ENABLE_HEVC
+			"hevc",
+#endif
+		};
+
+		bool is_streaming_codec = false;
+		for (const char *test_codec : streaming_codecs) {
+			if (strcmp(codec, test_codec) == 0) {
+				is_streaming_codec = true;
+				break;
+			}
+		}
+		if ((caps & ENCODER_HIDE_FLAGS) != 0)
+			continue;
+
+		QString qName = QT_UTF8(name);
+		QString qType = QT_UTF8(type);
+
+		if (is_streaming_codec && service_supports_codec(codecs, codec))
+			ui->advOutEncoder->addItem(qName, qType);
+		if (!streamOnly)
+			ui->advOutRecEncoder->addItem(qName, qType);
+	}
+
+	/* ------------------------------------------------- */
+	/* load simple stream encoders                       */
+
+#define ENCODER_STR(str) QTStr("Basic.Settings.Output.Simple.Encoder." str)
+
+	ui->simpleOutStrEncoder->addItem(ENCODER_STR("Software"),
+					 QString(SIMPLE_ENCODER_X264));
+	if (service_supports_encoder(codecs, "obs_qsv11"))
+		ui->simpleOutStrEncoder->addItem(
+			ENCODER_STR("Hardware.QSV.H264"),
+			QString(SIMPLE_ENCODER_QSV));
+	if (service_supports_encoder(codecs, "ffmpeg_nvenc"))
+		ui->simpleOutStrEncoder->addItem(
+			ENCODER_STR("Hardware.NVENC.H264"),
+			QString(SIMPLE_ENCODER_NVENC));
+#ifdef ENABLE_HEVC
+	if (service_supports_encoder(codecs, "h265_texture_amf"))
+		ui->simpleOutStrEncoder->addItem(
+			ENCODER_STR("Hardware.AMD.HEVC"),
+			QString(SIMPLE_ENCODER_AMD_HEVC));
+	if (service_supports_encoder(codecs, "ffmpeg_hevc_nvenc"))
+		ui->simpleOutStrEncoder->addItem(
+			ENCODER_STR("Hardware.NVENC.HEVC"),
+			QString(SIMPLE_ENCODER_NVENC_HEVC));
+#endif
+	if (service_supports_encoder(codecs, "h264_texture_amf"))
+		ui->simpleOutStrEncoder->addItem(
+			ENCODER_STR("Hardware.AMD.H264"),
+			QString(SIMPLE_ENCODER_AMD));
+/* Preprocessor guard required for the macOS version check */
+#ifdef __APPLE__
+	if (service_supports_encoder(
+		    codecs, "com.apple.videotoolbox.videoencoder.ave.avc")
+#ifndef __aarch64__
+	    && os_get_emulation_status() == true
+#endif
+	) {
+		if (__builtin_available(macOS 13.0, *)) {
+			ui->simpleOutStrEncoder->addItem(
+				ENCODER_STR("Hardware.Apple.H264"),
+				QString(SIMPLE_ENCODER_APPLE_H264));
+		}
+	}
+#endif
+#undef ENCODER_STR
+
+	/* ------------------------------------------------- */
+	/* Find fallback encoders                            */
+
+	if (!lastAdvEnc.isEmpty()) {
+		int idx = ui->advOutEncoder->findData(lastAdvEnc);
+		if (idx == -1) {
+			lastAdvEnc = get_adv_fallback(lastAdvEnc);
+			ui->advOutEncoder->setProperty("changed",
+						       QVariant(true));
+			OutputsChanged();
+		}
+
+		idx = ui->advOutEncoder->findData(lastAdvEnc);
+		ui->advOutEncoder->setCurrentIndex(idx);
+	}
+
+	if (!lastEnc.isEmpty()) {
+		int idx = ui->simpleOutStrEncoder->findData(lastEnc);
+		if (idx == -1) {
+			lastEnc = get_simple_fallback(lastEnc);
+			ui->simpleOutStrEncoder->setProperty("changed",
+							     QVariant(true));
+			OutputsChanged();
+		}
+
+		idx = ui->simpleOutStrEncoder->findData(lastEnc);
+		ui->simpleOutStrEncoder->setCurrentIndex(idx);
+	}
+}

UI/window-basic-settings.cpp

@@ -50,9 +50,6 @@
 #include <util/dstr.hpp>
 #include "ui-config.h"
 
-#define ENCODER_HIDE_FLAGS \
-	(OBS_ENCODER_CAP_DEPRECATED | OBS_ENCODER_CAP_INTERNAL)
-
 using namespace std;
 
 class SettingsEventFilter : public QObject {
@@ -704,7 +701,6 @@ OBSBasicSettings::OBSBasicSettings(QWidget *parent)
 
 	installEventFilter(new SettingsEventFilter());
 
-	LoadEncoderTypes();
 	LoadColorRanges();
 	LoadColorSpaces();
 	LoadColorFormats();
@@ -752,7 +748,6 @@ OBSBasicSettings::OBSBasicSettings(QWidget *parent)
 				   this);
 
 	FillSimpleRecordingValues();
-	FillSimpleStreamingValues();
 	if (obs_audio_monitoring_available())
 		FillAudioMonitoringDevices();
 
@@ -998,49 +993,6 @@ void OBSBasicSettings::SaveSpinBox(QSpinBox *widget, const char *section,
 		config_set_int(main->Config(), section, value, widget->value());
 }
 
-#define TEXT_USE_STREAM_ENC \
-	QTStr("Basic.Settings.Output.Adv.Recording.UseStreamEncoder")
-
-void OBSBasicSettings::LoadEncoderTypes()
-{
-	const char *type;
-	size_t idx = 0;
-
-	ui->advOutRecEncoder->addItem(TEXT_USE_STREAM_ENC, "none");
-
-	while (obs_enum_encoder_types(idx++, &type)) {
-		const char *name = obs_encoder_get_display_name(type);
-		const char *codec = obs_get_encoder_codec(type);
-		uint32_t caps = obs_get_encoder_caps(type);
-
-		if (obs_get_encoder_type(type) != OBS_ENCODER_VIDEO)
-			continue;
-
-		const char *streaming_codecs[] = {
-			"h264",
-#ifdef ENABLE_HEVC
-			"hevc",
-#endif
-		};
-		bool is_streaming_codec = false;
-		for (const char *test_codec : streaming_codecs) {
-			if (strcmp(codec, test_codec) == 0) {
-				is_streaming_codec = true;
-				break;
-			}
-		}
-		if ((caps & ENCODER_HIDE_FLAGS) != 0)
-			continue;
-
-		QString qName = QT_UTF8(name);
-		QString qType = QT_UTF8(type);
-
-		if (is_streaming_codec)
-			ui->advOutEncoder->addItem(qName, qType);
-		ui->advOutRecEncoder->addItem(qName, qType);
-	}
-}
-
 #define CS_PARTIAL_STR QTStr("Basic.Settings.Advanced.Video.ColorRange.Partial")
 #define CS_FULL_STR QTStr("Basic.Settings.Advanced.Video.ColorRange.Full")
 
@@ -2246,6 +2198,8 @@ void OBSBasicSettings::LoadOutputSettings()
 {
 	loading = true;
 
+	ResetEncoders();
+
 	const char *mode = config_get_string(main->Config(), "Output", "Mode");
 
 	int modeIdx = astrcmpi(mode, "Advanced") == 0 ? 1 : 0;
@@ -4797,46 +4751,6 @@ void OBSBasicSettings::FillSimpleRecordingValues()
 			ENCODER_STR("Hardware.Apple.H264"),
 			QString(SIMPLE_ENCODER_APPLE_H264));
 #undef ADD_QUALITY
-}
-
-void OBSBasicSettings::FillSimpleStreamingValues()
-{
-	ui->simpleOutStrEncoder->addItem(ENCODER_STR("Software"),
-					 QString(SIMPLE_ENCODER_X264));
-	if (EncoderAvailable("obs_qsv11"))
-		ui->simpleOutStrEncoder->addItem(
-			ENCODER_STR("Hardware.QSV.H264"),
-			QString(SIMPLE_ENCODER_QSV));
-	if (EncoderAvailable("ffmpeg_nvenc"))
-		ui->simpleOutStrEncoder->addItem(
-			ENCODER_STR("Hardware.NVENC.H264"),
-			QString(SIMPLE_ENCODER_NVENC));
-#ifdef ENABLE_HEVC
-	if (EncoderAvailable("h265_texture_amf"))
-		ui->simpleOutStrEncoder->addItem(
-			ENCODER_STR("Hardware.AMD.HEVC"),
-			QString(SIMPLE_ENCODER_AMD_HEVC));
-	if (EncoderAvailable("ffmpeg_hevc_nvenc"))
-		ui->simpleOutStrEncoder->addItem(
-			ENCODER_STR("Hardware.NVENC.HEVC"),
-			QString(SIMPLE_ENCODER_NVENC_HEVC));
-#endif
-	if (EncoderAvailable("h264_texture_amf"))
-		ui->simpleOutStrEncoder->addItem(
-			ENCODER_STR("Hardware.AMD.H264"),
-			QString(SIMPLE_ENCODER_AMD));
-/* Preprocessor guard required for the macOS version check */
-#ifdef __APPLE__
-	if (EncoderAvailable("com.apple.videotoolbox.videoencoder.ave.avc")
-#ifndef __aarch64__
-	    && os_get_emulation_status() == true
-#endif
-	)
-		if (__builtin_available(macOS 13.0, *))
-			ui->simpleOutStrEncoder->addItem(
-				ENCODER_STR("Hardware.Apple.H264"),
-				QString(SIMPLE_ENCODER_APPLE_H264));
-#endif
 #undef ENCODER_STR
 }
 
@@ -5026,6 +4940,9 @@ void OBSBasicSettings::SimpleReplayBufferChanged()
 	UpdateAutomaticReplayBufferCheckboxes();
 }
 
+#define TEXT_USE_STREAM_ENC \
+	QTStr("Basic.Settings.Output.Adv.Recording.UseStreamEncoder")
+
 void OBSBasicSettings::AdvReplayBufferChanged()
 {
 	obs_data_t *settings;

UI/window-basic-settings.hpp

@@ -127,6 +127,9 @@ private:
 	int lastIgnoreRecommended = -1;
 	int lastChannelSetupIdx = 0;
 
+	static constexpr uint32_t ENCODER_HIDE_FLAGS =
+		(OBS_ENCODER_CAP_DEPRECATED | OBS_ENCODER_CAP_INTERNAL);
+
 	OBSFFFormatDesc formats;
 
 	OBSPropertiesView *streamProperties = nullptr;
@@ -221,7 +224,7 @@ private:
 
 	bool QueryChanges();
 
-	void LoadEncoderTypes();
+	void ResetEncoders(bool streamOnly = false);
 	void LoadColorRanges();
 	void LoadColorSpaces();
 	void LoadColorFormats();
@@ -319,7 +322,6 @@ private:
 	void UpdateAdvOutStreamDelayEstimate();
 
 	void FillSimpleRecordingValues();
-	void FillSimpleStreamingValues();
 	void FillAudioMonitoringDevices();
 
 	void RecalcOutputResPixels(const char *resText);
@@ -348,6 +350,9 @@ private:
 
 	bool IsServiceOutputHasNetworkFeatures();
 
+	bool ServiceAndCodecCompatible();
+	bool ServiceSupportsCodecCheck();
+
 private slots:
 	void on_theme_activated(int idx);