UI: If hardware encoder selected, disable post rescale

The post-output rescale option in advanced output mode cannot currently
be used with texture-based encoders, so disable the option when a
texture-based encoder is selected.

UI/window-basic-settings.cpp

@@ -3421,35 +3421,50 @@ void OBSBasicSettings::on_advOutFFPathBrowse_clicked()
 
 void OBSBasicSettings::on_advOutEncoder_currentIndexChanged(int idx)
 {
-	if (loading)
-		return;
-
 	QString encoder = GetComboData(ui->advOutEncoder);
-	bool loadSettings = encoder == curAdvStreamEncoder;
+	if (!loading) {
+		bool loadSettings = encoder == curAdvStreamEncoder;
 
-	delete streamEncoderProps;
-	streamEncoderProps = CreateEncoderPropertyView(QT_TO_UTF8(encoder),
-			loadSettings ? "streamEncoder.json" : nullptr, true);
-	ui->advOutputStreamTab->layout()->addWidget(streamEncoderProps);
+		delete streamEncoderProps;
+		streamEncoderProps = CreateEncoderPropertyView(
+				QT_TO_UTF8(encoder),
+				loadSettings ? "streamEncoder.json" : nullptr,
+				true);
+		ui->advOutputStreamTab->layout()->addWidget(streamEncoderProps);
+	}
+
+	uint32_t caps = obs_get_encoder_caps(QT_TO_UTF8(encoder));
+
+	if (caps & OBS_ENCODER_CAP_PASS_TEXTURE) {
+		ui->advOutUseRescale->setChecked(false);
+		ui->advOutUseRescale->setEnabled(false);
+		ui->advOutRescale->setEnabled(false);
+	} else {
+		ui->advOutUseRescale->setEnabled(true);
+		ui->advOutRescale->setEnabled(true);
+	}
 
 	UNUSED_PARAMETER(idx);
 }
 
 void OBSBasicSettings::on_advOutRecEncoder_currentIndexChanged(int idx)
 {
-	if (loading)
-		return;
+	if (!loading) {
+		ui->advOutRecUseRescale->setEnabled(idx > 0);
+		ui->advOutRecRescaleContainer->setEnabled(idx > 0);
 
-	ui->advOutRecUseRescale->setEnabled(idx > 0);
-	ui->advOutRecRescaleContainer->setEnabled(idx > 0);
+		delete recordEncoderProps;
+		recordEncoderProps = nullptr;
+	}
 
-	delete recordEncoderProps;
-	recordEncoderProps = nullptr;
+	if (idx <= 0) {
+		return;
+	}
 
-	if (idx > 0) {
-		QString encoder = GetComboData(ui->advOutRecEncoder);
-		bool loadSettings = encoder == curAdvRecordEncoder;
+	QString encoder = GetComboData(ui->advOutRecEncoder);
+	bool loadSettings = encoder == curAdvRecordEncoder;
 
+	if (!loading) {
 		recordEncoderProps = CreateEncoderPropertyView(
 				QT_TO_UTF8(encoder),
 				loadSettings ? "recordEncoder.json" : nullptr,
@@ -3458,6 +3473,17 @@ void OBSBasicSettings::on_advOutRecEncoder_currentIndexChanged(int idx)
 		connect(recordEncoderProps, SIGNAL(Changed()),
 				this, SLOT(AdvReplayBufferChanged()));
 	}
+
+	uint32_t caps = obs_get_encoder_caps(QT_TO_UTF8(encoder));
+
+	if (caps & OBS_ENCODER_CAP_PASS_TEXTURE) {
+		ui->advOutRecUseRescale->setChecked(false);
+		ui->advOutRecUseRescale->setEnabled(false);
+		ui->advOutRecRescale->setEnabled(false);
+	} else {
+		ui->advOutRecUseRescale->setEnabled(true);
+		ui->advOutRecRescale->setEnabled(true);
+	}
 }
 
 void OBSBasicSettings::on_advOutFFIgnoreCompat_stateChanged(int)