obs-ffmpeg: Block 8-bit HDR for AV1 encoders

Don't want to silently generate lossy video.

plugins/obs-ffmpeg/data/locale/en-US.ini

@@ -92,4 +92,6 @@ NVENC.UnsupportedDevice="NVENC Error: Unsupported device. Check your video card
 NVENC.TooManySessions="NVENC Error: Too many concurrent sessions. Try closing other recording software which might be using NVENC such as NVIDIA Shadowplay or Windows 10 Game DVR."
 NVENC.CheckDrivers="Please check your video drivers are up to date."
 
+AV1.8bitUnsupportedHdr="OBS does not support 8-bit output of Rec. 2100."
+
 ReconnectDelayTime="Reconnect Delay"

plugins/obs-ffmpeg/obs-ffmpeg-av1.c

@@ -170,6 +170,25 @@ static void on_first_packet(void *data, AVPacket *pkt, struct darray *da)
 static void *av1_create_internal(obs_data_t *settings, obs_encoder_t *encoder,
 				 const char *enc_lib, const char *enc_name)
 {
+	video_t *video = obs_encoder_video(encoder);
+	const struct video_output_info *voi = video_output_get_info(video);
+	switch (voi->format) {
+	case VIDEO_FORMAT_I010:
+	case VIDEO_FORMAT_P010:
+		break;
+	default:
+		switch (voi->colorspace) {
+		case VIDEO_CS_2100_PQ:
+		case VIDEO_CS_2100_HLG: {
+			const char *const text =
+				obs_module_text("AV1.8bitUnsupportedHdr");
+			obs_encoder_set_last_error(encoder, text);
+			blog(LOG_ERROR, "[AV1 encoder] %s", text);
+			return NULL;
+		}
+		}
+	}
+
 	struct av1_encoder *enc = bzalloc(sizeof(*enc));
 
 	if (strcmp(enc_lib, "libsvtav1") == 0)