obs-x264: Disallow HDR attempts gracefully

Replace crash with standard recording failure dialog, and log message.

plugins/obs-x264/obs-x264.c

@@ -30,9 +30,8 @@
 
 #include <x264.h>
 
-#define do_log(level, format, ...)                  \
-	blog(level, "[x264 encoder: '%s'] " format, \
-	     obs_encoder_get_name(obsx264->encoder), ##__VA_ARGS__)
+#define do_log(level, format, ...) \
+	blog(level, "[x264 encoder] " format, ##__VA_ARGS__)
 
 #define warn(format, ...) do_log(LOG_WARNING, format, ##__VA_ARGS__)
 #define info(format, ...) do_log(LOG_INFO, format, ##__VA_ARGS__)
@@ -470,27 +469,37 @@ static void update_params(struct obs_x264 *obsx264, obs_data_t *settings,
 
 	static const char *const smpte170m = "smpte170m";
 	static const char *const bt709 = "bt709";
-	static const char *const iec61966_2_1 = "iec61966-2-1";
-	const char *colorprim = NULL;
-	const char *transfer = NULL;
-	const char *colmatrix = NULL;
+	static const char *const bt2020 = "bt2020";
+	static const char *const bt2020nc = "bt2020nc";
+	const char *colorprim = bt709;
+	const char *transfer = bt709;
+	const char *colmatrix = bt709;
 	switch (info.colorspace) {
-	case VIDEO_CS_601:
-		colorprim = smpte170m;
-		transfer = smpte170m;
-		colmatrix = smpte170m;
-		break;
 	case VIDEO_CS_DEFAULT:
 	case VIDEO_CS_709:
 		colorprim = bt709;
 		transfer = bt709;
 		colmatrix = bt709;
 		break;
+	case VIDEO_CS_601:
+		colorprim = smpte170m;
+		transfer = smpte170m;
+		colmatrix = smpte170m;
+		break;
 	case VIDEO_CS_SRGB:
 		colorprim = bt709;
-		transfer = iec61966_2_1;
+		transfer = "iec61966-2-1";
 		colmatrix = bt709;
 		break;
+	case VIDEO_CS_2100_PQ:
+		colorprim = bt2020;
+		transfer = "smpte2084";
+		colmatrix = bt2020nc;
+		break;
+	case VIDEO_CS_2100_HLG:
+		colorprim = bt2020;
+		transfer = "arib-std-b67";
+		colmatrix = bt2020nc;
 	}
 
 	obsx264->params.vui.i_sar_height = 1;
@@ -682,6 +691,15 @@ static void load_headers(struct obs_x264 *obsx264)
 
 static void *obs_x264_create(obs_data_t *settings, obs_encoder_t *encoder)
 {
+	video_t *video = obs_encoder_video(encoder);
+	const struct video_output_info *voi = video_output_get_info(video);
+	switch (voi->colorspace) {
+	case VIDEO_CS_2100_PQ:
+	case VIDEO_CS_2100_HLG:
+		warn("OBS does not support using x264 with Rec. 2100");
+		return NULL;
+	}
+
 	struct obs_x264 *obsx264 = bzalloc(sizeof(struct obs_x264));
 	obsx264->encoder = encoder;