Merge pull request #10039 from derrod/roi-libobs

libobs/plugins: Add region of interest (ROI) encoder feature

docs/sphinx/reference-encoders.rst

@@ -163,6 +163,7 @@ Encoder Definition Structure (obs_encoder_info)
    values:
 
    - **OBS_ENCODER_CAP_DEPRECATED** - Encoder is deprecated
+   - **OBS_ENCODER_CAP_ROI** - Encoder supports region of interest feature
 
 
 Encoder Packet Structure (encoder_packet)
@@ -265,6 +266,27 @@ Raw Frame Data Structure (encoder_frame)
    Presentation timestamp.
 
 
+Encoder Region of Interest Structure (obs_encoder_roi)
+-----------------------------------------
+
+.. struct:: obs_encoder_roi
+
+   Encoder region of interest structure.
+
+.. member:: uint32_t top
+            uint32_t bottom
+            uint32_t left
+            uint32_t right
+   
+   The rectangle edges of the region are specified as number of pixels from the input video's top and left edges (i.e. row/column 0).
+
+.. member:: float priority
+
+   Priority is specified as a float value between *-1.0f* and *1*.
+   These are converted to encoder-specific values by the encoder.
+   Values above 0 tell the encoder to increase quality for that region, values below tell it to worsen it.
+   Not all encoders support negative values and they may be ignored.
+
 General Encoder Functions
 -------------------------
 
@@ -517,6 +539,42 @@ General Encoder Functions
 
 ---------------------
 
+.. function:: bool obs_encoder_add_roi(obs_encoder_t *encoder, const struct obs_encoder_roi *roi)
+
+    Adds a new region of interest to the encoder if ROI feature is supported.
+
+   :return: *true* if adding succeeded, *false* otherwise.
+
+---------------------
+
+.. function:: bool obs_encoder_has_roi(obs_encoder_t *encoder)
+
+   :return: *true* if encoder has ROI regions set, *false* otherwise.
+
+---------------------
+
+.. function:: void obs_encoder_clear_roi(obs_encoder_t *encoder)
+
+    Clear region of interest list, if any.
+
+---------------------
+
+.. function:: void obs_encoder_enum_roi(obs_encoder_t *encoder, void (*enum_proc)(void *, struct obs_encoder_roi *), void *param)
+
+    Enumerate currently configured ROIs by invoking callback for each entry, in reverse order of addition (i.e. most recent to oldest).
+    
+    **Note:** If the encoder has scaling enabled the struct passed to the callback will be scaled accordingly.
+
+---------------------
+
+.. function:: uint32_t obs_encoder_get_roi_increment(const obs_encoder_t *encoder)
+
+   Encoders shall refresh their ROI configuration if the increment value changes.
+    
+   :return: Increment/revision of ROI list
+
+---------------------
+
 
 Functions used by encoders
 --------------------------

libobs/obs-encoder.c

@@ -52,6 +52,7 @@ static bool init_encoder(struct obs_encoder *encoder, const char *name,
 	pthread_mutex_init_value(&encoder->callbacks_mutex);
 	pthread_mutex_init_value(&encoder->outputs_mutex);
 	pthread_mutex_init_value(&encoder->pause.mutex);
+	pthread_mutex_init_value(&encoder->roi_mutex);
 
 	if (!obs_context_data_init(&encoder->context, OBS_OBJ_TYPE_ENCODER,
 				   settings, name, NULL, hotkey_data, false))
@@ -64,6 +65,8 @@ static bool init_encoder(struct obs_encoder *encoder, const char *name,
 		return false;
 	if (pthread_mutex_init(&encoder->pause.mutex, NULL) != 0)
 		return false;
+	if (pthread_mutex_init(&encoder->roi_mutex, NULL) != 0)
+		return false;
 
 	if (encoder->orig_info.get_defaults) {
 		encoder->orig_info.get_defaults(encoder->context.settings);
@@ -377,10 +380,12 @@ static void obs_encoder_actually_destroy(obs_encoder_t *encoder)
 		if (encoder->context.data)
 			encoder->info.destroy(encoder->context.data);
 		da_free(encoder->callbacks);
+		da_free(encoder->roi);
 		pthread_mutex_destroy(&encoder->init_mutex);
 		pthread_mutex_destroy(&encoder->callbacks_mutex);
 		pthread_mutex_destroy(&encoder->outputs_mutex);
 		pthread_mutex_destroy(&encoder->pause.mutex);
+		pthread_mutex_destroy(&encoder->roi_mutex);
 		obs_context_data_free(&encoder->context);
 		if (encoder->owns_info_id)
 			bfree((void *)encoder->info.id);
@@ -1874,3 +1879,90 @@ uint64_t obs_encoder_get_pause_offset(const obs_encoder_t *encoder)
 {
 	return encoder ? encoder->pause.ts_offset : 0;
 }
+
+bool obs_encoder_has_roi(const obs_encoder_t *encoder)
+{
+	return encoder->roi.num > 0;
+}
+
+bool obs_encoder_add_roi(obs_encoder_t *encoder,
+			 const struct obs_encoder_roi *roi)
+{
+	if (!roi)
+		return false;
+	if (!(encoder->info.caps & OBS_ENCODER_CAP_ROI))
+		return false;
+	/* Area smaller than the smallest possible block (16x16) */
+	if (roi->bottom - roi->top < 16 || roi->right - roi->left < 16)
+		return false;
+	/* Other invalid ROIs */
+	if (roi->priority < -1.0f || roi->priority > 1.0f)
+		return false;
+
+	pthread_mutex_lock(&encoder->roi_mutex);
+	da_push_back(encoder->roi, roi);
+	encoder->roi_increment++;
+	pthread_mutex_unlock(&encoder->roi_mutex);
+
+	return true;
+}
+
+void obs_encoder_clear_roi(obs_encoder_t *encoder)
+{
+	if (!encoder->roi.num)
+		return;
+	pthread_mutex_lock(&encoder->roi_mutex);
+	da_clear(encoder->roi);
+	encoder->roi_increment++;
+	pthread_mutex_unlock(&encoder->roi_mutex);
+}
+
+void obs_encoder_enum_roi(obs_encoder_t *encoder,
+			  void (*enum_proc)(void *, struct obs_encoder_roi *),
+			  void *param)
+{
+	float scale_x = 0;
+	float scale_y = 0;
+
+	/* Scale ROI passed to callback to output size */
+	if (encoder->scaled_height && encoder->scaled_width) {
+		const uint32_t width = video_output_get_width(encoder->media);
+		const uint32_t height = video_output_get_height(encoder->media);
+
+		if (!width || !height)
+			return;
+
+		scale_x = (float)encoder->scaled_width / (float)width;
+		scale_y = (float)encoder->scaled_height / (float)height;
+	}
+
+	pthread_mutex_lock(&encoder->roi_mutex);
+
+	size_t idx = encoder->roi.num;
+	while (idx) {
+		struct obs_encoder_roi *roi = &encoder->roi.array[--idx];
+
+		if (scale_x > 0 && scale_y > 0) {
+			struct obs_encoder_roi scaled_roi = {
+				.top = (uint32_t)((float)roi->top * scale_y),
+				.bottom = (uint32_t)((float)roi->bottom *
+						     scale_y),
+				.left = (uint32_t)((float)roi->left * scale_x),
+				.right =
+					(uint32_t)((float)roi->right * scale_x),
+				.priority = roi->priority,
+			};
+
+			enum_proc(param, &scaled_roi);
+		} else {
+			enum_proc(param, roi);
+		}
+	}
+
+	pthread_mutex_unlock(&encoder->roi_mutex);
+}
+
+uint32_t obs_encoder_get_roi_increment(const obs_encoder_t *encoder)
+{
+	return encoder->roi_increment;
+}

libobs/obs-encoder.h

@@ -33,6 +33,7 @@ extern "C" {
 #define OBS_ENCODER_CAP_PASS_TEXTURE (1 << 1)
 #define OBS_ENCODER_CAP_DYN_BITRATE (1 << 2)
 #define OBS_ENCODER_CAP_INTERNAL (1 << 3)
+#define OBS_ENCODER_CAP_ROI (1 << 4)
 
 /** Specifies the encoder type */
 enum obs_encoder_type {
@@ -102,6 +103,23 @@ struct encoder_frame {
 	int64_t pts;
 };
 
+/** Encoder region of interest */
+struct obs_encoder_roi {
+	/* The rectangle edges of the region are specified as number of pixels
+	 * from the input video's top and left edges (i.e. row/column 0). */
+	uint32_t top;
+	uint32_t bottom;
+	uint32_t left;
+	uint32_t right;
+
+	/* Priority is specified as a float value between -1 and 1.
+	 * These are converted to encoder-specific values by the encoder.
+	 * Values above 0 tell the encoder to increase quality for that region,
+	 * values below tell it to worsen it.
+	 * Not all encoders support negative values and they may be ignored. */
+	float priority;
+};
+
 /**
  * Encoder interface
  *

libobs/obs-internal.h

@@ -1263,6 +1263,11 @@ struct obs_encoder {
 	uint32_t frame_rate_divisor_counter; // only used for GPU encoders
 	video_t *fps_override;
 
+	/* Regions of interest to prioritize during encoding */
+	pthread_mutex_t roi_mutex;
+	DARRAY(struct obs_encoder_roi) roi;
+	uint32_t roi_increment;
+
 	int64_t cur_pts;
 
 	struct circlebuf audio_input_buffer[MAX_AV_PLANES];

libobs/obs.h

@@ -2457,6 +2457,27 @@ EXPORT void obs_encoder_set_gpu_scale_type(obs_encoder_t *encoder,
 EXPORT bool obs_encoder_set_frame_rate_divisor(obs_encoder_t *encoder,
 					       uint32_t divisor);
 
+/**
+ * Adds region of interest (ROI) for an encoder. This allows prioritizing
+ * quality of regions of the frame.
+ * If regions overlap, regions added earlier take precedence.
+ *
+ * Returns false if the encoder does not support ROI or region is invalid.
+ */
+EXPORT bool obs_encoder_add_roi(obs_encoder_t *encoder,
+				const struct obs_encoder_roi *roi);
+/** For video encoders, returns true if any ROIs were set */
+EXPORT bool obs_encoder_has_roi(const obs_encoder_t *encoder);
+/** Clear all regions */
+EXPORT void obs_encoder_clear_roi(obs_encoder_t *encoder);
+/** Enumerate regions with callback (reverse order of addition) */
+EXPORT void obs_encoder_enum_roi(obs_encoder_t *encoder,
+				 void (*enum_proc)(void *,
+						   struct obs_encoder_roi *),
+				 void *param);
+/** Get ROI increment, encoders must rebuild their ROI map if it has changed */
+EXPORT uint32_t obs_encoder_get_roi_increment(const obs_encoder_t *encoder);
+
 /** For video encoders, returns true if pre-encode scaling is enabled */
 EXPORT bool obs_encoder_scaling_enabled(const obs_encoder_t *encoder);
 

plugins/obs-ffmpeg/obs-nvenc.c

@@ -126,6 +126,10 @@ struct nvenc_data {
 
 	uint8_t *sei;
 	size_t sei_size;
+
+	int8_t *roi_map;
+	size_t roi_map_size;
+	uint32_t roi_increment;
 };
 
 /* ------------------------------------------------------------------------- */
@@ -683,6 +687,7 @@ static bool init_encoder_base(struct nvenc_data *enc, obs_data_t *settings,
 	config->rcParams.maxBitRate = vbr ? max_bitrate * 1000 : bitrate * 1000;
 	config->rcParams.vbvBufferSize = bitrate * 1000;
 	config->rcParams.multiPass = nv_multipass;
+	config->rcParams.qpMapMode = NV_ENC_QP_MAP_DELTA;
 
 	/* -------------------------- */
 	/* initialize                 */
@@ -1253,6 +1258,7 @@ static void nvenc_destroy(void *data)
 	da_free(enc->bitstreams);
 	da_free(enc->input_textures);
 	da_free(enc->packet_data);
+	bfree(enc->roi_map);
 	bfree(enc);
 }
 
@@ -1381,6 +1387,96 @@ static bool get_encoded_packet(struct nvenc_data *enc, bool finalize)
 	return true;
 }
 
+struct roi_params {
+	uint32_t mb_width;
+	uint32_t mb_height;
+	uint32_t mb_size;
+	bool av1;
+	int8_t *map;
+};
+
+static void roi_cb(void *param, struct obs_encoder_roi *roi)
+{
+	const struct roi_params *rp = param;
+
+	int8_t qp_val;
+	/* AV1 has a larger QP range than HEVC/H.264 */
+	if (rp->av1) {
+		qp_val = (int8_t)(-128.0f * roi->priority);
+	} else {
+		qp_val = (int8_t)(-51.0f * roi->priority);
+	}
+
+	const uint32_t roi_left = roi->left / rp->mb_size;
+	const uint32_t roi_top = roi->top / rp->mb_size;
+	const uint32_t roi_right = (roi->right - 1) / rp->mb_size;
+	const uint32_t roi_bottom = (roi->bottom - 1) / rp->mb_size;
+
+	for (uint32_t mb_y = 0; mb_y < rp->mb_height; mb_y++) {
+		if (mb_y < roi_top || mb_y > roi_bottom)
+			continue;
+
+		for (uint32_t mb_x = 0; mb_x < rp->mb_width; mb_x++) {
+			if (mb_x < roi_left || mb_x > roi_right)
+				continue;
+
+			rp->map[mb_y * rp->mb_width + mb_x] = qp_val;
+		}
+	}
+}
+
+static void add_roi(struct nvenc_data *enc, NV_ENC_PIC_PARAMS *params)
+{
+	const uint32_t increment = obs_encoder_get_roi_increment(enc->encoder);
+
+	if (enc->roi_map && enc->roi_increment == increment) {
+		params->qpDeltaMap = enc->roi_map;
+		params->qpDeltaMapSize = (uint32_t)enc->roi_map_size;
+		return;
+	}
+
+	uint32_t mb_size;
+	switch (enc->codec) {
+	case CODEC_H264:
+		/* H.264 is always 16x16 */
+		mb_size = 16;
+		break;
+	case CODEC_HEVC:
+		/* HEVC can be 16x16, 32x32, or 64x64, but NVENC is always 32x32 */
+		mb_size = 32;
+		break;
+	case CODEC_AV1:
+		/* AV1 can be 64x64 or 128x128, but NVENC is always 64x64 */
+		mb_size = 64;
+		break;
+	}
+
+	const uint32_t mb_width = (enc->cx + mb_size - 1) / mb_size;
+	const uint32_t mb_height = (enc->cy + mb_size - 1) / mb_size;
+	const size_t map_size = mb_width * mb_height * sizeof(int8_t);
+
+	if (map_size != enc->roi_map_size) {
+		enc->roi_map = brealloc(enc->roi_map, map_size);
+		enc->roi_map_size = map_size;
+	}
+
+	memset(enc->roi_map, 0, enc->roi_map_size);
+
+	struct roi_params par = {
+		.mb_width = mb_width,
+		.mb_height = mb_height,
+		.mb_size = mb_size,
+		.av1 = enc->codec == CODEC_AV1,
+		.map = enc->roi_map,
+	};
+
+	obs_encoder_enum_roi(enc->encoder, roi_cb, &par);
+
+	enc->roi_increment = increment;
+	params->qpDeltaMap = enc->roi_map;
+	params->qpDeltaMapSize = (uint32_t)map_size;
+}
+
 static bool nvenc_encode_tex(void *data, uint32_t handle, int64_t pts,
 			     uint64_t lock_key, uint64_t *next_key,
 			     struct encoder_packet *packet,
@@ -1453,6 +1549,10 @@ static bool nvenc_encode_tex(void *data, uint32_t handle, int64_t pts,
 	params.inputPitch = enc->cx;
 	params.outputBitstream = bs->ptr;
 
+	/* Add ROI map if enabled */
+	if (obs_encoder_has_roi(enc->encoder))
+		add_roi(enc, &params);
+
 	err = nv.nvEncEncodePicture(enc->session, &params);
 	if (err != NV_ENC_SUCCESS && err != NV_ENC_ERR_NEED_MORE_INPUT) {
 		nv_failed(enc->encoder, err, __FUNCTION__,
@@ -1537,7 +1637,8 @@ struct obs_encoder_info h264_nvenc_info = {
 	.id = "jim_nvenc",
 	.codec = "h264",
 	.type = OBS_ENCODER_VIDEO,
-	.caps = OBS_ENCODER_CAP_PASS_TEXTURE | OBS_ENCODER_CAP_DYN_BITRATE,
+	.caps = OBS_ENCODER_CAP_PASS_TEXTURE | OBS_ENCODER_CAP_DYN_BITRATE |
+		OBS_ENCODER_CAP_ROI,
 	.get_name = h264_nvenc_get_name,
 	.create = h264_nvenc_create,
 	.destroy = nvenc_destroy,
@@ -1554,7 +1655,8 @@ struct obs_encoder_info hevc_nvenc_info = {
 	.id = "jim_hevc_nvenc",
 	.codec = "hevc",
 	.type = OBS_ENCODER_VIDEO,
-	.caps = OBS_ENCODER_CAP_PASS_TEXTURE | OBS_ENCODER_CAP_DYN_BITRATE,
+	.caps = OBS_ENCODER_CAP_PASS_TEXTURE | OBS_ENCODER_CAP_DYN_BITRATE |
+		OBS_ENCODER_CAP_ROI,
 	.get_name = hevc_nvenc_get_name,
 	.create = hevc_nvenc_create,
 	.destroy = nvenc_destroy,
@@ -1571,7 +1673,8 @@ struct obs_encoder_info av1_nvenc_info = {
 	.id = "jim_av1_nvenc",
 	.codec = "av1",
 	.type = OBS_ENCODER_VIDEO,
-	.caps = OBS_ENCODER_CAP_PASS_TEXTURE | OBS_ENCODER_CAP_DYN_BITRATE,
+	.caps = OBS_ENCODER_CAP_PASS_TEXTURE | OBS_ENCODER_CAP_DYN_BITRATE |
+		OBS_ENCODER_CAP_ROI,
 	.get_name = av1_nvenc_get_name,
 	.create = av1_nvenc_create,
 	.destroy = nvenc_destroy,

plugins/obs-ffmpeg/texture-amf.cpp

@@ -95,6 +95,7 @@ struct amf_base {
 	AMFBufferPtr packet_data;
 	AMFRate amf_frame_rate;
 	AMFBufferPtr header;
+	AMFSurfacePtr roi_map;
 
 	std::deque<AMFDataPtr> queued_packets;
 
@@ -110,11 +111,13 @@ struct amf_base {
 	uint32_t cx;
 	uint32_t cy;
 	uint32_t linesize = 0;
+	uint32_t roi_increment = 0;
 	int fps_num;
 	int fps_den;
 	bool full_range;
 	bool bframes_supported = false;
 	bool first_update = true;
+	bool roi_supported = false;
 
 	inline amf_base(bool fallback) : fallback(fallback) {}
 	virtual ~amf_base() = default;
@@ -580,6 +583,100 @@ static void convert_to_encoder_packet(amf_base *enc, AMFDataPtr &data,
 #define SEC_TO_NSEC 1000000000ULL
 #endif
 
+struct roi_params {
+	uint32_t mb_width;
+	uint32_t mb_height;
+	amf_int32 pitch;
+	bool h264;
+	amf_uint32 *buf;
+};
+
+static void roi_cb(void *param, obs_encoder_roi *roi)
+{
+	const roi_params *rp = static_cast<roi_params *>(param);
+
+	/* AMF does not support negative priority */
+	if (roi->priority < 0)
+		return;
+
+	const uint32_t mb_size = rp->h264 ? 16 : 64;
+	const uint32_t roi_left = roi->left / mb_size;
+	const uint32_t roi_top = roi->top / mb_size;
+	const uint32_t roi_right = (roi->right - 1) / mb_size;
+	const uint32_t roi_bottom = (roi->bottom - 1) / mb_size;
+	/* Importance value range is 0..10 */
+	const amf_uint32 priority = (amf_uint32)(10.0f * roi->priority);
+
+	for (uint32_t mb_y = 0; mb_y < rp->mb_height; mb_y++) {
+		if (mb_y < roi_top || mb_y > roi_bottom)
+			continue;
+
+		for (uint32_t mb_x = 0; mb_x < rp->mb_width; mb_x++) {
+			if (mb_x < roi_left || mb_x > roi_right)
+				continue;
+
+			rp->buf[mb_y * rp->pitch / sizeof(amf_uint32) + mb_x] =
+				priority;
+		}
+	}
+}
+
+static void create_roi(amf_base *enc, AMFSurface *amf_surf)
+{
+	uint32_t mb_size = 16; /* H.264 is always 16x16 */
+	if (enc->codec == amf_codec_type::HEVC ||
+	    enc->codec == amf_codec_type::AV1)
+		mb_size = 64; /* AMF HEVC & AV1 use 64x64 blocks */
+
+	const uint32_t mb_width = (enc->cx + mb_size - 1) / mb_size;
+	const uint32_t mb_height = (enc->cy + mb_size - 1) / mb_size;
+
+	if (!enc->roi_map) {
+		AMFContext1Ptr context1(enc->amf_context);
+		AMF_RESULT res = context1->AllocSurfaceEx(
+			AMF_MEMORY_HOST, AMF_SURFACE_GRAY32, mb_width,
+			mb_height,
+			AMF_SURFACE_USAGE_DEFAULT | AMF_SURFACE_USAGE_LINEAR,
+			AMF_MEMORY_CPU_DEFAULT, &enc->roi_map);
+
+		if (res != AMF_OK) {
+			warn("Failed allocating surface for ROI map!");
+			/* Clear ROI to prevent failure the next frame */
+			obs_encoder_clear_roi(enc->encoder);
+			return;
+		}
+	}
+
+	/* This is just following the SimpleROI example. */
+	amf_uint32 *pBuf =
+		(amf_uint32 *)enc->roi_map->GetPlaneAt(0)->GetNative();
+	amf_int32 pitch = enc->roi_map->GetPlaneAt(0)->GetHPitch();
+	memset(pBuf, 0, pitch * mb_height);
+
+	roi_params par{mb_width, mb_height, pitch,
+		       enc->codec == amf_codec_type::AVC, pBuf};
+	obs_encoder_enum_roi(enc->encoder, roi_cb, &par);
+
+	enc->roi_increment = obs_encoder_get_roi_increment(enc->encoder);
+}
+
+static void add_roi(amf_base *enc, AMFSurface *amf_surf)
+{
+	const uint32_t increment = obs_encoder_get_roi_increment(enc->encoder);
+
+	if (increment != enc->roi_increment || !enc->roi_increment)
+		create_roi(enc, amf_surf);
+
+	if (enc->codec == amf_codec_type::AVC)
+		amf_surf->SetProperty(AMF_VIDEO_ENCODER_ROI_DATA, enc->roi_map);
+	else if (enc->codec == amf_codec_type::HEVC)
+		amf_surf->SetProperty(AMF_VIDEO_ENCODER_HEVC_ROI_DATA,
+				      enc->roi_map);
+	else if (enc->codec == amf_codec_type::AV1)
+		amf_surf->SetProperty(AMF_VIDEO_ENCODER_AV1_ROI_DATA,
+				      enc->roi_map);
+}
+
 static void amf_encode_base(amf_base *enc, AMFSurface *amf_surf,
 			    encoder_packet *packet, bool *received_packet)
 {
@@ -591,6 +688,11 @@ static void amf_encode_base(amf_base *enc, AMFSurface *amf_surf,
 
 	bool waiting = true;
 	while (waiting) {
+		/* ----------------------------------- */
+		/* add ROI data (if any)               */
+		if (enc->roi_supported && obs_encoder_has_roi(enc->encoder))
+			add_roi(enc, amf_surf);
+
 		/* ----------------------------------- */
 		/* submit frame                        */
 
@@ -1377,6 +1479,8 @@ static void amf_avc_create_internal(amf_base *enc, obs_data_t *settings)
 				  &enc->max_throughput);
 		caps->GetProperty(AMF_VIDEO_ENCODER_CAP_REQUESTED_THROUGHPUT,
 				  &enc->requested_throughput);
+		caps->GetProperty(AMF_VIDEO_ENCODER_CAP_ROI,
+				  &enc->roi_supported);
 	}
 
 	const char *preset = obs_data_get_string(settings, "preset");
@@ -1507,13 +1611,15 @@ static void register_avc()
 	amf_encoder_info.get_properties = amf_avc_properties;
 	amf_encoder_info.get_extra_data = amf_extra_data;
 	amf_encoder_info.caps = OBS_ENCODER_CAP_PASS_TEXTURE |
-				OBS_ENCODER_CAP_DYN_BITRATE;
+				OBS_ENCODER_CAP_DYN_BITRATE |
+				OBS_ENCODER_CAP_ROI;
 
 	obs_register_encoder(&amf_encoder_info);
 
 	amf_encoder_info.id = "h264_fallback_amf";
 	amf_encoder_info.caps = OBS_ENCODER_CAP_INTERNAL |
-				OBS_ENCODER_CAP_DYN_BITRATE;
+				OBS_ENCODER_CAP_DYN_BITRATE |
+				OBS_ENCODER_CAP_ROI;
 	amf_encoder_info.encode_texture = nullptr;
 	amf_encoder_info.create = amf_avc_create_fallback;
 	amf_encoder_info.encode = amf_encode_fallback;
@@ -1713,6 +1819,8 @@ static void amf_hevc_create_internal(amf_base *enc, obs_data_t *settings)
 		caps->GetProperty(
 			AMF_VIDEO_ENCODER_HEVC_CAP_REQUESTED_THROUGHPUT,
 			&enc->requested_throughput);
+		caps->GetProperty(AMF_VIDEO_ENCODER_HEVC_CAP_ROI,
+				  &enc->roi_supported);
 	}
 
 	const bool is10bit = enc->amf_format == AMF_SURFACE_P010;
@@ -1861,13 +1969,15 @@ static void register_hevc()
 	amf_encoder_info.get_properties = amf_hevc_properties;
 	amf_encoder_info.get_extra_data = amf_extra_data;
 	amf_encoder_info.caps = OBS_ENCODER_CAP_PASS_TEXTURE |
-				OBS_ENCODER_CAP_DYN_BITRATE;
+				OBS_ENCODER_CAP_DYN_BITRATE |
+				OBS_ENCODER_CAP_ROI;
 
 	obs_register_encoder(&amf_encoder_info);
 
 	amf_encoder_info.id = "h265_fallback_amf";
 	amf_encoder_info.caps = OBS_ENCODER_CAP_INTERNAL |
-				OBS_ENCODER_CAP_DYN_BITRATE;
+				OBS_ENCODER_CAP_DYN_BITRATE |
+				OBS_ENCODER_CAP_ROI;
 	amf_encoder_info.encode_texture = nullptr;
 	amf_encoder_info.create = amf_hevc_create_fallback;
 	amf_encoder_info.encode = amf_encode_fallback;
@@ -2055,6 +2165,8 @@ static void amf_av1_create_internal(amf_base *enc, obs_data_t *settings)
 		caps->GetProperty(
 			AMF_VIDEO_ENCODER_AV1_CAP_REQUESTED_THROUGHPUT,
 			&enc->requested_throughput);
+		/* For some reason there's no specific CAP for AV1, but should always be supported */
+		enc->roi_supported = true;
 	}
 
 	const bool is10bit = enc->amf_format == AMF_SURFACE_P010;
@@ -2187,13 +2299,15 @@ static void register_av1()
 	amf_encoder_info.get_properties = amf_av1_properties;
 	amf_encoder_info.get_extra_data = amf_extra_data;
 	amf_encoder_info.caps = OBS_ENCODER_CAP_PASS_TEXTURE |
-				OBS_ENCODER_CAP_DYN_BITRATE;
+				OBS_ENCODER_CAP_DYN_BITRATE |
+				OBS_ENCODER_CAP_ROI;
 
 	obs_register_encoder(&amf_encoder_info);
 
 	amf_encoder_info.id = "av1_fallback_amf";
 	amf_encoder_info.caps = OBS_ENCODER_CAP_INTERNAL |
-				OBS_ENCODER_CAP_DYN_BITRATE;
+				OBS_ENCODER_CAP_DYN_BITRATE |
+				OBS_ENCODER_CAP_ROI;
 	amf_encoder_info.encode_texture = nullptr;
 	amf_encoder_info.create = amf_av1_create_fallback;
 	amf_encoder_info.encode = amf_encode_fallback;

plugins/obs-qsv11/QSV_Encoder.cpp

@@ -206,6 +206,22 @@ int qsv_encoder_headers(qsv_t *pContext, uint8_t **pSPS, uint8_t **pPPS,
 	return 0;
 }
 
+void qsv_encoder_add_roi(qsv_t *pContext, const obs_encoder_roi *roi)
+{
+	QSV_Encoder_Internal *pEncoder = (QSV_Encoder_Internal *)pContext;
+
+	/* QP value is range 0..51 */
+	// ToDo figure out if this is different for AV1
+	mfxI16 delta = (mfxI16)(-51.0f * roi->priority);
+	pEncoder->AddROI(roi->left, roi->top, roi->right, roi->bottom, delta);
+}
+
+void qsv_encoder_clear_roi(qsv_t *pContext)
+{
+	QSV_Encoder_Internal *pEncoder = (QSV_Encoder_Internal *)pContext;
+	pEncoder->ClearROI();
+}
+
 int qsv_encoder_encode(qsv_t *pContext, uint64_t ts, uint8_t *pDataY,
 		       uint8_t *pDataUV, uint32_t strideY, uint32_t strideUV,
 		       mfxBitstream **pBS)

plugins/obs-qsv11/QSV_Encoder.h

@@ -160,6 +160,8 @@ int qsv_encoder_reconfig(qsv_t *, qsv_param_t *);
 void qsv_encoder_version(unsigned short *major, unsigned short *minor);
 qsv_t *qsv_encoder_open(qsv_param_t *, enum qsv_codec codec);
 bool qsv_encoder_is_dgpu(qsv_t *);
+void qsv_encoder_add_roi(qsv_t *, const struct obs_encoder_roi *roi);
+void qsv_encoder_clear_roi(qsv_t *pContext);
 int qsv_encoder_encode(qsv_t *, uint64_t, uint8_t *, uint8_t *, uint32_t,
 		       uint32_t, mfxBitstream **pBS);
 int qsv_encoder_encode_tex(qsv_t *, uint64_t, uint32_t, uint64_t, uint64_t *,

plugins/obs-qsv11/QSV_Encoder_Internal.cpp

@@ -442,6 +442,9 @@ mfxStatus QSV_Encoder_Internal::InitParams(qsv_param_t *pParams,
 		}
 	}
 
+	memset(&m_ctrl, 0, sizeof(m_ctrl));
+	memset(&m_roi, 0, sizeof(m_roi));
+
 	return sts;
 }
 
@@ -776,7 +779,7 @@ mfxStatus QSV_Encoder_Internal::Encode(uint64_t ts, uint8_t *pDataY,
 
 	for (;;) {
 		// Encode a frame asynchronously (returns immediately)
-		sts = m_pmfxENC->EncodeFrameAsync(NULL, pSurface,
+		sts = m_pmfxENC->EncodeFrameAsync(&m_ctrl, pSurface,
 						  &m_pTaskPool[nTaskIdx].mfxBS,
 						  &m_pTaskPool[nTaskIdx].syncp);
 
@@ -841,7 +844,7 @@ mfxStatus QSV_Encoder_Internal::Encode_tex(uint64_t ts, uint32_t tex_handle,
 
 	for (;;) {
 		// Encode a frame asynchronously (returns immediately)
-		sts = m_pmfxENC->EncodeFrameAsync(NULL, pSurface,
+		sts = m_pmfxENC->EncodeFrameAsync(&m_ctrl, pSurface,
 						  &m_pTaskPool[nTaskIdx].mfxBS,
 						  &m_pTaskPool[nTaskIdx].syncp);
 
@@ -939,3 +942,39 @@ mfxStatus QSV_Encoder_Internal::Reset(qsv_param_t *pParams,
 
 	return sts;
 }
+
+void QSV_Encoder_Internal::AddROI(mfxU32 left, mfxU32 top, mfxU32 right,
+				  mfxU32 bottom, mfxI16 delta)
+{
+	if (m_roi.NumROI == 256) {
+		warn("Maximum number of ROIs hit, ignoring additional ROI!");
+		return;
+	}
+
+	m_roi.Header.BufferId = MFX_EXTBUFF_ENCODER_ROI;
+	m_roi.Header.BufferSz = sizeof(mfxExtEncoderROI);
+	m_roi.ROIMode = MFX_ROI_MODE_QP_DELTA;
+	/* The SDK will automatically align the values to block sizes so we
+	 * don't have to do any maths here. */
+	m_roi.ROI[m_roi.NumROI].Left = left;
+	m_roi.ROI[m_roi.NumROI].Top = top;
+	m_roi.ROI[m_roi.NumROI].Right = right;
+	m_roi.ROI[m_roi.NumROI].Bottom = bottom;
+	m_roi.ROI[m_roi.NumROI].DeltaQP = delta;
+	m_roi.NumROI++;
+
+	/* Right now ROI is the only thing we add so this is fine */
+	if (m_extbuf.empty())
+		m_extbuf.push_back((mfxExtBuffer *)&m_roi);
+
+	m_ctrl.ExtParam = m_extbuf.data();
+	m_ctrl.NumExtParam = (mfxU16)m_extbuf.size();
+}
+
+void QSV_Encoder_Internal::ClearROI()
+{
+	m_roi.NumROI = 0;
+	m_ctrl.ExtParam = nullptr;
+	m_ctrl.NumExtParam = 0;
+	m_extbuf.clear();
+}

plugins/obs-qsv11/QSV_Encoder_Internal.h

@@ -81,6 +81,9 @@ public:
 	mfxStatus Reset(qsv_param_t *pParams, enum qsv_codec codec);
 	mfxStatus ReconfigureEncoder();
 	bool UpdateParams(qsv_param_t *pParams);
+	void AddROI(mfxU32 left, mfxU32 top, mfxU32 right, mfxU32 bottom,
+		    mfxI16 delta);
+	void ClearROI();
 
 	bool IsDGPU() const { return m_isDGPU; }
 
@@ -135,4 +138,8 @@ private:
 	static mfxU16 g_numEncodersOpen;
 	static mfxHDL
 		g_DX_Handle; // we only want one handle for all instances to use;
+
+	mfxEncodeCtrl m_ctrl;
+	mfxExtEncoderROI m_roi;
+	std::vector<mfxExtBuffer *> m_extbuf;
 };

plugins/obs-qsv11/obs-qsv11.c

@@ -99,6 +99,8 @@ struct obs_qsv {
 	size_t sei_size;
 
 	os_performance_token_t *performance_token;
+
+	uint32_t roi_increment;
 };
 
 /* ------------------------------------------------------------------------- */
@@ -1377,6 +1379,37 @@ static void parse_packet_hevc(struct obs_qsv *obsqsv,
 	g_bFirst = false;
 }
 
+static void roi_cb(void *param, struct obs_encoder_roi *roi)
+{
+	struct darray *da = param;
+	darray_push_back(sizeof(struct obs_encoder_roi), da, roi);
+}
+
+static void obs_qsv_setup_rois(struct obs_qsv *obsqsv)
+{
+	const uint32_t increment =
+		obs_encoder_get_roi_increment(obsqsv->encoder);
+	if (obsqsv->roi_increment == increment)
+		return;
+
+	qsv_encoder_clear_roi(obsqsv->context);
+	/* Because we pass-through the ROIs more or less directly we need to
+	 * pass them in reverse order, so make a temporary copy and then use
+	 * that instead. */
+	DARRAY(struct obs_encoder_roi) rois;
+	da_init(rois);
+
+	obs_encoder_enum_roi(obsqsv->encoder, roi_cb, &rois);
+
+	size_t idx = rois.num;
+	while (idx)
+		qsv_encoder_add_roi(obsqsv->context, &rois.array[--idx]);
+
+	da_free(rois);
+
+	obsqsv->roi_increment = increment;
+}
+
 static bool obs_qsv_encode(void *data, struct encoder_frame *frame,
 			   struct encoder_packet *packet, bool *received_packet)
 {
@@ -1396,6 +1429,9 @@ static bool obs_qsv_encode(void *data, struct encoder_frame *frame,
 
 	mfxU64 qsvPTS = ts_obs_to_mfx(frame->pts, voi);
 
+	if (obs_encoder_has_roi(obsqsv->encoder))
+		obs_qsv_setup_rois(obsqsv);
+
 	// FIXME: remove null check from the top of this function
 	// if we actually do expect null frames to complete output.
 	if (frame)
@@ -1452,6 +1488,9 @@ static bool obs_qsv_encode_tex(void *data, uint32_t handle, int64_t pts,
 
 	mfxU64 qsvPTS = ts_obs_to_mfx(pts, voi);
 
+	if (obs_encoder_has_roi(obsqsv->encoder))
+		obs_qsv_setup_rois(obsqsv);
+
 	ret = qsv_encoder_encode_tex(obsqsv->context, qsvPTS, handle, lock_key,
 				     next_key, &pBS);
 
@@ -1516,7 +1555,8 @@ struct obs_encoder_info obs_qsv_encoder_tex_v2 = {
 	.get_name = obs_qsv_getname,
 	.create = obs_qsv_create_tex_h264_v2,
 	.destroy = obs_qsv_destroy,
-	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_PASS_TEXTURE,
+	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_PASS_TEXTURE |
+		OBS_ENCODER_CAP_ROI,
 	.encode_texture = obs_qsv_encode_tex,
 	.update = obs_qsv_update,
 	.get_properties = obs_qsv_props_h264_v2,
@@ -1540,7 +1580,8 @@ struct obs_encoder_info obs_qsv_encoder_v2 = {
 	.get_extra_data = obs_qsv_extra_data,
 	.get_sei_data = obs_qsv_sei,
 	.get_video_info = obs_qsv_video_info,
-	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_INTERNAL,
+	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_INTERNAL |
+		OBS_ENCODER_CAP_ROI,
 };
 
 struct obs_encoder_info obs_qsv_av1_encoder_tex = {
@@ -1550,7 +1591,8 @@ struct obs_encoder_info obs_qsv_av1_encoder_tex = {
 	.get_name = obs_qsv_getname_av1,
 	.create = obs_qsv_create_tex_av1,
 	.destroy = obs_qsv_destroy,
-	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_PASS_TEXTURE,
+	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_PASS_TEXTURE |
+		OBS_ENCODER_CAP_ROI,
 	.encode_texture = obs_qsv_encode_tex,
 	.update = obs_qsv_update,
 	.get_properties = obs_qsv_props_av1,
@@ -1572,7 +1614,8 @@ struct obs_encoder_info obs_qsv_av1_encoder = {
 	.get_defaults = obs_qsv_defaults_av1,
 	.get_extra_data = obs_qsv_extra_data,
 	.get_video_info = obs_qsv_video_plus_hdr_info,
-	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_INTERNAL,
+	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_INTERNAL |
+		OBS_ENCODER_CAP_ROI,
 };
 
 struct obs_encoder_info obs_qsv_hevc_encoder_tex = {
@@ -1582,7 +1625,8 @@ struct obs_encoder_info obs_qsv_hevc_encoder_tex = {
 	.get_name = obs_qsv_getname_hevc,
 	.create = obs_qsv_create_tex_hevc,
 	.destroy = obs_qsv_destroy,
-	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_PASS_TEXTURE,
+	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_PASS_TEXTURE |
+		OBS_ENCODER_CAP_ROI,
 	.encode_texture = obs_qsv_encode_tex,
 	.update = obs_qsv_update,
 	.get_properties = obs_qsv_props_hevc,
@@ -1604,5 +1648,6 @@ struct obs_encoder_info obs_qsv_hevc_encoder = {
 	.get_defaults = obs_qsv_defaults_hevc,
 	.get_extra_data = obs_qsv_extra_data,
 	.get_video_info = obs_qsv_video_plus_hdr_info,
-	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_INTERNAL,
+	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_INTERNAL |
+		OBS_ENCODER_CAP_ROI,
 };

plugins/obs-x264/obs-x264.c

@@ -61,6 +61,9 @@ struct obs_x264 {
 	size_t sei_size;
 
 	os_performance_token_t *performance_token;
+
+	uint32_t roi_increment;
+	float *quant_offsets;
 };
 
 /* ------------------------------------------------------------------------- */
@@ -79,6 +82,7 @@ static void clear_data(struct obs_x264 *obsx264)
 		x264_encoder_close(obsx264->context);
 		bfree(obsx264->sei);
 		bfree(obsx264->extra_data);
+		bfree(obsx264->quant_offsets);
 
 		obsx264->context = NULL;
 		obsx264->sei = NULL;
@@ -775,6 +779,66 @@ static inline void init_pic_data(struct obs_x264 *obsx264, x264_picture_t *pic,
 	}
 }
 
+/* H.264 always uses 16x16 macroblocks */
+static const uint32_t MB_SIZE = 16;
+
+struct roi_params {
+	uint32_t mb_width;
+	uint32_t mb_height;
+	float *map;
+};
+
+static void roi_cb(void *param, struct obs_encoder_roi *roi)
+{
+	const struct roi_params *rp = param;
+
+	const uint32_t roi_left = roi->left / MB_SIZE;
+	const uint32_t roi_top = roi->top / MB_SIZE;
+	const uint32_t roi_right = (roi->right - 1) / MB_SIZE;
+	const uint32_t roi_bottom = (roi->bottom - 1) / MB_SIZE;
+	/* QP range is 0..51 */
+	const float qp_offset = -51.0f * roi->priority;
+
+	for (uint32_t mb_y = 0; mb_y < rp->mb_height; mb_y++) {
+		if (mb_y < roi_top || mb_y > roi_bottom)
+			continue;
+
+		for (uint32_t mb_x = 0; mb_x < rp->mb_width; mb_x++) {
+			if (mb_x < roi_left || mb_x > roi_right)
+				continue;
+
+			rp->map[mb_y * rp->mb_width + mb_x] = qp_offset;
+		}
+	}
+}
+
+static void add_roi(struct obs_x264 *obsx264, x264_picture_t *pic)
+{
+	const uint32_t increment =
+		obs_encoder_get_roi_increment(obsx264->encoder);
+
+	if (obsx264->quant_offsets && obsx264->roi_increment == increment) {
+		pic->prop.quant_offsets = obsx264->quant_offsets;
+		return;
+	}
+
+	const uint32_t width = obs_encoder_get_width(obsx264->encoder);
+	const uint32_t height = obs_encoder_get_height(obsx264->encoder);
+	const uint32_t mb_width = (width + MB_SIZE - 1) / MB_SIZE;
+	const uint32_t mb_height = (height + MB_SIZE - 1) / MB_SIZE;
+	const size_t map_size = sizeof(float) * mb_width * mb_height;
+
+	float *map = bzalloc(map_size);
+
+	struct roi_params par = {mb_width, mb_height, map};
+
+	obs_encoder_enum_roi(obsx264->encoder, roi_cb, &par);
+
+	pic->prop.quant_offsets = map;
+	obsx264->quant_offsets = map;
+	obsx264->roi_increment = increment;
+}
+
 static bool obs_x264_encode(void *data, struct encoder_frame *frame,
 			    struct encoder_packet *packet,
 			    bool *received_packet)
@@ -791,6 +855,9 @@ static bool obs_x264_encode(void *data, struct encoder_frame *frame,
 	if (frame)
 		init_pic_data(obsx264, &pic, frame);
 
+	if (obs_encoder_has_roi(obsx264->encoder))
+		add_roi(obsx264, &pic);
+
 	ret = x264_encoder_encode(obsx264->context, &nals, &nal_count,
 				  (frame ? &pic : NULL), &pic_out);
 	if (ret < 0) {
@@ -863,5 +930,5 @@ struct obs_encoder_info obs_x264_encoder = {
 	.get_extra_data = obs_x264_extra_data,
 	.get_sei_data = obs_x264_sei,
 	.get_video_info = obs_x264_video_info,
-	.caps = OBS_ENCODER_CAP_DYN_BITRATE,
+	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_ROI,
 };