obs-outputs: Add support for metadata-based multitrack

plugins/obs-outputs/flv-mux.c

@@ -268,3 +268,254 @@ void flv_packet_mux(struct encoder_packet *packet, int32_t dts_offset,
 	*output = data.bytes.array;
 	*size = data.bytes.num;
 }
+
+/* ------------------------------------------------------------------------- */
+/* stuff for additional media streams                                        */
+
+#define s_amf_conststring(s, str)                   \
+	do {                                        \
+		const size_t len = sizeof(str) - 1; \
+		s_wb16(s, (uint16_t)len);           \
+		serialize(s, str, len);             \
+	} while (false)
+
+#define s_amf_double(s, d)                            \
+	do {                                          \
+		double d_val = d;                     \
+		uint64_t u_val = *(uint64_t *)&d_val; \
+		s_wb64(s, u_val);                     \
+	} while (false)
+
+static void flv_build_additional_meta_data(uint8_t **data, size_t *size)
+{
+	struct array_output_data out;
+	struct serializer s;
+
+	array_output_serializer_init(&s, &out);
+
+	s_w8(&s, AMF_STRING);
+	s_amf_conststring(&s, "@setDataFrame");
+
+	s_w8(&s, AMF_STRING);
+	s_amf_conststring(&s, "onExpectAdditionalMedia");
+
+	s_w8(&s, AMF_OBJECT);
+	{
+		s_amf_conststring(&s, "processingIntents");
+
+		s_w8(&s, AMF_STRICT_ARRAY);
+		s_wb32(&s, 1);
+		{
+			s_w8(&s, AMF_STRING);
+			s_amf_conststring(&s, "ArchiveProgramNarrationAudio");
+		}
+
+		/* ---- */
+
+		s_amf_conststring(&s, "additionalMedia");
+
+		s_w8(&s, AMF_OBJECT);
+		{
+			s_amf_conststring(&s, "stream0");
+
+			s_w8(&s, AMF_OBJECT);
+			{
+				s_amf_conststring(&s, "type");
+
+				s_w8(&s, AMF_NUMBER);
+				s_amf_double(&s, RTMP_PACKET_TYPE_AUDIO);
+
+				/* ---- */
+
+				s_amf_conststring(&s, "mediaLabels");
+
+				s_w8(&s, AMF_OBJECT);
+				{
+					s_amf_conststring(&s, "contentType");
+
+					s_w8(&s, AMF_STRING);
+					s_amf_conststring(&s, "PNAR");
+				}
+				s_wb24(&s, AMF_OBJECT_END);
+			}
+			s_wb24(&s, AMF_OBJECT_END);
+		}
+		s_wb24(&s, AMF_OBJECT_END);
+
+		/* ---- */
+
+		s_amf_conststring(&s, "defaultMedia");
+
+		s_w8(&s, AMF_OBJECT);
+		{
+			s_amf_conststring(&s, "audio");
+
+			s_w8(&s, AMF_OBJECT);
+			{
+				s_amf_conststring(&s, "mediaLabels");
+
+				s_w8(&s, AMF_OBJECT);
+				{
+					s_amf_conststring(&s, "contentType");
+
+					s_w8(&s, AMF_STRING);
+					s_amf_conststring(&s, "PRM");
+				}
+				s_wb24(&s, AMF_OBJECT_END);
+			}
+			s_wb24(&s, AMF_OBJECT_END);
+		}
+		s_wb24(&s, AMF_OBJECT_END);
+	}
+	s_wb24(&s, AMF_OBJECT_END);
+
+	*data = out.bytes.array;
+	*size = out.bytes.num;
+}
+
+bool flv_additional_meta_data(obs_output_t *context, uint8_t **data,
+			      size_t *size)
+{
+	obs_encoder_t *aencoder = obs_output_get_audio_encoder(context, 1);
+	struct array_output_data out;
+	struct serializer s;
+	uint8_t *meta_data = NULL;
+	size_t meta_data_size;
+
+	if (!aencoder)
+		return false;
+
+	flv_build_additional_meta_data(&meta_data, &meta_data_size);
+
+	array_output_serializer_init(&s, &out);
+
+	s_w8(&s, RTMP_PACKET_TYPE_INFO); //18
+
+	s_wb24(&s, (uint32_t)meta_data_size);
+	s_wb32(&s, 0);
+	s_wb24(&s, 0);
+
+	s_write(&s, meta_data, meta_data_size);
+	bfree(meta_data);
+
+	s_wb32(&s, (uint32_t)serializer_get_pos(&s) - 1);
+
+	*data = out.bytes.array;
+	*size = out.bytes.num;
+	return false;
+}
+
+static inline void s_u29(struct serializer *s, uint32_t val)
+{
+	if (val <= 0x7F) {
+		s_w8(s, val);
+	} else if (val <= 0x3FFF) {
+		s_w8(s, 0x80 | (val >> 7));
+		s_w8(s, val & 0x7F);
+	} else if (val <= 0x1FFFFF) {
+		s_w8(s, 0x80 | (val >> 14));
+		s_w8(s, 0x80 | ((val >> 7) & 0x7F));
+		s_w8(s, val & 0x7F);
+	} else {
+		s_w8(s, 0x80 | (val >> 22));
+		s_w8(s, 0x80 | ((val >> 15) & 0x7F));
+		s_w8(s, 0x80 | ((val >> 8) & 0x7F));
+		s_w8(s, val & 0xFF);
+	}
+}
+
+static inline void s_u29b_value(struct serializer *s, uint32_t val)
+{
+	s_u29(s, 1 | ((val & 0xFFFFFFF) << 1));
+}
+
+static void flv_build_additional_audio(uint8_t **data, size_t *size,
+				       struct encoder_packet *packet,
+				       bool is_header, size_t index)
+{
+	struct array_output_data out;
+	struct serializer s;
+
+	array_output_serializer_init(&s, &out);
+
+	s_w8(&s, AMF_STRING);
+	s_amf_conststring(&s, "additionalMedia");
+
+	s_w8(&s, AMF_OBJECT);
+	{
+		s_amf_conststring(&s, "id");
+
+		s_w8(&s, AMF_STRING);
+		s_amf_conststring(&s, "stream0");
+
+		/* ----- */
+
+		s_amf_conststring(&s, "media");
+
+		s_w8(&s, AMF_AVMPLUS);
+		s_w8(&s, AMF3_BYTE_ARRAY);
+		s_u29b_value(&s, (uint32_t)packet->size + 2);
+		s_w8(&s, 0xaf);
+		s_w8(&s, is_header ? 0 : 1);
+		s_write(&s, packet->data, packet->size);
+	}
+	s_wb24(&s, AMF_OBJECT_END);
+
+	*data = out.bytes.array;
+	*size = out.bytes.num;
+}
+
+static void flv_additional_audio(struct serializer *s, int32_t dts_offset,
+				 struct encoder_packet *packet, bool is_header,
+				 size_t index)
+{
+	int32_t time_ms = get_ms_time(packet, packet->dts) - dts_offset;
+	uint8_t *data;
+	size_t size;
+
+	if (!packet->data || !packet->size)
+		return;
+
+	flv_build_additional_audio(&data, &size, packet, is_header, index);
+
+	s_w8(s, RTMP_PACKET_TYPE_INFO); //18
+
+#ifdef DEBUG_TIMESTAMPS
+	blog(LOG_DEBUG, "Audio2: %lu", time_ms);
+
+	if (last_time > time_ms)
+		blog(LOG_DEBUG, "Non-monotonic");
+
+	last_time = time_ms;
+#endif
+
+	s_wb24(s, (uint32_t)size);
+	s_wb24(s, time_ms);
+	s_w8(s, (time_ms >> 24) & 0x7F);
+	s_wb24(s, 0);
+
+	serialize(s, data, size);
+	bfree(data);
+
+	s_wb32(s, (uint32_t)serializer_get_pos(s) - 1);
+}
+
+void flv_additional_packet_mux(struct encoder_packet *packet,
+			       int32_t dts_offset, uint8_t **data, size_t *size,
+			       bool is_header, size_t index)
+{
+	struct array_output_data out;
+	struct serializer s;
+
+	array_output_serializer_init(&s, &out);
+
+	if (packet->type == OBS_ENCODER_VIDEO) {
+		//currently unsupported
+		bcrash("who said you could output an additional video packet?");
+	} else {
+		flv_additional_audio(&s, dts_offset, packet, is_header, index);
+	}
+
+	*data = out.bytes.array;
+	*size = out.bytes.num;
+}

plugins/obs-outputs/flv-mux.h

@@ -30,5 +30,11 @@ extern void write_file_info(FILE *file, int64_t duration_ms, int64_t size);
 
 extern bool flv_meta_data(obs_output_t *context, uint8_t **output, size_t *size,
 			  bool write_header, size_t audio_idx);
+extern bool flv_additional_meta_data(obs_output_t *context, uint8_t **output,
+				     size_t *size);
 extern void flv_packet_mux(struct encoder_packet *packet, int32_t dts_offset,
 			   uint8_t **output, size_t *size, bool is_header);
+extern void flv_additional_packet_mux(struct encoder_packet *packet,
+				      int32_t dts_offset, uint8_t **output,
+				      size_t *size, bool is_header,
+				      size_t index);

plugins/obs-outputs/rtmp-stream.c

@@ -423,13 +423,23 @@ static int send_packet(struct rtmp_stream *stream,
 		}
 	}
 
-	flv_packet_mux(packet, is_header ? 0 : stream->start_dts_offset, &data,
-		       &size, is_header);
+	if (stream->using_metadata_multitrack && idx > 0) {
+		flv_additional_packet_mux(
+			packet, is_header ? 0 : stream->start_dts_offset, &data,
+			&size, is_header, idx);
+	} else {
+		flv_packet_mux(packet, is_header ? 0 : stream->start_dts_offset,
+			       &data, &size, is_header);
+	}
 
 #ifdef TEST_FRAMEDROPS
 	droptest_cap_data_rate(stream, size);
 #endif
 
+	if (stream->using_metadata_multitrack) {
+		idx = 0;
+	}
+
 	ret = RTMP_Write(&stream->rtmp, (char *)data, (int)size, (int)idx);
 	bfree(data);
 
@@ -659,6 +669,23 @@ static void *send_thread(void *data)
 	return NULL;
 }
 
+static bool send_additional_meta_data(struct rtmp_stream *stream, size_t idx,
+				      bool *next)
+{
+	uint8_t *meta_data;
+	size_t meta_data_size;
+	bool success = true;
+
+	*next = flv_additional_meta_data(stream->output, &meta_data,
+					 &meta_data_size);
+
+	success = RTMP_Write(&stream->rtmp, (char *)meta_data,
+			     (int)meta_data_size, 0) >= 0;
+	bfree(meta_data);
+
+	return success;
+}
+
 static bool send_meta_data(struct rtmp_stream *stream, size_t idx, bool *next)
 {
 	uint8_t *meta_data;
@@ -866,7 +893,13 @@ static int init_send(struct rtmp_stream *stream)
 
 	os_atomic_set_bool(&stream->active, true);
 	while (next) {
-		if (!send_meta_data(stream, idx++, &next)) {
+		bool success =
+			(stream->using_metadata_multitrack && idx != 0)
+				? send_additional_meta_data(stream, idx, &next)
+				: send_meta_data(stream, idx, &next);
+		idx++;
+
+		if (!success) {
 			warn("Disconnected while attempting to connect to "
 			     "server.");
 			set_output_error(stream);
@@ -982,16 +1015,18 @@ static int try_connect(struct rtmp_stream *stream)
 
 	RTMP_AddStream(&stream->rtmp, stream->key.array);
 
-	for (size_t idx = 1;; idx++) {
-		obs_encoder_t *encoder =
-			obs_output_get_audio_encoder(stream->output, idx);
-		const char *encoder_name;
+	if (!stream->using_metadata_multitrack) {
+		for (size_t idx = 1;; idx++) {
+			obs_encoder_t *encoder = obs_output_get_audio_encoder(
+				stream->output, idx);
+			const char *encoder_name;
 
-		if (!encoder)
-			break;
+			if (!encoder)
+				break;
 
-		encoder_name = obs_encoder_get_name(encoder);
-		RTMP_AddStream(&stream->rtmp, encoder_name);
+			encoder_name = obs_encoder_get_name(encoder);
+			RTMP_AddStream(&stream->rtmp, encoder_name);
+		}
 	}
 
 	stream->rtmp.m_outChunkSize = 4096;
@@ -1052,6 +1087,8 @@ static bool init_connect(struct rtmp_stream *stream)
 	drop_p = (int64_t)obs_data_get_int(settings, OPT_PFRAME_DROP_THRESHOLD);
 	stream->max_shutdown_time_sec =
 		(int)obs_data_get_int(settings, OPT_MAX_SHUTDOWN_TIME_SEC);
+	stream->using_metadata_multitrack =
+		obs_data_get_bool(settings, OPT_METADATA_MULTITRACK);
 
 	obs_encoder_t *venc = obs_output_get_video_encoder(stream->output);
 	obs_encoder_t *aenc = obs_output_get_audio_encoder(stream->output, 0);
@@ -1451,6 +1488,7 @@ static void rtmp_stream_defaults(obs_data_t *defaults)
 	obs_data_set_default_string(defaults, OPT_BIND_IP, "default");
 	obs_data_set_default_bool(defaults, OPT_NEWSOCKETLOOP_ENABLED, false);
 	obs_data_set_default_bool(defaults, OPT_LOWLATENCY_ENABLED, false);
+	obs_data_set_default_bool(defaults, OPT_METADATA_MULTITRACK, true);
 }
 
 static obs_properties_t *rtmp_stream_properties(void *unused)

plugins/obs-outputs/rtmp-stream.h

@@ -31,6 +31,7 @@
 #define OPT_BIND_IP "bind_ip"
 #define OPT_NEWSOCKETLOOP_ENABLED "new_socket_loop_enabled"
 #define OPT_LOWLATENCY_ENABLED "low_latency_mode_enabled"
+#define OPT_METADATA_MULTITRACK "metadata_multitrack"
 
 //#define TEST_FRAMEDROPS
 //#define TEST_FRAMEDROPS_WITH_BITRATE_SHORTCUTS
@@ -58,6 +59,7 @@ struct rtmp_stream {
 	pthread_mutex_t packets_mutex;
 	struct circlebuf packets;
 	bool sent_headers;
+	bool using_metadata_multitrack;
 
 	bool got_first_video;
 	int64_t start_dts_offset;

plugins/rtmp-services/rtmp-common.c

@@ -14,6 +14,8 @@ struct rtmp_common {
 	char *key;
 
 	char *output;
+
+	bool supports_additional_audio_track;
 };
 
 static const char *rtmp_common_getname(void *unused)
@@ -25,6 +27,7 @@ static const char *rtmp_common_getname(void *unused)
 static json_t *open_services_file(void);
 static inline json_t *find_service(json_t *root, const char *name,
 				   const char **p_new_name);
+static inline bool get_bool_val(json_t *service, const char *key);
 static inline const char *get_string_val(json_t *service, const char *key);
 
 extern void twitch_ingests_refresh(int seconds);
@@ -74,6 +77,7 @@ static void rtmp_common_update(void *data, obs_data_t *settings)
 	service->service = bstrdup(obs_data_get_string(settings, "service"));
 	service->server = bstrdup(obs_data_get_string(settings, "server"));
 	service->key = bstrdup(obs_data_get_string(settings, "key"));
+	service->supports_additional_audio_track = false;
 	service->output = NULL;
 
 	json_t *root = open_services_file();
@@ -94,6 +98,8 @@ static void rtmp_common_update(void *data, obs_data_t *settings)
 					service->output = bstrdup(out);
 			}
 
+			service->supports_additional_audio_track = get_bool_val(
+				serv, "supports_additional_audio_track");
 			ensure_valid_url(service, serv, settings);
 		}
 	}
@@ -622,6 +628,12 @@ static const char *rtmp_common_key(void *data)
 	return service->key;
 }
 
+static bool supports_multitrack(void *data)
+{
+	struct rtmp_common *service = data;
+	return service->supports_additional_audio_track;
+}
+
 struct obs_service_info rtmp_common_service = {
 	.id = "rtmp_common",
 	.get_name = rtmp_common_getname,
@@ -633,4 +645,5 @@ struct obs_service_info rtmp_common_service = {
 	.get_key = rtmp_common_key,
 	.apply_encoder_settings = rtmp_common_apply_settings,
 	.get_output_type = rtmp_common_get_output_type,
+	.supports_multitrack = supports_multitrack,
 };