obs-ffmpeg: Add ability to debug ffmpeg-mux subprocess

Adds a cmake variable (DEBUG_FFMPEG_MUX) which enables FFmpeg debug
output in the ffmpeg-mux subprocess, and if on Windows, shows the
console window of the ffmpeg-mux subprocess so the current output can be
seen.

CMakeLists.txt

@@ -8,6 +8,7 @@ endif()
 project(obs-studio)
 
 option(BUILD_CAPTIONS "Build captions" FALSE)
+option(DEBUG_FFMPEG_MUX "Debug FFmpeg muxer subprocess" FALSE)
 
 set_property(GLOBAL PROPERTY USE_FOLDERS ON)
 

libobs/CMakeLists.txt

@@ -11,6 +11,10 @@ if (NOT "${FFMPEG_AVCODEC_LIBRARIES}" STREQUAL "")
 	list(REMOVE_ITEM FFMPEG_LIBRARIES ${FFMPEG_AVCODEC_LIBRARIES})
 endif()
 
+if(DEBUG_FFMPEG_MUX)
+	add_definitions(-DSHOW_SUBPROCESSES)
+endif()
+
 if(UNIX)
 	if (NOT APPLE)
 		find_package(X11 REQUIRED)

libobs/util/pipe-windows.c

@@ -57,11 +57,15 @@ static inline bool create_process(const char *cmd_line, HANDLE stdin_handle,
 	si.hStdOutput = stdout_handle;
 	si.hStdError = stderr_handle;
 
+	DWORD flags = 0;
+#ifndef SHOW_SUBPROCESSES
+	flags = CREATE_NO_WINDOW;
+#endif
+
 	os_utf8_to_wcs_ptr(cmd_line, 0, &cmd_line_w);
 	if (cmd_line_w) {
 		success = !!CreateProcessW(NULL, cmd_line_w, NULL, NULL, true,
-					   CREATE_NO_WINDOW, NULL, NULL, &si,
-					   &pi);
+					   flags, NULL, NULL, &si, &pi);
 
 		if (success) {
 			*process = pi.hProcess;

plugins/obs-ffmpeg/ffmpeg-mux/CMakeLists.txt

@@ -1,5 +1,9 @@
 project(obs-ffmpeg-mux)
 
+if(DEBUG_FFMPEG_MUX)
+	add_definitions(-DDEBUG_FFMPEG)
+endif()
+
 find_package(FFmpeg REQUIRED
 	COMPONENTS avcodec avutil avformat)
 include_directories(${FFMPEG_INCLUDE_DIRS})