Quellcode durchsuchen

obs-outputs: Add support for and use mbedTLS for SSL

This diff adds mbedTLS support to the obs-outputs plugin.  PolarSSL and
mbedTLS have grown so different between 2015-or-so when libRTMP was
written, and now it's no longer feasible to just use the USE_POLARSSL
flag.

This commit adds a WITH_RTMPS tri-state CMake variable (auto/on/off),
set to "Auto" by default.  "Auto" will use RTMPS if mbedTLS is found,
otherwise will disable RTMPS.  "On" will make it require mbedTLS,
otherwise fails configuration, and "Off" disables RTMPS support
altogether.

Closes obsproject/obs-studio#1360
Michael Goulet vor 7 Jahren
Ursprung
Commit
e67e2e12e6

+ 143 - 0
cmake/Modules/FindMbedTLS.cmake

@@ -0,0 +1,143 @@
+# Once done these will be defined:
+#
+#  LIBMBEDTLS_FOUND
+#  LIBMBEDTLS_INCLUDE_DIRS
+#  LIBMBEDTLS_LIBRARIES
+#
+# For use in OBS:
+#
+#  MBEDTLS_INCLUDE_DIR
+
+find_package(PkgConfig QUIET)
+if (PKG_CONFIG_FOUND)
+	pkg_check_modules(_MBEDTLS QUIET mbedtls)
+endif()
+
+if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+	set(_lib_suffix 64)
+else()
+	set(_lib_suffix 32)
+endif()
+
+# If we're on MacOS or Linux, please try to statically-link mbedtls.
+if(STATIC_MBEDTLS AND (APPLE OR UNIX))
+	set(_MBEDTLS_LIBRARIES libmbedtls.a)
+	set(_MBEDCRYPTO_LIBRARIES libmbedcrypto.a)
+	set(_MBEDX509_LIBRARIES libmbedx509.a)
+endif()
+
+find_path(MBEDTLS_INCLUDE_DIR
+	NAMES mbedtls/ssl.h
+	HINTS
+		ENV mbedtlsPath${_lib_suffix}
+		ENV mbedtlsPath
+		ENV DepsPath${_lib_suffix}
+		ENV DepsPath
+		${mbedtlsPath${_lib_suffix}}
+		${mbedtlsPath}
+		${DepsPath${_lib_suffix}}
+		${DepsPath}
+		${_MBEDTLS_INCLUDE_DIRS}
+	PATHS
+		/usr/include /usr/local/include /opt/local/include /sw/include
+	PATH_SUFFIXES
+		include)
+
+find_library(MBEDTLS_LIB
+	NAMES ${_MBEDTLS_LIBRARIES} mbedtls libmbedtls
+	HINTS
+		ENV mbedtlsPath${_lib_suffix}
+		ENV mbedtlsPath
+		ENV DepsPath${_lib_suffix}
+		ENV DepsPath
+		${mbedtlsPath${_lib_suffix}}
+		${mbedtlsPath}
+		${DepsPath${_lib_suffix}}
+		${DepsPath}
+		${_MBEDTLS_LIBRARY_DIRS}
+	PATHS
+		/usr/lib /usr/local/lib /opt/local/lib /sw/lib
+	PATH_SUFFIXES
+		lib${_lib_suffix} lib
+		libs${_lib_suffix} libs
+		bin${_lib_suffix} bin
+		../lib${_lib_suffix} ../lib
+		../libs${_lib_suffix} ../libs
+		../bin${_lib_suffix} ../bin)
+
+find_library(MBEDCRYPTO_LIB
+	NAMES ${_MBEDCRYPTO_LIBRARIES} mbedcrypto libmbedcrypto
+	HINTS
+		ENV mbedcryptoPath${_lib_suffix}
+		ENV mbedcryptoPath
+		ENV DepsPath${_lib_suffix}
+		ENV DepsPath
+		${mbedcryptoPath${_lib_suffix}}
+		${mbedcryptoPath}
+		${DepsPath${_lib_suffix}}
+		${DepsPath}
+		${_MBEDCRYPTO_LIBRARY_DIRS}
+	PATHS
+		/usr/lib /usr/local/lib /opt/local/lib /sw/lib
+	PATH_SUFFIXES
+		lib${_lib_suffix} lib
+		libs${_lib_suffix} libs
+		bin${_lib_suffix} bin
+		../lib${_lib_suffix} ../lib
+		../libs${_lib_suffix} ../libs
+		../bin${_lib_suffix} ../bin)
+
+find_library(MBEDX509_LIB
+	NAMES ${_MBEDX509_LIBRARIES} mbedx509 libmbedx509
+	HINTS
+		ENV mbedx509Path${_lib_suffix}
+		ENV mbedx509Path
+		ENV DepsPath${_lib_suffix}
+		ENV DepsPath
+		${mbedx509Path${_lib_suffix}}
+		${mbedx509Path}
+		${DepsPath${_lib_suffix}}
+		${DepsPath}
+		${_MBEDX509_LIBRARY_DIRS}
+	PATHS
+		/usr/lib /usr/local/lib /opt/local/lib /sw/lib
+	PATH_SUFFIXES
+		lib${_lib_suffix} lib
+		libs${_lib_suffix} libs
+		bin${_lib_suffix} bin
+		../lib${_lib_suffix} ../lib
+		../libs${_lib_suffix} ../libs
+		../bin${_lib_suffix} ../bin)
+
+# Sometimes mbedtls is split between three libs, and sometimes it isn't.
+# If it isn't, let's check if the symbols we need are all in MBEDTLS_LIB.
+if(MBEDTLS_LIB AND NOT MBEDCRYPTO_LIB AND NOT MBEDX509_LIB)
+	set(CMAKE_REQUIRED_LIBRARIES ${MBEDTLS_LIB})
+	set(CMAKE_REQUIRED_INCLUDES ${MBEDTLS_INCLUDE_DIR})
+	check_symbol_exists(mbedtls_x509_crt_init "mbedtls/x509_crt.h" MBEDTLS_INCLUDES_X509)
+	check_symbol_exists(mbedtls_sha256_init "mbedtls/sha256.h" MBEDTLS_INCLUDES_CRYPTO)
+	unset(CMAKE_REQUIRED_INCLUDES)
+	unset(CMAKE_REQUIRED_LIBRARIES)
+endif()
+
+# If we find all three libraries, then go ahead.
+if(MBEDTLS_LIB AND MBEDCRYPTO_LIB AND MBEDX509_LIB)
+	set(LIBMBEDTLS_INCLUDE_DIRS ${MBEDTLS_INCLUDE_DIR})
+	set(LIBMBEDTLS_LIBRARIES ${MBEDTLS_LIB} ${MBEDCRYPTO_LIB} ${MBEDX509_LIB})
+	set(MBEDTLS_INCLUDE_DIRS ${LIBMBEDTLS_INCLUDE_DIRS})
+	set(MBEDTLS_LIBRARIES ${LIBMBEDTLS_LIBRARIES})
+
+# Otherwise, if we find MBEDTLS_LIB, and it has both CRYPTO and x509
+# within the single lib (i.e. a windows build environment), then also
+# feel free to go ahead.
+elseif(MBEDTLS_LIB AND MBEDTLS_INCLUDES_CRYPTO AND MBEDTLS_INCLUDES_X509)
+	set(LIBMBEDTLS_INCLUDE_DIRS ${MBEDTLS_INCLUDE_DIR})
+	set(LIBMBEDTLS_LIBRARIES ${MBEDTLS_LIB})
+	set(MBEDTLS_INCLUDE_DIRS ${LIBMBEDTLS_INCLUDE_DIRS})
+	set(MBEDTLS_LIBRARIES ${LIBMBEDTLS_LIBRARIES})
+endif()
+
+# Now we've accounted for the 3-vs-1 library case:
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Libmbedtls DEFAULT_MSG MBEDTLS_LIBRARIES MBEDTLS_INCLUDE_DIRS)
+mark_as_advanced(MBEDTLS_INCLUDE_DIR MBEDTLS_LIBRARIES MBEDTLS_INCLUDE_DIRS)

+ 0 - 77
cmake/Modules/FindSSL.cmake

@@ -1,77 +0,0 @@
-# Once done these will be defined:
-#
-#  SSL_FOUND
-#  SSL_INCLUDE_DIRS
-#  SSL_LIBRARIES
-#
-# For use in OBS: 
-#
-#  SSL_INCLUDE_DIR
-
-find_package(PkgConfig QUIET)
-if (PKG_CONFIG_FOUND)
-	pkg_check_modules(_CRYPTO QUIET libcrypto)
-	pkg_check_modules(_SSL QUIET libssl)
-endif()
-
-if(CMAKE_SIZEOF_VOID_P EQUAL 8)
-	set(_lib_suffix 64)
-else()
-	set(_lib_suffix 32)
-endif()
-
-set(_SSL_BASE_HINTS
-		ENV sslPath${_lib_suffix}
-		ENV sslPath
-		ENV DepsPath${_lib_suffix}
-		ENV DepsPath
-		${sslPath${_lib_suffix}}
-		${sslPath}
-		${DepsPath${_lib_suffix}}
-		${DepsPath})
-
-set(_SSL_LIB_SUFFIXES
-		lib${_lib_suffix} lib
-		libs${_lib_suffix} libs
-		bin${_lib_suffix} bin
-		../lib${_lib_suffix} ../lib
-		../libs${_lib_suffix} ../libs
-		../bin${_lib_suffix} ../bin)
-
-find_path(SSL_INCLUDE_DIR
-	NAMES openssl/ssl.h
-	HINTS
-		${_SSL_BASE_HINTS}
-		${_CRYPTO_INCLUDE_DIRS}
-		${_SSL_INCLUDE_DIRS}
-	PATHS
-		/usr/include /usr/local/include /opt/local/include /sw/include
-	PATH_SUFFIXES
-		include)
-
-find_library(_SSL_LIB
-	NAMES ${_SSL_LIBRARIES} ssleay32 ssl
-	HINTS
-		${_SSL_BASE_HINTS}
-		${_SSL_LIBRARY_DIRS}
-	PATHS
-		/usr/lib /usr/local/lib /opt/local/lib /sw/lib
-	PATH_SUFFIXES ${_SSL_LIB_SUFFIXES})
-
-find_library(_CRYPTO_LIB
-	NAMES ${_CRYPTO_LIBRARIES} libeay32 crypto
-	HINTS
-		${_SSL_BASE_HINTS}
-		${_CRYPTO_LIBRARY_DIRS}
-	PATHS
-		/usr/lib /usr/local/lib /opt/local/lib /sw/lib
-	PATH_SUFFIXES ${_SSL_LIB_SUFFIXES})
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(ssl DEFAULT_MSG _SSL_LIB _CRYPTO_LIB SSL_INCLUDE_DIR)
-mark_as_advanced(SSL_INCLUDE_DIR _SSL_LIB _CRYPTO_LIB)
-
-if(SSL_FOUND)
-	set(SSL_INCLUDE_DIRS ${SSL_INCLUDE_DIR})
-	set(SSL_LIBRARIES ${_SSL_LIB} ${_CRYPTO_LIB})
-endif()

+ 32 - 11
plugins/obs-outputs/CMakeLists.txt

@@ -1,20 +1,25 @@
 project(obs-outputs)
 project(obs-outputs)
 
 
-option(USE_SSL "Enable rtmps support with OpenSSL" OFF)
+set(WITH_RTMPS AUTO CACHE STRING "Enable RTMPS support with mbedTLS")
+set_property(CACHE WITH_RTMPS PROPERTY STRINGS AUTO ON OFF)
 
 
-if (USE_SSL)
-	find_package(SSL QUIET)
+option(STATIC_MBEDTLS "Statically link mbedTLS into binary" OFF)
+
+if (WITH_RTMPS OR (WITH_RTMPS STREQUAL "AUTO"))
+	find_package(MbedTLS QUIET)
 	find_package(ZLIB QUIET)
 	find_package(ZLIB QUIET)
 endif()
 endif()
 
 
-if (SSL_FOUND AND ZLIB_FOUND)
-	add_definitions(-DCRYPTO -DUSE_OPENSSL)
-	include_directories(${SSL_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
+if (LIBMBEDTLS_FOUND AND ZLIB_FOUND)
+	add_definitions(-DCRYPTO -DUSE_MBEDTLS)
+	include_directories(${LIBMBEDTLS_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
 else()
 else()
-	if (USE_SSL)
-		message(WARNING "SSL enabled by user, but OpenSSL was not found")
+	if(WITH_RTMPS STREQUAL "AUTO")
+		message(WARNING "mbedTLS was not found, RTMPS will be auto-disabled")
+	elseif (WITH_RTMPS)
+		message(FATAL_ERROR "RTMPS enabled by user, but mbedTLS was not found")
 	endif()
 	endif()
-	unset(SSL_LIBRARIES)
+	unset(LIBMBEDTLS_LIBRARIES)
 	unset(ZLIB_LIBRARIES)
 	unset(ZLIB_LIBRARIES)
 	add_definitions(-DNO_CRYPTO)
 	add_definitions(-DNO_CRYPTO)
 endif()
 endif()
@@ -84,6 +89,12 @@ if(WIN32)
 		ws2_32
 		ws2_32
 		winmm
 		winmm
 		Iphlpapi)
 		Iphlpapi)
+
+	if (WITH_RTMPS OR (WITH_RTMPS STREQUAL "AUTO"))
+		SET(obs-outputs_PLATFORM_DEPS
+			${obs-outputs_PLATFORM_DEPS}
+			crypt32)
+	endif()
 endif()
 endif()
 
 
 if(MSVC)
 if(MSVC)
@@ -92,6 +103,16 @@ if(MSVC)
 		w32-pthreads)
 		w32-pthreads)
 endif()
 endif()
 
 
+if(APPLE AND (WITH_RTMPS OR (WITH_RTMPS STREQUAL "AUTO")))
+	find_library(FOUNDATION_FRAMEWORK Foundation)
+	find_library(SECURITY_FRAMEWORK Security)
+
+	set(obs-outputs_PLATFORM_DEPS
+		${obs-outputs_PLATFORM_DEPS}
+		${FOUNDATION_FRAMEWORK}
+		${SECURITY_FRAMEWORK})
+endif()
+
 set(obs-outputs_librtmp_HEADERS
 set(obs-outputs_librtmp_HEADERS
 	librtmp/amf.h
 	librtmp/amf.h
 	librtmp/bytes.h
 	librtmp/bytes.h
@@ -133,7 +154,7 @@ set(obs-outputs_SOURCES
 	flv-output.c
 	flv-output.c
 	flv-mux.c
 	flv-mux.c
 	net-if.c)
 	net-if.c)
-	
+
 add_library(obs-outputs MODULE
 add_library(obs-outputs MODULE
 	${ftl_SOURCES}
 	${ftl_SOURCES}
 	${ftl_HEADERS}
 	${ftl_HEADERS}
@@ -143,7 +164,7 @@ add_library(obs-outputs MODULE
 	${obs-outputs_librtmp_HEADERS})
 	${obs-outputs_librtmp_HEADERS})
 target_link_libraries(obs-outputs
 target_link_libraries(obs-outputs
 	libobs
 	libobs
-	${SSL_LIBRARIES}
+	${LIBMBEDTLS_LIBRARIES}
 	${ZLIB_LIBRARIES}
 	${ZLIB_LIBRARIES}
 	${ftl_IMPORTS}
 	${ftl_IMPORTS}
 	${obs-outputs_PLATFORM_DEPS})
 	${obs-outputs_PLATFORM_DEPS})

+ 1 - 0
plugins/obs-outputs/data/locale/en-US.ini

@@ -11,3 +11,4 @@ ConnectionReset="The connection was reset by the peer. This usually indicates in
 HostNotFound="Hostname not found. Make sure you entered a valid streaming server and your internet connection / DNS are working correctly."
 HostNotFound="Hostname not found. Make sure you entered a valid streaming server and your internet connection / DNS are working correctly."
 NoData="Hostname found, but no data of the requested type. This can occur if you have bound to an IPv6 address and your streaming service only has IPv4 addresses (see  Settings / Advanced)."
 NoData="Hostname found, but no data of the requested type. This can occur if you have bound to an IPv6 address and your streaming service only has IPv4 addresses (see  Settings / Advanced)."
 AddressNotAvailable="Address not available. You may have tried to bind to an invalid IP address (see  Settings / Advanced)."
 AddressNotAvailable="Address not available. You may have tried to bind to an invalid IP address (see  Settings / Advanced)."
+SSLCertVerifyFailed="The RTMP server sent an invalid SSL certificate."

+ 54 - 2
plugins/obs-outputs/librtmp/dh.h

@@ -21,7 +21,59 @@
  *  http://www.gnu.org/copyleft/lgpl.html
  *  http://www.gnu.org/copyleft/lgpl.html
  */
  */
 
 
-#ifdef USE_POLARSSL
+#if defined(USE_MBEDTLS)
+#include <mbedtls/dhm.h>
+#include <mbedtls/bignum.h>
+typedef  mbedtls_mpi* MP_t;
+#define MP_new(m)	m = malloc(sizeof(mbedtls_mpi)); mbedtls_mpi_init(m)
+#define MP_set_w(mpi, w)	mbedtls_mpi_lset(mpi, w)
+#define MP_cmp(u, v)	mbedtls_mpi_cmp_mpi(u, v)
+#define MP_set(u, v)	mbedtls_mpi_copy(u, v)
+#define MP_sub_w(mpi, w)	mbedtls_mpi_sub_int(mpi, mpi, w)
+#define MP_cmp_1(mpi)	mbedtls_mpi_cmp_int(mpi, 1)
+#define MP_modexp(r, y, q, p)	mbedtls_mpi_exp_mod(r, y, q, p, NULL)
+#define MP_free(mpi)	mbedtls_mpi_free(mpi); free(mpi)
+#define MP_gethex(u, hex, res)	MP_new(u); res = mbedtls_mpi_read_string(u, 16, hex) == 0
+#define MP_bytes(u)	mbedtls_mpi_size(u)
+#define MP_setbin(u,buf,len)	mbedtls_mpi_write_binary(u,buf,len)
+#define MP_getbin(u,buf,len)	MP_new(u); mbedtls_mpi_read_binary(u,buf,len)
+
+typedef struct MDH
+{
+    MP_t p;
+    MP_t g;
+    MP_t pub_key;
+    MP_t priv_key;
+    long length;
+    mbedtls_dhm_context ctx;
+} MDH;
+
+#define MDH_new()	calloc(1,sizeof(MDH))
+#define MDH_free(vp)	{MDH *_dh = vp; mbedtls_dhm_free(&_dh->ctx); MP_free(_dh->p); MP_free(_dh->g); MP_free(_dh->pub_key); MP_free(_dh->priv_key); free(_dh);}
+
+static int MDH_generate_key(MDH *dh)
+{
+    unsigned char out[2];
+    MP_set(&dh->ctx.P, dh->p);
+    MP_set(&dh->ctx.G, dh->g);
+    dh->ctx.len = 128;
+    mbedtls_dhm_make_public(&dh->ctx, 1024, out, 1, mbedtls_ctr_drbg_random, &RTMP_TLS_ctx->ctr_drbg);
+    MP_new(dh->pub_key);
+    MP_new(dh->priv_key);
+    MP_set(dh->pub_key, &dh->ctx.GX);
+    MP_set(dh->priv_key, &dh->ctx.X);
+    return 1;
+}
+
+static int MDH_compute_key(uint8_t *secret, size_t len, MP_t pub, MDH *dh)
+{
+    MP_set(&dh->ctx.GY, pub);
+    size_t olen;
+    mbedtls_dhm_calc_secret(&dh->ctx, secret, len, &olen, NULL, NULL);
+    return 0;
+}
+
+#elif defined(USE_POLARSSL)
 #include <polarssl/dhm.h>
 #include <polarssl/dhm.h>
 typedef mpi * MP_t;
 typedef mpi * MP_t;
 #define MP_new(m)	m = malloc(sizeof(mpi)); mpi_init(m)
 #define MP_new(m)	m = malloc(sizeof(mpi)); mpi_init(m)
@@ -271,7 +323,7 @@ DHGetPublicKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen)
     if (!dh || !dh->pub_key)
     if (!dh || !dh->pub_key)
         return 0;
         return 0;
 
 
-    len = MP_bytes(dh->pub_key);
+    len = (int)MP_bytes(dh->pub_key);
     if (len <= 0 || len > (int) nPubkeyLen)
     if (len <= 0 || len > (int) nPubkeyLen)
         return 0;
         return 0;
 
 

+ 33 - 1
plugins/obs-outputs/librtmp/handshake.h

@@ -24,7 +24,28 @@
 
 
 /* This file is #included in rtmp.c, it is not meant to be compiled alone */
 /* This file is #included in rtmp.c, it is not meant to be compiled alone */
 
 
-#ifdef USE_POLARSSL
+#if defined(USE_MBEDTLS)
+#include <mbedtls/md.h>
+#include <mbedtls/arc4.h>
+#ifndef SHA256_DIGEST_LENGTH
+#define SHA256_DIGEST_LENGTH	32
+#endif
+typedef mbedtls_md_context_t *HMAC_CTX;
+#define HMAC_setup(ctx, key, len)	ctx = malloc(sizeof(mbedtls_md_context_t)); mbedtls_md_init(ctx); \
+  mbedtls_md_setup(ctx, mbedtls_md_info_from_type(MBEDTLS_MD_SHA256), 1); \
+  mbedtls_md_hmac_starts(ctx, (const unsigned char *)key, len)
+#define HMAC_crunch(ctx, buf, len) mbedtls_md_hmac_update(ctx, buf, len)
+#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; mbedtls_md_hmac_finish(ctx, dig)
+#define HMAC_close(ctx) mbedtls_md_free(ctx); free(ctx); ctx = NULL
+
+typedef mbedtls_arc4_context*	RC4_handle;
+#define RC4_alloc(h)	*h = malloc(sizeof(mbedtls_arc4_context)); mbedtls_arc4_init(*h)
+#define RC4_setkey(h,l,k)	mbedtls_arc4_setup(h,k,l)
+#define RC4_encrypt(h,l,d)	mbedtls_arc4_crypt(h,l,(unsigned char *)d,(unsigned char *)d)
+#define RC4_encrypt2(h,l,s,d)	mbedtls_arc4_crypt(h,l,(unsigned char *)s,(unsigned char *)d)
+#define RC4_free(h)	mbedtls_arc4_free(h); free(h); h = NULL
+
+#elif defined(USE_POLARSSL)
 #include <polarssl/sha2.h>
 #include <polarssl/sha2.h>
 #include <polarssl/arc4.h>
 #include <polarssl/arc4.h>
 #ifndef SHA256_DIGEST_LENGTH
 #ifndef SHA256_DIGEST_LENGTH
@@ -148,6 +169,8 @@ typedef unsigned int (getoff)(uint8_t *buf, unsigned int len);
 static unsigned int
 static unsigned int
 GetDHOffset2(uint8_t *handshake, unsigned int len)
 GetDHOffset2(uint8_t *handshake, unsigned int len)
 {
 {
+    (void) len;
+
     unsigned int offset = 0;
     unsigned int offset = 0;
     uint8_t *ptr = handshake + 768;
     uint8_t *ptr = handshake + 768;
     unsigned int res;
     unsigned int res;
@@ -177,6 +200,8 @@ GetDHOffset2(uint8_t *handshake, unsigned int len)
 static unsigned int
 static unsigned int
 GetDigestOffset2(uint8_t *handshake, unsigned int len)
 GetDigestOffset2(uint8_t *handshake, unsigned int len)
 {
 {
+    (void) len;
+
     unsigned int offset = 0;
     unsigned int offset = 0;
     uint8_t *ptr = handshake + 772;
     uint8_t *ptr = handshake + 772;
     unsigned int res;
     unsigned int res;
@@ -206,6 +231,8 @@ GetDigestOffset2(uint8_t *handshake, unsigned int len)
 static unsigned int
 static unsigned int
 GetDHOffset1(uint8_t *handshake, unsigned int len)
 GetDHOffset1(uint8_t *handshake, unsigned int len)
 {
 {
+    (void) len;
+
     unsigned int offset = 0;
     unsigned int offset = 0;
     uint8_t *ptr = handshake + 1532;
     uint8_t *ptr = handshake + 1532;
     unsigned int res;
     unsigned int res;
@@ -235,6 +262,8 @@ GetDHOffset1(uint8_t *handshake, unsigned int len)
 static unsigned int
 static unsigned int
 GetDigestOffset1(uint8_t *handshake, unsigned int len)
 GetDigestOffset1(uint8_t *handshake, unsigned int len)
 {
 {
+    (void) len;
+
     unsigned int offset = 0;
     unsigned int offset = 0;
     uint8_t *ptr = handshake + 8;
     uint8_t *ptr = handshake + 8;
     unsigned int res;
     unsigned int res;
@@ -1128,6 +1157,7 @@ HandShake(RTMP * r, int FP9HandShake)
                      __FUNCTION__);
                      __FUNCTION__);
         }
         }
     }
     }
+    // TODO(mgoulet): Should this have a HMAC_finish here?
 
 
     RTMP_Log(RTMP_LOGDEBUG, "%s: Handshaking finished....", __FUNCTION__);
     RTMP_Log(RTMP_LOGDEBUG, "%s: Handshaking finished....", __FUNCTION__);
     return TRUE;
     return TRUE;
@@ -1482,6 +1512,8 @@ SHandShake(RTMP * r)
         }
         }
     }
     }
 
 
+    // TODO(mgoulet): Should this have an Rc4_free?
+
     RTMP_Log(RTMP_LOGDEBUG, "%s: Handshaking finished....", __FUNCTION__);
     RTMP_Log(RTMP_LOGDEBUG, "%s: Handshaking finished....", __FUNCTION__);
     return TRUE;
     return TRUE;
 }
 }

+ 34 - 10
plugins/obs-outputs/librtmp/hashswf.c

@@ -30,7 +30,20 @@
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 #endif
 #endif
 
 
-#ifdef USE_POLARSSL
+#if defined(USE_MBEDTLS)
+#include <mbedtls/md.h>
+#ifndef SHA256_DIGEST_LENGTH
+#define SHA256_DIGEST_LENGTH	32
+#endif
+typedef mbedtls_md_context_t *HMAC_CTX;
+#define HMAC_setup(ctx, key, len)	ctx = malloc(sizeof(mbedtls_md_context_t)); mbedtls_md_init(ctx); \
+  mbedtls_md_setup(ctx, mbedtls_md_info_from_type(MBEDTLS_MD_SHA256), 1); \
+  mbedtls_md_hmac_starts(ctx, (const unsigned char *)key, len)
+#define HMAC_crunch(ctx, buf, len) mbedtls_md_hmac_update(ctx, buf, len)
+#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; mbedtls_md_hmac_finish(ctx, dig)
+#define HMAC_close(ctx) free(ctx); mbedtls_md_free(ctx); ctx = NULL
+
+#elif defined(USE_POLARSSL)
 #include <polarssl/sha2.h>
 #include <polarssl/sha2.h>
 #ifndef SHA256_DIGEST_LENGTH
 #ifndef SHA256_DIGEST_LENGTH
 #define SHA256_DIGEST_LENGTH	32
 #define SHA256_DIGEST_LENGTH	32
@@ -40,6 +53,7 @@
 #define HMAC_crunch(ctx, buf, len)	sha2_hmac_update(&ctx, buf, len)
 #define HMAC_crunch(ctx, buf, len)	sha2_hmac_update(&ctx, buf, len)
 #define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig)
 #define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig)
 #define HMAC_close(ctx)
 #define HMAC_close(ctx)
+
 #elif defined(USE_GNUTLS)
 #elif defined(USE_GNUTLS)
 #include <nettle/hmac.h>
 #include <nettle/hmac.h>
 #ifndef SHA256_DIGEST_LENGTH
 #ifndef SHA256_DIGEST_LENGTH
@@ -51,6 +65,7 @@
 #define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(&ctx, len, buf)
 #define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(&ctx, len, buf)
 #define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig)
 #define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig)
 #define HMAC_close(ctx)
 #define HMAC_close(ctx)
+
 #else	/* USE_OPENSSL */
 #else	/* USE_OPENSSL */
 #include <openssl/ssl.h>
 #include <openssl/ssl.h>
 #include <openssl/sha.h>
 #include <openssl/sha.h>
@@ -161,8 +176,17 @@ HTTP_get(struct HTTP_ctx *http, const char *url, HTTP_read_callback *cb)
         goto leave;
         goto leave;
 #else
 #else
         TLS_client(RTMP_TLS_ctx, sb.sb_ssl);
         TLS_client(RTMP_TLS_ctx, sb.sb_ssl);
+
+#if defined(USE_MBEDTLS)
+        mbedtls_net_context *server_fd = &RTMP_TLS_ctx->net;
+        server_fd->fd = sb.sb_socket;
+        TLS_setfd(sb.sb_ssl, server_fd);
+#else
         TLS_setfd(sb.sb_ssl, sb.sb_socket);
         TLS_setfd(sb.sb_ssl, sb.sb_socket);
-        if (TLS_connect(sb.sb_ssl) < 0)
+#endif
+
+        int connect_return = TLS_connect(sb.sb_ssl);
+        if (connect_return < 0)
         {
         {
             RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed", __FUNCTION__);
             RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed", __FUNCTION__);
             ret = HTTPRES_LOST_CONNECTION;
             ret = HTTPRES_LOST_CONNECTION;
@@ -318,21 +342,21 @@ swfcrunch(void *ptr, size_t size, size_t nmemb, void *stream)
     {
     {
         unsigned char out[CHUNK];
         unsigned char out[CHUNK];
         i->zs->next_in = (unsigned char *)p;
         i->zs->next_in = (unsigned char *)p;
-        i->zs->avail_in = len;
+        i->zs->avail_in = (uInt)len;
         do
         do
         {
         {
             i->zs->avail_out = CHUNK;
             i->zs->avail_out = CHUNK;
             i->zs->next_out = out;
             i->zs->next_out = out;
             inflate(i->zs, Z_NO_FLUSH);
             inflate(i->zs, Z_NO_FLUSH);
             len = CHUNK - i->zs->avail_out;
             len = CHUNK - i->zs->avail_out;
-            i->size += len;
+            i->size += (int)len;
             HMAC_crunch(i->ctx, out, len);
             HMAC_crunch(i->ctx, out, len);
         }
         }
         while (i->zs->avail_out == 0);
         while (i->zs->avail_out == 0);
     }
     }
     else
     else
     {
     {
-        i->size += len;
+        i->size += (int)len;
         HMAC_crunch(i->ctx, (unsigned char *)p, len);
         HMAC_crunch(i->ctx, (unsigned char *)p, len);
     }
     }
     return size * nmemb;
     return size * nmemb;
@@ -469,7 +493,7 @@ RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash,
     home.av_val = "\\UserData";
     home.av_val = "\\UserData";
 #else
 #else
     hpre.av_val = getenv("HOMEDRIVE");
     hpre.av_val = getenv("HOMEDRIVE");
-    hpre.av_len = strlen(hpre.av_val);
+    hpre.av_len = (int)strlen(hpre.av_val);
     home.av_val = getenv("HOMEPATH");
     home.av_val = getenv("HOMEPATH");
 #endif
 #endif
 #define DIRSEP	"\\"
 #define DIRSEP	"\\"
@@ -482,7 +506,7 @@ RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash,
 #endif
 #endif
     if (!home.av_val)
     if (!home.av_val)
         home.av_val = ".";
         home.av_val = ".";
-    home.av_len = strlen(home.av_val);
+    home.av_len = (int)strlen(home.av_val);
 
 
     /* SWF hash info is cached in a fixed-format file.
     /* SWF hash info is cached in a fixed-format file.
      * url: <url of SWF file>
      * url: <url of SWF file>
@@ -528,7 +552,7 @@ RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash,
             if (strncmp(buf + 5, url, hlen))
             if (strncmp(buf + 5, url, hlen))
                 continue;
                 continue;
             r1 = strrchr(buf, '/');
             r1 = strrchr(buf, '/');
-            i = strlen(r1);
+            i = (int)strlen(r1);
             r1[--i] = '\0';
             r1[--i] = '\0';
             if (strncmp(r1, file, i))
             if (strncmp(r1, file, i))
                 continue;
                 continue;
@@ -543,7 +567,7 @@ RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash,
                 else if (!strncmp(buf, "hash: ", 6))
                 else if (!strncmp(buf, "hash: ", 6))
                 {
                 {
                     unsigned char *ptr = hash, *in = (unsigned char *)buf + 6;
                     unsigned char *ptr = hash, *in = (unsigned char *)buf + 6;
-                    int l = strlen((char *)in) - 1;
+                    int l = (int)strlen((char *)in) - 1;
                     for (i = 0; i < l; i += 2)
                     for (i = 0; i < l; i += 2)
                         *ptr++ = (HEX2BIN(in[i]) << 4) | HEX2BIN(in[i + 1]);
                         *ptr++ = (HEX2BIN(in[i]) << 4) | HEX2BIN(in[i + 1]);
                     got++;
                     got++;
@@ -625,7 +649,7 @@ RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash,
             if (q)
             if (q)
                 i = q - url;
                 i = q - url;
             else
             else
-                i = strlen(url);
+                i = (int)strlen(url);
 
 
             fprintf(f, "url: %.*s\n", i, url);
             fprintf(f, "url: %.*s\n", i, url);
         }
         }

+ 241 - 13
plugins/obs-outputs/librtmp/rtmp.c

@@ -40,7 +40,20 @@
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 #endif
 #endif
 
 
-#ifdef USE_POLARSSL
+#if defined(USE_MBEDTLS)
+#if defined(_WIN32)
+#include <windows.h>
+#include <wincrypt.h>
+#elif defined(__APPLE__)
+#include <Security/Security.h>
+#endif
+
+#include <mbedtls/ctr_drbg.h>
+#include <mbedtls/md5.h>
+#include <mbedtls/base64.h>
+#define MD5_DIGEST_LENGTH 16
+
+#elif defined(USE_POLARSSL)
 #include <polarssl/havege.h>
 #include <polarssl/havege.h>
 #include <polarssl/md5.h>
 #include <polarssl/md5.h>
 #include <polarssl/base64.h>
 #include <polarssl/base64.h>
@@ -57,7 +70,6 @@ static const char *my_dhm_P =
     "E8A700D60B7F1200FA8E77B0A979DABF";
     "E8A700D60B7F1200FA8E77B0A979DABF";
 
 
 static const char *my_dhm_G = "4";
 static const char *my_dhm_G = "4";
-
 #elif defined(USE_GNUTLS)
 #elif defined(USE_GNUTLS)
 #include <gnutls/gnutls.h>
 #include <gnutls/gnutls.h>
 #define MD5_DIGEST_LENGTH 16
 #define MD5_DIGEST_LENGTH 16
@@ -70,7 +82,8 @@ static const char *my_dhm_G = "4";
 #include <openssl/bio.h>
 #include <openssl/bio.h>
 #include <openssl/buffer.h>
 #include <openssl/buffer.h>
 #endif
 #endif
-TLS_CTX RTMP_TLS_ctx;
+
+TLS_CTX RTMP_TLS_ctx = NULL;
 #endif
 #endif
 
 
 #define RTMP_SIG_SIZE 1536
 #define RTMP_SIG_SIZE 1536
@@ -265,11 +278,101 @@ RTMP_LibVersion()
     return RTMP_LIB_VERSION;
     return RTMP_LIB_VERSION;
 }
 }
 
 
+void
+RTMP_TLS_LoadCerts() {
+#ifdef USE_MBEDTLS
+    mbedtls_x509_crt *chain = RTMP_TLS_ctx->cacert = calloc(1, sizeof(struct mbedtls_x509_crt));
+    mbedtls_x509_crt_init(chain);
+
+#if defined(_WIN32)
+    HCERTSTORE hCertStore;
+    PCCERT_CONTEXT pCertContext = NULL;
+
+    if (!(hCertStore = CertOpenSystemStore((HCRYPTPROV)NULL, L"ROOT"))) {
+        goto error;
+    }
+
+    while (pCertContext = CertEnumCertificatesInStore(hCertStore, pCertContext)) {
+        mbedtls_x509_crt_parse_der(chain,
+                                   (unsigned char *)pCertContext->pbCertEncoded,
+                                   pCertContext->cbCertEncoded);
+    }
+
+    CertFreeCertificateContext(pCertContext);
+    CertCloseStore(hCertStore, 0);
+#elif defined(__APPLE__)
+    SecKeychainRef keychain_ref;
+    CFMutableDictionaryRef search_settings_ref;
+    CFArrayRef result_ref;
+
+    if (SecKeychainOpen("/System/Library/Keychains/SystemRootCertificates.keychain",
+                        &keychain_ref)
+        != errSecSuccess) {
+      goto error;
+    }
+
+    search_settings_ref = CFDictionaryCreateMutable(NULL, 0, NULL, NULL);
+    CFDictionarySetValue(search_settings_ref, kSecClass, kSecClassCertificate);
+    CFDictionarySetValue(search_settings_ref, kSecMatchLimit, kSecMatchLimitAll);
+    CFDictionarySetValue(search_settings_ref, kSecReturnRef, kCFBooleanTrue);
+    CFDictionarySetValue(search_settings_ref, kSecMatchSearchList,
+                         CFArrayCreate(NULL, (const void **)&keychain_ref, 1, NULL));
+
+    if (SecItemCopyMatching(search_settings_ref, (CFTypeRef *)&result_ref)
+        != errSecSuccess) {
+      goto error;
+    }
+
+    for (CFIndex i = 0; i < CFArrayGetCount(result_ref); i++) {
+      SecCertificateRef item_ref = (SecCertificateRef)
+                                   CFArrayGetValueAtIndex(result_ref, i);
+      CFDataRef data_ref;
+
+      if ((data_ref = SecCertificateCopyData(item_ref))) {
+        mbedtls_x509_crt_parse_der(chain,
+                                   (unsigned char *)CFDataGetBytePtr(data_ref),
+                                   CFDataGetLength(data_ref));
+        CFRelease(data_ref);
+      }
+    }
+
+    CFRelease(keychain_ref);
+#elif defined(__linux__)
+    if (mbedtls_x509_crt_parse_path(chain, "/etc/ssl/certs/") != 0) {
+        goto error;
+    }
+#endif
+
+    mbedtls_ssl_conf_ca_chain(&RTMP_TLS_ctx->conf, chain, NULL);
+    return;
+
+error:
+    mbedtls_x509_crt_free(chain);
+    free(chain);
+    RTMP_TLS_ctx->cacert = NULL;
+#endif /* USE_MBEDTLS */
+}
+
 void
 void
 RTMP_TLS_Init()
 RTMP_TLS_Init()
 {
 {
 #ifdef CRYPTO
 #ifdef CRYPTO
-#ifdef USE_POLARSSL
+#if defined(USE_MBEDTLS)
+    const char * pers = "RTMP_TLS";
+    RTMP_TLS_ctx = calloc(1,sizeof(struct tls_ctx));
+
+    mbedtls_ssl_config_init(&RTMP_TLS_ctx->conf);
+    mbedtls_ctr_drbg_init(&RTMP_TLS_ctx->ctr_drbg);
+    mbedtls_entropy_init(&RTMP_TLS_ctx->entropy);
+
+    mbedtls_ctr_drbg_seed(&RTMP_TLS_ctx->ctr_drbg,
+                          mbedtls_entropy_func,
+                          &RTMP_TLS_ctx->entropy,
+                          (const unsigned char *)pers,
+                          strlen(pers));
+
+    RTMP_TLS_LoadCerts();
+#elif defined(USE_POLARSSL)
     /* Do this regardless of NO_SSL, we use havege for rtmpe too */
     /* Do this regardless of NO_SSL, we use havege for rtmpe too */
     RTMP_TLS_ctx = calloc(1,sizeof(struct tls_ctx));
     RTMP_TLS_ctx = calloc(1,sizeof(struct tls_ctx));
     havege_init(&RTMP_TLS_ctx->hs);
     havege_init(&RTMP_TLS_ctx->hs);
@@ -293,6 +396,26 @@ RTMP_TLS_Init()
     SSL_CTX_set_options(RTMP_TLS_ctx, SSL_OP_ALL);
     SSL_CTX_set_options(RTMP_TLS_ctx, SSL_OP_ALL);
     SSL_CTX_set_default_verify_paths(RTMP_TLS_ctx);
     SSL_CTX_set_default_verify_paths(RTMP_TLS_ctx);
 #endif
 #endif
+#else
+#endif
+}
+
+void
+RTMP_TLS_Free() {
+#ifdef USE_MBEDTLS
+    mbedtls_ssl_config_free(&RTMP_TLS_ctx->conf);
+    mbedtls_ctr_drbg_free(&RTMP_TLS_ctx->ctr_drbg);
+    mbedtls_entropy_free(&RTMP_TLS_ctx->entropy);
+
+    if (RTMP_TLS_ctx->cacert) {
+        mbedtls_x509_crt_free(RTMP_TLS_ctx->cacert);
+        free(RTMP_TLS_ctx->cacert);
+        RTMP_TLS_ctx->cacert = NULL;
+    }
+
+    // NO mbedtls_net_free() BECAUSE WE SET IT UP BY HAND!
+    free(RTMP_TLS_ctx);
+    RTMP_TLS_ctx = NULL;
 #endif
 #endif
 }
 }
 
 
@@ -303,7 +426,27 @@ RTMP_TLS_AllocServerContext(const char* cert, const char* key)
 #ifdef CRYPTO
 #ifdef CRYPTO
     if (!RTMP_TLS_ctx)
     if (!RTMP_TLS_ctx)
         RTMP_TLS_Init();
         RTMP_TLS_Init();
-#ifdef USE_POLARSSL
+#if defined(USE_MBEDTLS)
+    tls_server_ctx *tc = ctx = calloc(1, sizeof(struct tls_server_ctx));
+    tc->conf = &RTMP_TLS_ctx->conf;
+    tc->ctr_drbg = &RTMP_TLS_ctx->ctr_drbg;
+
+    mbedtls_x509_crt_init(&tc->cert);
+    if (mbedtls_x509_crt_parse_file(&tc->cert, cert))
+    {
+        free(tc);
+        return NULL;
+    }
+
+    mbedtls_pk_init(&tc->key);
+    if (mbedtls_pk_parse_keyfile(&tc->key, key, NULL))
+    {
+        mbedtls_x509_crt_free(&tc->cert);
+        mbedtls_pk_free(&tc->key);
+        free(tc);
+        return NULL;
+    }
+#elif defined(USE_POLARSSL)
     tls_server_ctx *tc = ctx = calloc(1, sizeof(struct tls_server_ctx));
     tls_server_ctx *tc = ctx = calloc(1, sizeof(struct tls_server_ctx));
     tc->dhm_P = my_dhm_P;
     tc->dhm_P = my_dhm_P;
     tc->dhm_G = my_dhm_G;
     tc->dhm_G = my_dhm_G;
@@ -350,7 +493,11 @@ void
 RTMP_TLS_FreeServerContext(void *ctx)
 RTMP_TLS_FreeServerContext(void *ctx)
 {
 {
 #ifdef CRYPTO
 #ifdef CRYPTO
-#ifdef USE_POLARSSL
+#if defined(USE_MBEDTLS)
+    mbedtls_x509_crt_free(&((tls_server_ctx*)ctx)->cert);
+    mbedtls_pk_free(&((tls_server_ctx*)ctx)->key);
+    free(ctx);
+#elif defined(USE_POLARSSL)
     x509_free(&((tls_server_ctx*)ctx)->cert);
     x509_free(&((tls_server_ctx*)ctx)->cert);
     rsa_free(&((tls_server_ctx*)ctx)->key);
     rsa_free(&((tls_server_ctx*)ctx)->key);
     free(ctx);
     free(ctx);
@@ -374,6 +521,10 @@ RTMP_Alloc()
 void
 void
 RTMP_Free(RTMP *r)
 RTMP_Free(RTMP *r)
 {
 {
+#if defined(CRYPTO) && defined(USE_MBEDTLS)
+  if (RTMP_TLS_ctx)
+    RTMP_TLS_Free();
+#endif
     free(r);
     free(r);
 }
 }
 
 
@@ -850,9 +1001,20 @@ int
 RTMP_TLS_Accept(RTMP *r, void *ctx)
 RTMP_TLS_Accept(RTMP *r, void *ctx)
 {
 {
 #if defined(CRYPTO) && !defined(NO_SSL)
 #if defined(CRYPTO) && !defined(NO_SSL)
-    TLS_server(ctx, r->m_sb.sb_ssl);
+    tls_server_ctx *srv_ctx = ctx;
+    TLS_server(srv_ctx, r->m_sb.sb_ssl);
+
+#if defined(USE_MBEDTLS)
+    mbedtls_net_context *client_fd = &RTMP_TLS_ctx->net;
+    mbedtls_net_init(client_fd);
+    client_fd->fd = r->m_sb.sb_socket;
+    TLS_setfd(r->m_sb.sb_ssl, client_fd);
+#else
     TLS_setfd(r->m_sb.sb_ssl, r->m_sb.sb_socket);
     TLS_setfd(r->m_sb.sb_ssl, r->m_sb.sb_socket);
-    if (TLS_accept(r->m_sb.sb_ssl) < 0)
+#endif
+
+    int connect_return = TLS_connect(r->m_sb.sb_ssl);
+    if (connect_return < 0)
     {
     {
         RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed", __FUNCTION__);
         RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed", __FUNCTION__);
         return FALSE;
         return FALSE;
@@ -872,10 +1034,59 @@ RTMP_Connect1(RTMP *r, RTMPPacket *cp)
     {
     {
 #if defined(CRYPTO) && !defined(NO_SSL)
 #if defined(CRYPTO) && !defined(NO_SSL)
         TLS_client(RTMP_TLS_ctx, r->m_sb.sb_ssl);
         TLS_client(RTMP_TLS_ctx, r->m_sb.sb_ssl);
+
+#if defined(USE_MBEDTLS)
+        mbedtls_net_context *server_fd = &RTMP_TLS_ctx->net;
+        server_fd->fd = r->m_sb.sb_socket;
+        TLS_setfd(r->m_sb.sb_ssl, server_fd);
+
+        // make sure we verify the certificate hostname
+        char hostname[MBEDTLS_SSL_MAX_HOST_NAME_LEN + 1];
+
+        if (r->Link.hostname.av_len >= MBEDTLS_SSL_MAX_HOST_NAME_LEN)
+            return FALSE;
+
+        memcpy(hostname, r->Link.hostname.av_val, r->Link.hostname.av_len);
+        hostname[r->Link.hostname.av_len] = 0;
+
+        if (mbedtls_ssl_set_hostname(r->m_sb.sb_ssl, hostname))
+            return FALSE;
+#else
         TLS_setfd(r->m_sb.sb_ssl, r->m_sb.sb_socket);
         TLS_setfd(r->m_sb.sb_ssl, r->m_sb.sb_socket);
-        if (TLS_connect(r->m_sb.sb_ssl) < 0)
+#endif
+
+        int connect_return = TLS_connect(r->m_sb.sb_ssl);
+        if (connect_return < 0)
         {
         {
-            RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed", __FUNCTION__);
+#if defined(USE_MBEDTLS)
+            if (connect_return == MBEDTLS_ERR_X509_CERT_VERIFY_FAILED)
+            {
+                r->last_error_code = connect_return;
+
+                // show a more detailed error in the log if possible
+                int verify_result = mbedtls_ssl_get_verify_result(r->m_sb.sb_ssl);
+                if (verify_result)
+                {
+                    char err[256], *e;
+                    if (mbedtls_x509_crt_verify_info(err, sizeof(err), "", verify_result) > 0)
+                    {
+                        e = strchr(err, '\n');
+                        if (e)
+                            *e = '\0';
+                    }
+                    else
+                    {
+                        strcpy(err, "unknown error");
+                    }
+                    RTMP_Log(RTMP_LOGERROR, "%s, Cert verify failed: %d (%s)", __FUNCTION__, verify_result, err);
+                    RTMP_Close(r);
+                    return FALSE;
+                }
+            }
+#endif
+            // output the error in a format that matches mbedTLS
+            connect_return = abs(connect_return);
+            RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed: -0x%x", __FUNCTION__, connect_return);
             RTMP_Close(r);
             RTMP_Close(r);
             return FALSE;
             return FALSE;
         }
         }
@@ -883,7 +1094,6 @@ RTMP_Connect1(RTMP *r, RTMPPacket *cp)
         RTMP_Log(RTMP_LOGERROR, "%s, no SSL/TLS support", __FUNCTION__);
         RTMP_Log(RTMP_LOGERROR, "%s, no SSL/TLS support", __FUNCTION__);
         RTMP_Close(r);
         RTMP_Close(r);
         return FALSE;
         return FALSE;
-
 #endif
 #endif
     }
     }
     if (r->Link.protocol & RTMP_FEATURE_HTTP)
     if (r->Link.protocol & RTMP_FEATURE_HTTP)
@@ -2402,7 +2612,19 @@ b64enc(const unsigned char *input, int length, char *output, int maxsize)
 {
 {
     (void)maxsize;
     (void)maxsize;
 
 
-#ifdef USE_POLARSSL
+#if defined(USE_MBEDTLS)
+    size_t osize;
+    if(mbedtls_base64_encode((unsigned char *) output, maxsize, &osize, input, length) == 0)
+    {
+        output[osize] = '\0';
+        return 1;
+    }
+    else
+    {
+        RTMP_Log(RTMP_LOGDEBUG, "%s, error", __FUNCTION__);
+        return 0;
+    }
+#elif defined(USE_POLARSSL)
     size_t buf_size = maxsize;
     size_t buf_size = maxsize;
     if(base64_encode((unsigned char *) output, &buf_size, input, length) == 0)
     if(base64_encode((unsigned char *) output, &buf_size, input, length) == 0)
     {
     {
@@ -2461,7 +2683,13 @@ b64enc(const unsigned char *input, int length, char *output, int maxsize)
     return 1;
     return 1;
 }
 }
 
 
-#ifdef USE_POLARSSL
+#if defined(USE_MBEDTLS)
+typedef	mbedtls_md5_context MD5_CTX;
+#define MD5_Init(ctx)	mbedtls_md5_init(ctx); mbedtls_md5_starts(ctx)
+#define MD5_Update(ctx,data,len)	mbedtls_md5_update(ctx,(unsigned char *)data,len)
+#define MD5_Final(dig,ctx)	mbedtls_md5_finish(ctx,dig); mbedtls_md5_free(ctx)
+
+#elif defined(USE_POLARSSL)
 #define MD5_CTX	md5_context
 #define MD5_CTX	md5_context
 #define MD5_Init(ctx)	md5_starts(ctx)
 #define MD5_Init(ctx)	md5_starts(ctx)
 #define MD5_Update(ctx,data,len)	md5_update(ctx,(unsigned char *)data,len)
 #define MD5_Update(ctx,data,len)	md5_update(ctx,(unsigned char *)data,len)

+ 71 - 1
plugins/obs-outputs/librtmp/rtmp_sys.h

@@ -80,7 +80,76 @@
 
 
 #include "rtmp.h"
 #include "rtmp.h"
 
 
-#ifdef USE_POLARSSL
+#if defined(USE_MBEDTLS)
+#include <mbedtls/version.h>
+#include <mbedtls/net.h>
+#include <mbedtls/ssl.h>
+#include <mbedtls/ctr_drbg.h>
+#include <mbedtls/entropy.h>
+
+#define my_dhm_P \
+    "E4004C1F94182000103D883A448B3F80" \
+    "2CE4B44A83301270002C20D0321CFD00" \
+    "11CCEF784C26A400F43DFB901BCA7538" \
+    "F2C6B176001CF5A0FD16D2C48B1D0C1C" \
+    "F6AC8E1DA6BCC3B4E1F96B0564965300" \
+    "FFA1D0B601EB2800F489AA512C4B248C" \
+    "01F76949A60BB7F00A40B1EAB64BDD48" \
+    "E8A700D60B7F1200FA8E77B0A979DABF"
+
+#define my_dhm_G "4"
+
+#define	SSL_SET_SESSION(S,resume,timeout,ctx)	mbedtls_ssl_set_session(S,ctx)
+
+typedef struct tls_ctx
+{
+    mbedtls_entropy_context entropy;
+    mbedtls_ctr_drbg_context ctr_drbg;
+    mbedtls_ssl_config conf;
+    mbedtls_ssl_session ssn;
+    mbedtls_x509_crt *cacert;
+    mbedtls_net_context net;
+} tls_ctx;
+
+typedef struct tls_server_ctx
+{
+  mbedtls_ssl_config *conf;
+  mbedtls_ctr_drbg_context *ctr_drbg;
+  mbedtls_pk_context key;
+  mbedtls_x509_crt cert;
+} tls_server_ctx;
+
+typedef tls_ctx *TLS_CTX;
+
+#define TLS_client(ctx,s)	\
+  s = malloc(sizeof(mbedtls_ssl_context));\
+  mbedtls_ssl_init(s);\
+  mbedtls_ssl_setup(s, &ctx->conf);\
+	mbedtls_ssl_config_defaults(&ctx->conf, MBEDTLS_SSL_IS_CLIENT, MBEDTLS_SSL_TRANSPORT_STREAM, MBEDTLS_SSL_PRESET_DEFAULT);\
+  mbedtls_ssl_conf_authmode(&ctx->conf, MBEDTLS_SSL_VERIFY_REQUIRED);\
+	mbedtls_ssl_conf_rng(&ctx->conf, mbedtls_ctr_drbg_random, &ctx->ctr_drbg)
+
+#define TLS_server(ctx,s)\
+  s = malloc(sizeof(mbedtls_ssl_context));\
+  mbedtls_ssl_init(s);\
+  mbedtls_ssl_setup(s, ctx->conf);\
+	mbedtls_ssl_conf_endpoint(ctx->conf, MBEDTLS_SSL_IS_SERVER);\
+  mbedtls_ssl_conf_authmode(ctx->conf, MBEDTLS_SSL_VERIFY_REQUIRED);\
+	mbedtls_ssl_conf_rng(ctx->conf, mbedtls_ctr_drbg_random, ctx->ctr_drbg);\
+	mbedtls_ssl_conf_own_cert(ctx->conf, &ctx->cert, &ctx->key);\
+	mbedtls_ssl_conf_dh_param_bin(ctx->conf,\
+    (const unsigned char *)my_dhm_P, strlen(my_dhm_P),\
+    (const unsigned char *)my_dhm_G, strlen(my_dhm_G))
+
+#define TLS_setfd(s,fd)	mbedtls_ssl_set_bio(s, fd, mbedtls_net_send, mbedtls_net_recv, NULL)
+#define TLS_connect(s)	mbedtls_ssl_handshake(s)
+#define TLS_accept(s)	mbedtls_ssl_handshake(s)
+#define TLS_read(s,b,l)	mbedtls_ssl_read(s,(unsigned char *)b,l)
+#define TLS_write(s,b,l)	mbedtls_ssl_write(s,(unsigned char *)b,l)
+#define TLS_shutdown(s)	mbedtls_ssl_close_notify(s)
+#define TLS_close(s)	mbedtls_ssl_free(s); free(s)
+
+#elif defined(USE_POLARSSL)
 #include <polarssl/version.h>
 #include <polarssl/version.h>
 #include <polarssl/net.h>
 #include <polarssl/net.h>
 #include <polarssl/ssl.h>
 #include <polarssl/ssl.h>
@@ -128,6 +197,7 @@ typedef struct tls_server_ctx
 #define TLS_shutdown(s)	ssl_close_notify(s)
 #define TLS_shutdown(s)	ssl_close_notify(s)
 #define TLS_close(s)	ssl_free(s); free(s)
 #define TLS_close(s)	ssl_free(s); free(s)
 
 
+
 #elif defined(USE_GNUTLS)
 #elif defined(USE_GNUTLS)
 #include <gnutls/gnutls.h>
 #include <gnutls/gnutls.h>
 typedef struct tls_ctx
 typedef struct tls_ctx

+ 9 - 0
plugins/obs-outputs/rtmp-stream.c

@@ -438,6 +438,15 @@ static void set_output_error(struct rtmp_stream *stream)
 	}
 	}
 #endif
 #endif
 
 
+	// non platform-specific errors
+	if (!msg) {
+		switch (stream->rtmp.last_error_code) {
+		case -0x2700:
+			msg = obs_module_text("SSLCertVerifyFailed");
+			break;
+		}
+	}
+
 	obs_output_set_last_error(stream->output, msg);
 	obs_output_set_last_error(stream->output, msg);
 }
 }
 
 

+ 4 - 4
plugins/obs-outputs/rtmp-windows.c

@@ -169,13 +169,13 @@ static enum data_ret write_data(struct rtmp_stream *stream, bool *can_write,
 		size_t send_len =
 		size_t send_len =
 			min(latency_packet_size, stream->write_buf_len);
 			min(latency_packet_size, stream->write_buf_len);
 
 
-		ret = send(stream->rtmp.m_sb.sb_socket,
+		ret = RTMPSockBuf_Send(&stream->rtmp.m_sb,
 				(const char *)stream->write_buf,
 				(const char *)stream->write_buf,
-				(int)send_len, 0);
+				(int)send_len);
 	} else {
 	} else {
-		ret = send(stream->rtmp.m_sb.sb_socket,
+		ret = RTMPSockBuf_Send(&stream->rtmp.m_sb,
 				(const char *)stream->write_buf,
 				(const char *)stream->write_buf,
-				(int)stream->write_buf_len, 0);
+				(int)stream->write_buf_len);
 	}
 	}
 
 
 	if (ret > 0) {
 	if (ret > 0) {