Browse Source

Merge pull request #2357 from obsproject/librtmp-thread-safety

obs-outputs: Various librtmp mbedtls fixes
Jim 5 years ago
parent
commit
86d3eb3088

+ 1 - 0
CI/install-dependencies-linux.sh

@@ -47,6 +47,7 @@ sudo apt-get install -y \
         libxcb-xinerama0-dev \
         libxcomposite-dev \
         libxinerama-dev \
+        libmbedtls-dev \
         pkg-config \
         python3-dev \
         qtbase5-dev \

+ 5 - 4
plugins/obs-outputs/librtmp/dh.h

@@ -51,13 +51,13 @@ typedef struct MDH
 #define MDH_new()	calloc(1,sizeof(MDH))
 #define MDH_free(vp)	{MDH *_dh = vp; mbedtls_dhm_free(&_dh->ctx); MP_free(_dh->p); MP_free(_dh->g); MP_free(_dh->pub_key); MP_free(_dh->priv_key); free(_dh);}
 
-static int MDH_generate_key(MDH *dh)
+static int MDH_generate_key(RTMP *r, MDH *dh)
 {
     unsigned char out[2];
     MP_set(&dh->ctx.P, dh->p);
     MP_set(&dh->ctx.G, dh->g);
     dh->ctx.len = 128;
-    mbedtls_dhm_make_public(&dh->ctx, 1024, out, 1, mbedtls_ctr_drbg_random, &RTMP_TLS_ctx->ctr_drbg);
+    mbedtls_dhm_make_public(&dh->ctx, 1024, out, 1, mbedtls_ctr_drbg_random, &r->RTMP_TLS_ctx->ctr_drbg);
     MP_new(dh->pub_key);
     MP_new(dh->priv_key);
     MP_set(dh->pub_key, &dh->ctx.GX);
@@ -283,8 +283,9 @@ failed:
 }
 
 static int
-DHGenerateKey(MDH *dh)
+DHGenerateKey(RTMP *r)
 {
+    MDH *dh = r->Link.dh;
     size_t res = 0;
     if (!dh)
         return 0;
@@ -293,7 +294,7 @@ DHGenerateKey(MDH *dh)
     {
         MP_t q1 = NULL;
 
-        if (!MDH_generate_key(dh))
+        if (!MDH_generate_key(r, dh))
             return 0;
 
         MP_gethex(q1, Q1024, res);

+ 1 - 356
plugins/obs-outputs/librtmp/handshake.h

@@ -890,7 +890,7 @@ HandShake(RTMP * r, int FP9HandShake)
             dhposClient = getdh(clientsig, RTMP_SIG_SIZE);
             RTMP_Log(RTMP_LOGDEBUG, "%s: DH pubkey position: %d", __FUNCTION__, dhposClient);
 
-            if (!DHGenerateKey(r->Link.dh))
+            if (!DHGenerateKey(r))
             {
                 RTMP_Log(RTMP_LOGERROR, "%s: Couldn't generate Diffie-Hellmann public key!",
                          __FUNCTION__);
@@ -1178,358 +1178,3 @@ HandShake(RTMP * r, int FP9HandShake)
     RTMP_Log(RTMP_LOGDEBUG, "%s: Handshaking finished....", __FUNCTION__);
     return TRUE;
 }
-
-static int
-SHandShake(RTMP * r)
-{
-    int i, offalg = 0;
-    int dhposServer = 0;
-    int digestPosServer = 0;
-    RC4_handle keyIn = 0;
-    RC4_handle keyOut = 0;
-    int FP9HandShake = FALSE;
-    int encrypted;
-
-#ifndef _DEBUG
-    int32_t *ip;
-#endif
-
-    uint8_t clientsig[RTMP_SIG_SIZE];
-    uint8_t serverbuf[RTMP_SIG_SIZE + 4], *serversig = serverbuf+4;
-    uint8_t type;
-    uint32_t uptime;
-    getoff *getdh = NULL, *getdig = NULL;
-
-    if (ReadN(r, (char *)&type, 1) != 1)	/* 0x03 or 0x06 */
-        return FALSE;
-
-    if (ReadN(r, (char *)clientsig, RTMP_SIG_SIZE) != RTMP_SIG_SIZE)
-        return FALSE;
-
-    RTMP_Log(RTMP_LOGDEBUG, "%s: Type Requested : %02X", __FUNCTION__, type);
-    RTMP_LogHex(RTMP_LOGDEBUG2, clientsig, RTMP_SIG_SIZE);
-
-    if (type == 3)
-    {
-        encrypted = FALSE;
-    }
-    else if (type == 6 || type == 8)
-    {
-        offalg = 1;
-        encrypted = TRUE;
-        FP9HandShake = TRUE;
-        r->Link.protocol |= RTMP_FEATURE_ENC;
-        /* use FP10 if client is capable */
-        if (clientsig[4] == 128)
-            type = 8;
-    }
-    else
-    {
-        RTMP_Log(RTMP_LOGERROR, "%s: Unknown version %02x",
-                 __FUNCTION__, type);
-        return FALSE;
-    }
-
-    if (!FP9HandShake && clientsig[4])
-        FP9HandShake = TRUE;
-
-    serversig[-1] = type;
-
-    r->Link.rc4keyIn = r->Link.rc4keyOut = 0;
-
-    uptime = htonl(RTMP_GetTime());
-    memcpy(serversig, &uptime, 4);
-
-    if (FP9HandShake)
-    {
-        /* Server version */
-        serversig[4] = 3;
-        serversig[5] = 5;
-        serversig[6] = 1;
-        serversig[7] = 1;
-
-        getdig = digoff[offalg];
-        getdh  = dhoff[offalg];
-    }
-    else
-    {
-        memset(&serversig[4], 0, 4);
-    }
-
-    /* generate random data */
-#ifdef _DEBUG
-    memset(serversig+8, 0, RTMP_SIG_SIZE-8);
-#else
-    ip = (int32_t *)(serversig+8);
-    for (i = 2; i < RTMP_SIG_SIZE/4; i++)
-        *ip++ = rand();
-#endif
-
-    /* set handshake digest */
-    if (FP9HandShake)
-    {
-        if (encrypted)
-        {
-            /* generate Diffie-Hellmann parameters */
-            r->Link.dh = DHInit(1024);
-            if (!r->Link.dh)
-            {
-                RTMP_Log(RTMP_LOGERROR, "%s: Couldn't initialize Diffie-Hellmann!",
-                         __FUNCTION__);
-                return FALSE;
-            }
-
-            dhposServer = getdh(serversig, RTMP_SIG_SIZE);
-            RTMP_Log(RTMP_LOGDEBUG, "%s: DH pubkey position: %d", __FUNCTION__, dhposServer);
-
-            if (!DHGenerateKey(r->Link.dh))
-            {
-                RTMP_Log(RTMP_LOGERROR, "%s: Couldn't generate Diffie-Hellmann public key!",
-                         __FUNCTION__);
-                return FALSE;
-            }
-
-            if (!DHGetPublicKey
-                    (r->Link.dh, (uint8_t *) &serversig[dhposServer], 128))
-            {
-                RTMP_Log(RTMP_LOGERROR, "%s: Couldn't write public key!", __FUNCTION__);
-                return FALSE;
-            }
-        }
-
-        digestPosServer = getdig(serversig, RTMP_SIG_SIZE);	/* reuse this value in verification */
-        RTMP_Log(RTMP_LOGDEBUG, "%s: Server digest offset: %d", __FUNCTION__,
-                 digestPosServer);
-
-        CalculateDigest(digestPosServer, serversig, GenuineFMSKey, 36,
-                        &serversig[digestPosServer]);
-
-        RTMP_Log(RTMP_LOGDEBUG, "%s: Initial server digest: ", __FUNCTION__);
-        RTMP_LogHex(RTMP_LOGDEBUG, serversig + digestPosServer,
-                    SHA256_DIGEST_LENGTH);
-    }
-
-    RTMP_Log(RTMP_LOGDEBUG2, "Serversig: ");
-    RTMP_LogHex(RTMP_LOGDEBUG2, serversig, RTMP_SIG_SIZE);
-
-    if (!WriteN(r, (char *)serversig-1, RTMP_SIG_SIZE + 1))
-        return FALSE;
-
-    /* decode client response */
-    memcpy(&uptime, clientsig, 4);
-    uptime = ntohl(uptime);
-
-    RTMP_Log(RTMP_LOGDEBUG, "%s: Client Uptime : %d", __FUNCTION__, uptime);
-    RTMP_Log(RTMP_LOGDEBUG, "%s: Player Version: %d.%d.%d.%d", __FUNCTION__, clientsig[4],
-             clientsig[5], clientsig[6], clientsig[7]);
-
-    if (FP9HandShake)
-    {
-        uint8_t digestResp[SHA256_DIGEST_LENGTH];
-        uint8_t *signatureResp = NULL;
-
-        /* we have to use this signature now to find the correct algorithms for getting the digest and DH positions */
-        int digestPosClient = getdig(clientsig, RTMP_SIG_SIZE);
-
-        if (!VerifyDigest(digestPosClient, clientsig, GenuineFPKey, 30))
-        {
-            RTMP_Log(RTMP_LOGWARNING, "Trying different position for client digest!");
-            offalg ^= 1;
-            getdig = digoff[offalg];
-            getdh  = dhoff[offalg];
-
-            digestPosClient = getdig(clientsig, RTMP_SIG_SIZE);
-
-            if (!VerifyDigest(digestPosClient, clientsig, GenuineFPKey, 30))
-            {
-                RTMP_Log(RTMP_LOGERROR, "Couldn't verify the client digest");	/* continuing anyway will probably fail */
-                return FALSE;
-            }
-        }
-
-        /* generate SWFVerification token (SHA256 HMAC hash of decompressed SWF, key are the last 32 bytes of the server handshake) */
-        if (r->Link.SWFSize)
-        {
-            const char swfVerify[] = { 0x01, 0x01 };
-            char *vend = r->Link.SWFVerificationResponse+sizeof(r->Link.SWFVerificationResponse);
-
-            memcpy(r->Link.SWFVerificationResponse, swfVerify, 2);
-            AMF_EncodeInt32(&r->Link.SWFVerificationResponse[2], vend, r->Link.SWFSize);
-            AMF_EncodeInt32(&r->Link.SWFVerificationResponse[6], vend, r->Link.SWFSize);
-            HMACsha256(r->Link.SWFHash, SHA256_DIGEST_LENGTH,
-                       &serversig[RTMP_SIG_SIZE - SHA256_DIGEST_LENGTH],
-                       SHA256_DIGEST_LENGTH,
-                       (uint8_t *)&r->Link.SWFVerificationResponse[10]);
-        }
-
-        /* do Diffie-Hellmann Key exchange for encrypted RTMP */
-        if (encrypted)
-        {
-            int dhposClient, len;
-            /* compute secret key */
-            uint8_t secretKey[128] = { 0 };
-
-            dhposClient = getdh(clientsig, RTMP_SIG_SIZE);
-            RTMP_Log(RTMP_LOGDEBUG, "%s: Client DH public key offset: %d", __FUNCTION__,
-                     dhposClient);
-            len =
-                DHComputeSharedSecretKey(r->Link.dh,
-                                         (uint8_t *) &clientsig[dhposClient], 128,
-                                         secretKey);
-            if (len < 0)
-            {
-                RTMP_Log(RTMP_LOGDEBUG, "%s: Wrong secret key position!", __FUNCTION__);
-                return FALSE;
-            }
-
-            RTMP_Log(RTMP_LOGDEBUG, "%s: Secret key: ", __FUNCTION__);
-            RTMP_LogHex(RTMP_LOGDEBUG, secretKey, 128);
-
-            InitRC4Encryption(secretKey,
-                              (uint8_t *) &clientsig[dhposClient],
-                              (uint8_t *) &serversig[dhposServer],
-                              &keyIn, &keyOut);
-        }
-
-
-        /* calculate response now */
-        signatureResp = clientsig+RTMP_SIG_SIZE-SHA256_DIGEST_LENGTH;
-
-        HMACsha256(&clientsig[digestPosClient], SHA256_DIGEST_LENGTH,
-                   GenuineFMSKey, sizeof(GenuineFMSKey), digestResp);
-        HMACsha256(clientsig, RTMP_SIG_SIZE - SHA256_DIGEST_LENGTH, digestResp,
-                   SHA256_DIGEST_LENGTH, signatureResp);
-#ifdef FP10
-        if (type == 8 )
-        {
-            uint8_t *dptr = digestResp;
-            uint8_t *sig = signatureResp;
-            /* encrypt signatureResp */
-            for (i=0; i<SHA256_DIGEST_LENGTH; i+=8)
-                rtmpe8_sig(sig+i, sig+i, dptr[i] % 15);
-        }
-        else if (type == 9)
-        {
-            uint8_t *dptr = digestResp;
-            uint8_t *sig = signatureResp;
-            /* encrypt signatureResp */
-            for (i=0; i<SHA256_DIGEST_LENGTH; i+=8)
-                rtmpe9_sig(sig+i, sig+i, dptr[i] % 15);
-        }
-#endif
-
-        /* some info output */
-        RTMP_Log(RTMP_LOGDEBUG,
-                 "%s: Calculated digest key from secure key and server digest: ",
-                 __FUNCTION__);
-        RTMP_LogHex(RTMP_LOGDEBUG, digestResp, SHA256_DIGEST_LENGTH);
-
-        RTMP_Log(RTMP_LOGDEBUG, "%s: Server signature calculated:", __FUNCTION__);
-        RTMP_LogHex(RTMP_LOGDEBUG, signatureResp, SHA256_DIGEST_LENGTH);
-    }
-#if 0
-    else
-    {
-        uptime = htonl(RTMP_GetTime());
-        memcpy(clientsig+4, &uptime, 4);
-    }
-#endif
-
-    RTMP_Log(RTMP_LOGDEBUG2, "%s: Sending handshake response: ",
-             __FUNCTION__);
-    RTMP_LogHex(RTMP_LOGDEBUG2, clientsig, RTMP_SIG_SIZE);
-
-    if (!WriteN(r, (char *)clientsig, RTMP_SIG_SIZE))
-        return FALSE;
-
-    /* 2nd part of handshake */
-    if (ReadN(r, (char *)clientsig, RTMP_SIG_SIZE) != RTMP_SIG_SIZE)
-        return FALSE;
-
-    RTMP_Log(RTMP_LOGDEBUG2, "%s: 2nd handshake: ", __FUNCTION__);
-    RTMP_LogHex(RTMP_LOGDEBUG2, clientsig, RTMP_SIG_SIZE);
-
-    if (FP9HandShake)
-    {
-        uint8_t signature[SHA256_DIGEST_LENGTH];
-        uint8_t digest[SHA256_DIGEST_LENGTH];
-
-        RTMP_Log(RTMP_LOGDEBUG, "%s: Client sent signature:", __FUNCTION__);
-        RTMP_LogHex(RTMP_LOGDEBUG, &clientsig[RTMP_SIG_SIZE - SHA256_DIGEST_LENGTH],
-                    SHA256_DIGEST_LENGTH);
-
-        /* verify client response */
-        HMACsha256(&serversig[digestPosServer], SHA256_DIGEST_LENGTH,
-                   GenuineFPKey, sizeof(GenuineFPKey), digest);
-        HMACsha256(clientsig, RTMP_SIG_SIZE - SHA256_DIGEST_LENGTH, digest,
-                   SHA256_DIGEST_LENGTH, signature);
-#ifdef FP10
-        if (type == 8 )
-        {
-            uint8_t *dptr = digest;
-            uint8_t *sig = signature;
-            /* encrypt signatureResp */
-            for (i=0; i<SHA256_DIGEST_LENGTH; i+=8)
-                rtmpe8_sig(sig+i, sig+i, dptr[i] % 15);
-        }
-        else if (type == 9)
-        {
-            uint8_t *dptr = digest;
-            uint8_t *sig = signature;
-            /* encrypt signatureResp */
-            for (i=0; i<SHA256_DIGEST_LENGTH; i+=8)
-                rtmpe9_sig(sig+i, sig+i, dptr[i] % 15);
-        }
-#endif
-
-        /* show some information */
-        RTMP_Log(RTMP_LOGDEBUG, "%s: Digest key: ", __FUNCTION__);
-        RTMP_LogHex(RTMP_LOGDEBUG, digest, SHA256_DIGEST_LENGTH);
-
-        RTMP_Log(RTMP_LOGDEBUG, "%s: Signature calculated:", __FUNCTION__);
-        RTMP_LogHex(RTMP_LOGDEBUG, signature, SHA256_DIGEST_LENGTH);
-        if (memcmp
-                (signature, &clientsig[RTMP_SIG_SIZE - SHA256_DIGEST_LENGTH],
-                 SHA256_DIGEST_LENGTH) != 0)
-        {
-            RTMP_Log(RTMP_LOGWARNING, "%s: Client not genuine Adobe!", __FUNCTION__);
-            return FALSE;
-        }
-        else
-        {
-            RTMP_Log(RTMP_LOGDEBUG, "%s: Genuine Adobe Flash Player", __FUNCTION__);
-        }
-
-        if (encrypted)
-        {
-            char buff[RTMP_SIG_SIZE];
-            /* set keys for encryption from now on */
-            r->Link.rc4keyIn = keyIn;
-            r->Link.rc4keyOut = keyOut;
-
-            /* update the keystreams */
-            if (r->Link.rc4keyIn)
-            {
-                RC4_encrypt(r->Link.rc4keyIn, RTMP_SIG_SIZE, (uint8_t *) buff);
-            }
-
-            if (r->Link.rc4keyOut)
-            {
-                RC4_encrypt(r->Link.rc4keyOut, RTMP_SIG_SIZE, (uint8_t *) buff);
-            }
-        }
-    }
-    else
-    {
-        if (memcmp(serversig, clientsig, RTMP_SIG_SIZE) != 0)
-        {
-            RTMP_Log(RTMP_LOGWARNING, "%s: client signature does not match!",
-                     __FUNCTION__);
-        }
-    }
-
-    // TODO(mgoulet): Should this have an Rc4_free?
-
-    RTMP_Log(RTMP_LOGDEBUG, "%s: Handshaking finished....", __FUNCTION__);
-    return TRUE;
-}

+ 2 - 3
plugins/obs-outputs/librtmp/hashswf.c

@@ -20,6 +20,7 @@
  *  http://www.gnu.org/copyleft/lgpl.html
  */
 
+#ifdef USE_HASHSWF
 #include "rtmp_sys.h"
 #include "log.h"
 #include "http.h"
@@ -77,9 +78,6 @@ typedef mbedtls_md_context_t *HMAC_CTX;
 #define HMAC_close(ctx)	HMAC_CTX_cleanup(&ctx)
 #endif
 
-extern void RTMP_TLS_Init();
-extern TLS_CTX RTMP_TLS_ctx;
-
 #include <zlib.h>
 
 #endif /* CRYPTO */
@@ -692,3 +690,4 @@ RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash,
     return -1;
 }
 #endif
+#endif

+ 50 - 226
plugins/obs-outputs/librtmp/rtmp.c

@@ -87,7 +87,6 @@ static const char *my_dhm_G = "4";
 #include <openssl/buffer.h>
 #endif
 
-TLS_CTX RTMP_TLS_ctx = NULL;
 #endif
 
 #define RTMP_SIG_SIZE 1536
@@ -285,9 +284,9 @@ RTMP_LibVersion()
 }
 
 void
-RTMP_TLS_LoadCerts() {
+RTMP_TLS_LoadCerts(RTMP *r) {
 #ifdef USE_MBEDTLS
-    mbedtls_x509_crt *chain = RTMP_TLS_ctx->cacert = calloc(1, sizeof(struct mbedtls_x509_crt));
+    mbedtls_x509_crt *chain = r->RTMP_TLS_ctx->cacert = calloc(1, sizeof(struct mbedtls_x509_crt));
     mbedtls_x509_crt_init(chain);
 
 #if defined(_WIN32)
@@ -345,39 +344,44 @@ RTMP_TLS_LoadCerts() {
     CFRelease(keychain_ref);
 #elif defined(__linux__)
     if (mbedtls_x509_crt_parse_path(chain, "/etc/ssl/certs/") < 0) {
+        RTMP_Log(RTMP_LOGERROR, "mbedtls_x509_crt_parse_path: Couldn't parse "
+            "/etc/ssl/certs");
         goto error;
     }
 #endif
 
-    mbedtls_ssl_conf_ca_chain(&RTMP_TLS_ctx->conf, chain, NULL);
+    mbedtls_ssl_conf_ca_chain(&r->RTMP_TLS_ctx->conf, chain, NULL);
     return;
 
 error:
+    RTMP_Log(RTMP_LOGERROR, "RTMP_TLS_LoadCerts: Failed to load "
+        "root certificate chains, RTMPS connections will likely "
+        "fail");
     mbedtls_x509_crt_free(chain);
     free(chain);
-    RTMP_TLS_ctx->cacert = NULL;
+    r->RTMP_TLS_ctx->cacert = NULL;
 #endif /* USE_MBEDTLS */
 }
 
 void
-RTMP_TLS_Init()
+RTMP_TLS_Init(RTMP *r)
 {
 #ifdef CRYPTO
 #if defined(USE_MBEDTLS)
     const char * pers = "RTMP_TLS";
-    RTMP_TLS_ctx = calloc(1,sizeof(struct tls_ctx));
+    r->RTMP_TLS_ctx = calloc(1,sizeof(struct tls_ctx));
 
-    mbedtls_ssl_config_init(&RTMP_TLS_ctx->conf);
-    mbedtls_ctr_drbg_init(&RTMP_TLS_ctx->ctr_drbg);
-    mbedtls_entropy_init(&RTMP_TLS_ctx->entropy);
+    mbedtls_ssl_config_init(&r->RTMP_TLS_ctx->conf);
+    mbedtls_ctr_drbg_init(&r->RTMP_TLS_ctx->ctr_drbg);
+    mbedtls_entropy_init(&r->RTMP_TLS_ctx->entropy);
 
-    mbedtls_ctr_drbg_seed(&RTMP_TLS_ctx->ctr_drbg,
+    mbedtls_ctr_drbg_seed(&r->RTMP_TLS_ctx->ctr_drbg,
                           mbedtls_entropy_func,
-                          &RTMP_TLS_ctx->entropy,
+                          &r->RTMP_TLS_ctx->entropy,
                           (const unsigned char *)pers,
                           strlen(pers));
 
-    RTMP_TLS_LoadCerts();
+    RTMP_TLS_LoadCerts(r);
 #elif defined(USE_POLARSSL)
     /* Do this regardless of NO_SSL, we use havege for rtmpe too */
     RTMP_TLS_ctx = calloc(1,sizeof(struct tls_ctx));
@@ -407,114 +411,24 @@ RTMP_TLS_Init()
 }
 
 void
-RTMP_TLS_Free() {
+RTMP_TLS_Free(RTMP *r) {
 #ifdef USE_MBEDTLS
-    mbedtls_ssl_config_free(&RTMP_TLS_ctx->conf);
-    mbedtls_ctr_drbg_free(&RTMP_TLS_ctx->ctr_drbg);
-    mbedtls_entropy_free(&RTMP_TLS_ctx->entropy);
-
-    if (RTMP_TLS_ctx->cacert) {
-        mbedtls_x509_crt_free(RTMP_TLS_ctx->cacert);
-        free(RTMP_TLS_ctx->cacert);
-        RTMP_TLS_ctx->cacert = NULL;
-    }
-
-    // NO mbedtls_net_free() BECAUSE WE SET IT UP BY HAND!
-    free(RTMP_TLS_ctx);
-    RTMP_TLS_ctx = NULL;
-#endif
-}
 
-void *
-RTMP_TLS_AllocServerContext(const char* cert, const char* key)
-{
-    void *ctx = NULL;
-#ifdef CRYPTO
-    if (!RTMP_TLS_ctx)
-        RTMP_TLS_Init();
-#if defined(USE_MBEDTLS)
-    tls_server_ctx *tc = ctx = calloc(1, sizeof(struct tls_server_ctx));
-    tc->conf = &RTMP_TLS_ctx->conf;
-    tc->ctr_drbg = &RTMP_TLS_ctx->ctr_drbg;
+    if (!r->RTMP_TLS_ctx)
+        return;
+    mbedtls_ssl_config_free(&r->RTMP_TLS_ctx->conf);
+    mbedtls_ctr_drbg_free(&r->RTMP_TLS_ctx->ctr_drbg);
+    mbedtls_entropy_free(&r->RTMP_TLS_ctx->entropy);
 
-    mbedtls_x509_crt_init(&tc->cert);
-    if (mbedtls_x509_crt_parse_file(&tc->cert, cert))
-    {
-        free(tc);
-        return NULL;
+    if (r->RTMP_TLS_ctx->cacert) {
+        mbedtls_x509_crt_free(r->RTMP_TLS_ctx->cacert);
+        free(r->RTMP_TLS_ctx->cacert);
+        r->RTMP_TLS_ctx->cacert = NULL;
     }
 
-    mbedtls_pk_init(&tc->key);
-    if (mbedtls_pk_parse_keyfile(&tc->key, key, NULL))
-    {
-        mbedtls_x509_crt_free(&tc->cert);
-        mbedtls_pk_free(&tc->key);
-        free(tc);
-        return NULL;
-    }
-#elif defined(USE_POLARSSL)
-    tls_server_ctx *tc = ctx = calloc(1, sizeof(struct tls_server_ctx));
-    tc->dhm_P = my_dhm_P;
-    tc->dhm_G = my_dhm_G;
-    tc->hs = &RTMP_TLS_ctx->hs;
-    if (x509parse_crtfile(&tc->cert, cert))
-    {
-        free(tc);
-        return NULL;
-    }
-    if (x509parse_keyfile(&tc->key, key, NULL))
-    {
-        x509_free(&tc->cert);
-        free(tc);
-        return NULL;
-    }
-#elif defined(USE_GNUTLS) && !defined(NO_SSL)
-    gnutls_certificate_allocate_credentials((gnutls_certificate_credentials*) &ctx);
-    if (gnutls_certificate_set_x509_key_file(ctx, cert, key, GNUTLS_X509_FMT_PEM) != 0)
-    {
-        gnutls_certificate_free_credentials(ctx);
-        return NULL;
-    }
-#elif !defined(NO_SSL) /* USE_OPENSSL */
-    ctx = SSL_CTX_new(SSLv23_server_method());
-    if (!SSL_CTX_use_certificate_chain_file(ctx, cert))
-    {
-        SSL_CTX_free(ctx);
-        return NULL;
-    }
-    if (!SSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM))
-    {
-        SSL_CTX_free(ctx);
-        return NULL;
-    }
-#endif
-#else
-    (void)cert;
-    (void)key;
-#endif
-    return ctx;
-}
-
-void
-RTMP_TLS_FreeServerContext(void *ctx)
-{
-#ifdef CRYPTO
-#if defined(USE_MBEDTLS)
-    mbedtls_x509_crt_free(&((tls_server_ctx*)ctx)->cert);
-    mbedtls_pk_free(&((tls_server_ctx*)ctx)->key);
-    free(ctx);
-#elif defined(USE_POLARSSL)
-    x509_free(&((tls_server_ctx*)ctx)->cert);
-    rsa_free(&((tls_server_ctx*)ctx)->key);
-    free(ctx);
-#elif defined(USE_GNUTLS) && !defined(NO_SSL)
-    gnutls_certificate_free_credentials(ctx);
-#elif !defined(NO_SSL) /* USE_OPENSSL */
-    SSL_CTX_free(ctx);
-#endif
-
-#else
-    (void)ctx;
+    // NO mbedtls_net_free() BECAUSE WE SET IT UP BY HAND!
+    free(r->RTMP_TLS_ctx);
+    r->RTMP_TLS_ctx = NULL;
 #endif
 }
 
@@ -528,8 +442,7 @@ void
 RTMP_Free(RTMP *r)
 {
 #if defined(CRYPTO) && defined(USE_MBEDTLS)
-  if (RTMP_TLS_ctx)
-    RTMP_TLS_Free();
+    RTMP_TLS_Free(r);
 #endif
     free(r);
 }
@@ -537,11 +450,6 @@ RTMP_Free(RTMP *r)
 void
 RTMP_Init(RTMP *r)
 {
-#ifdef CRYPTO
-    if (!RTMP_TLS_ctx)
-        RTMP_TLS_Init();
-#endif
-
     memset(r, 0, sizeof(RTMP));
     r->m_sb.sb_socket = -1;
     r->m_inChunkSize = RTMP_DEFAULT_CHUNKSIZE;
@@ -557,6 +465,11 @@ RTMP_Init(RTMP *r)
     r->Link.nStreams = 0;
     r->Link.timeout = 30;
     r->Link.swfAge = 30;
+
+#ifdef CRYPTO
+    RTMP_TLS_Init(r);
+#endif
+
 }
 
 void
@@ -780,8 +693,12 @@ int RTMP_SetupURL(RTMP *r, char *url)
 
 #ifdef CRYPTO
     if ((r->Link.lFlags & RTMP_LF_SWFV) && r->Link.swfUrl.av_len)
+#ifdef USE_HASHSWF
         RTMP_HashSWF(r->Link.swfUrl.av_val, &r->Link.SWFSize,
         (unsigned char *)r->Link.SWFHash, r->Link.swfAge);
+#else
+        return FALSE;
+#endif
 #endif
 
     SocksSetup(r, &r->Link.sockshost);
@@ -1008,46 +925,16 @@ RTMP_Connect0(RTMP *r, struct sockaddr * service, socklen_t addrlen)
     return TRUE;
 }
 
-int
-RTMP_TLS_Accept(RTMP *r, void *ctx)
-{
-#if defined(CRYPTO) && !defined(NO_SSL)
-    tls_server_ctx *srv_ctx = ctx;
-    TLS_server(srv_ctx, r->m_sb.sb_ssl);
-
-#if defined(USE_MBEDTLS)
-    mbedtls_net_context *client_fd = &RTMP_TLS_ctx->net;
-    mbedtls_net_init(client_fd);
-    client_fd->fd = r->m_sb.sb_socket;
-    TLS_setfd(r->m_sb.sb_ssl, client_fd);
-#else
-    TLS_setfd(r->m_sb.sb_ssl, r->m_sb.sb_socket);
-#endif
-
-    int connect_return = TLS_connect(r->m_sb.sb_ssl);
-    if (connect_return < 0)
-    {
-        RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed", __FUNCTION__);
-        return FALSE;
-    }
-    return TRUE;
-#else
-    (void)r;
-    (void)ctx;
-    return FALSE;
-#endif
-}
-
 int
 RTMP_Connect1(RTMP *r, RTMPPacket *cp)
 {
     if (r->Link.protocol & RTMP_FEATURE_SSL)
     {
 #if defined(CRYPTO) && !defined(NO_SSL)
-        TLS_client(RTMP_TLS_ctx, r->m_sb.sb_ssl);
+        TLS_client(r->RTMP_TLS_ctx, r->m_sb.sb_ssl);
 
 #if defined(USE_MBEDTLS)
-        mbedtls_net_context *server_fd = &RTMP_TLS_ctx->net;
+        mbedtls_net_context *server_fd = &r->RTMP_TLS_ctx->net;
         server_fd->fd = r->m_sb.sb_socket;
         TLS_setfd(r->m_sb.sb_ssl, server_fd);
 
@@ -1070,10 +957,9 @@ RTMP_Connect1(RTMP *r, RTMPPacket *cp)
         if (connect_return < 0)
         {
 #if defined(USE_MBEDTLS)
+            r->last_error_code = connect_return;
             if (connect_return == MBEDTLS_ERR_X509_CERT_VERIFY_FAILED)
             {
-                r->last_error_code = connect_return;
-
                 // show a more detailed error in the log if possible
                 int verify_result = mbedtls_ssl_get_verify_result(r->m_sb.sb_ssl);
                 if (verify_result)
@@ -2696,9 +2582,16 @@ b64enc(const unsigned char *input, int length, char *output, int maxsize)
 
 #if defined(USE_MBEDTLS)
 typedef	mbedtls_md5_context MD5_CTX;
+
+#if MBEDTLS_VERSION_NUMBER >= 0x02040000
 #define MD5_Init(ctx)	mbedtls_md5_init(ctx); mbedtls_md5_starts_ret(ctx)
 #define MD5_Update(ctx,data,len)	mbedtls_md5_update_ret(ctx,(unsigned char *)data,len)
 #define MD5_Final(dig,ctx)	mbedtls_md5_finish_ret(ctx,dig); mbedtls_md5_free(ctx)
+#else
+#define MD5_Init(ctx)	mbedtls_md5_init(ctx); mbedtls_md5_starts(ctx)
+#define MD5_Update(ctx,data,len)	mbedtls_md5_update(ctx,(unsigned char *)data,len)
+#define MD5_Final(dig,ctx)	mbedtls_md5_finish(ctx,dig); mbedtls_md5_free(ctx)
+#endif
 
 #elif defined(USE_POLARSSL)
 #define MD5_CTX	md5_context
@@ -4115,69 +4008,6 @@ HandShake(RTMP *r, int FP9HandShake)
     (void)FP9HandShake;
     return TRUE;
 }
-
-static int
-SHandShake(RTMP *r)
-{
-    int i;
-    char serverbuf[RTMP_SIG_SIZE + 1], *serversig = serverbuf + 1;
-    char clientsig[RTMP_SIG_SIZE];
-    uint32_t uptime;
-    int bMatch;
-
-    if (ReadN(r, serverbuf, 1) != 1)	/* 0x03 or 0x06 */
-        return FALSE;
-
-    RTMP_Log(RTMP_LOGDEBUG, "%s: Type Request  : %02X", __FUNCTION__, serverbuf[0]);
-
-    if (serverbuf[0] != 3)
-    {
-        RTMP_Log(RTMP_LOGERROR, "%s: Type unknown: client sent %02X",
-                 __FUNCTION__, serverbuf[0]);
-        return FALSE;
-    }
-
-    uptime = htonl(RTMP_GetTime());
-    memcpy(serversig, &uptime, 4);
-
-    memset(&serversig[4], 0, 4);
-#ifdef _DEBUG
-    for (i = 8; i < RTMP_SIG_SIZE; i++)
-        serversig[i] = 0xff;
-#else
-    for (i = 8; i < RTMP_SIG_SIZE; i++)
-        serversig[i] = (char)(rand() % 256);
-#endif
-
-    if (!WriteN(r, serverbuf, RTMP_SIG_SIZE + 1))
-        return FALSE;
-
-    if (ReadN(r, clientsig, RTMP_SIG_SIZE) != RTMP_SIG_SIZE)
-        return FALSE;
-
-    /* decode client response */
-
-    memcpy(&uptime, clientsig, 4);
-    uptime = ntohl(uptime);
-
-    RTMP_Log(RTMP_LOGDEBUG, "%s: Client Uptime : %d", __FUNCTION__, uptime);
-    RTMP_Log(RTMP_LOGDEBUG, "%s: Player Version: %d.%d.%d.%d", __FUNCTION__,
-             clientsig[4], clientsig[5], clientsig[6], clientsig[7]);
-
-    /* 2nd part of handshake */
-    if (!WriteN(r, clientsig, RTMP_SIG_SIZE))
-        return FALSE;
-
-    if (ReadN(r, clientsig, RTMP_SIG_SIZE) != RTMP_SIG_SIZE)
-        return FALSE;
-
-    bMatch = (memcmp(serversig, clientsig, RTMP_SIG_SIZE) == 0);
-    if (!bMatch)
-    {
-        RTMP_Log(RTMP_LOGWARNING, "%s, client signature does not match!", __FUNCTION__);
-    }
-    return TRUE;
-}
 #endif
 
 int
@@ -4421,12 +4251,6 @@ RTMP_SendPacket(RTMP *r, RTMPPacket *packet, int queue)
     return TRUE;
 }
 
-int
-RTMP_Serve(RTMP *r)
-{
-    return SHandShake(r);
-}
-
 void
 RTMP_Close(RTMP *r)
 {

+ 133 - 7
plugins/obs-outputs/librtmp/rtmp.h

@@ -49,6 +49,132 @@
 
 #include "amf.h"
 
+#ifdef CRYPTO
+#if defined(USE_MBEDTLS)
+#include <mbedtls/version.h>
+
+#if MBEDTLS_VERSION_NUMBER >= 0x02040000
+#include <mbedtls/net_sockets.h>
+#else
+#include <mbedtls/net.h>
+#endif
+
+#include <mbedtls/ssl.h>
+#include <mbedtls/ctr_drbg.h>
+#include <mbedtls/entropy.h>
+
+#define my_dhm_P \
+    "E4004C1F94182000103D883A448B3F80" \
+    "2CE4B44A83301270002C20D0321CFD00" \
+    "11CCEF784C26A400F43DFB901BCA7538" \
+    "F2C6B176001CF5A0FD16D2C48B1D0C1C" \
+    "F6AC8E1DA6BCC3B4E1F96B0564965300" \
+    "FFA1D0B601EB2800F489AA512C4B248C" \
+    "01F76949A60BB7F00A40B1EAB64BDD48" \
+    "E8A700D60B7F1200FA8E77B0A979DABF"
+
+#define my_dhm_G "4"
+
+#define	SSL_SET_SESSION(S,resume,timeout,ctx)	mbedtls_ssl_set_session(S,ctx)
+
+typedef struct tls_ctx
+{
+    mbedtls_entropy_context entropy;
+    mbedtls_ctr_drbg_context ctr_drbg;
+    mbedtls_ssl_config conf;
+    mbedtls_ssl_session ssn;
+    mbedtls_x509_crt *cacert;
+    mbedtls_net_context net;
+} tls_ctx;
+
+typedef tls_ctx *TLS_CTX;
+
+#define TLS_client(ctx,s)	\
+  s = malloc(sizeof(mbedtls_ssl_context));\
+  mbedtls_ssl_init(s);\
+  mbedtls_ssl_setup(s, &ctx->conf);\
+	mbedtls_ssl_config_defaults(&ctx->conf, MBEDTLS_SSL_IS_CLIENT, MBEDTLS_SSL_TRANSPORT_STREAM, MBEDTLS_SSL_PRESET_DEFAULT);\
+  mbedtls_ssl_conf_authmode(&ctx->conf, MBEDTLS_SSL_VERIFY_REQUIRED);\
+	mbedtls_ssl_conf_rng(&ctx->conf, mbedtls_ctr_drbg_random, &ctx->ctr_drbg)
+
+#define TLS_setfd(s,fd)	mbedtls_ssl_set_bio(s, fd, mbedtls_net_send, mbedtls_net_recv, NULL)
+#define TLS_connect(s)	mbedtls_ssl_handshake(s)
+#define TLS_accept(s)	mbedtls_ssl_handshake(s)
+#define TLS_read(s,b,l)	mbedtls_ssl_read(s,(unsigned char *)b,l)
+#define TLS_write(s,b,l)	mbedtls_ssl_write(s,(unsigned char *)b,l)
+#define TLS_shutdown(s)	mbedtls_ssl_close_notify(s)
+#define TLS_close(s)	mbedtls_ssl_free(s); free(s)
+
+#elif defined(USE_POLARSSL)
+#include <polarssl/version.h>
+#include <polarssl/net.h>
+#include <polarssl/ssl.h>
+#include <polarssl/havege.h>
+#if POLARSSL_VERSION_NUMBER < 0x01010000
+#define havege_random	havege_rand
+#endif
+#if POLARSSL_VERSION_NUMBER >= 0x01020000
+#define	SSL_SET_SESSION(S,resume,timeout,ctx)	ssl_set_session(S,ctx)
+#else
+#define	SSL_SET_SESSION(S,resume,timeout,ctx)	ssl_set_session(S,resume,timeout,ctx)
+#endif
+typedef struct tls_ctx
+{
+    havege_state hs;
+    ssl_session ssn;
+} tls_ctx;
+
+#define TLS_CTX tls_ctx *
+#define TLS_client(ctx,s)	s = malloc(sizeof(ssl_context)); ssl_init(s);\
+	ssl_set_endpoint(s, SSL_IS_CLIENT); ssl_set_authmode(s, SSL_VERIFY_NONE);\
+	ssl_set_rng(s, havege_random, &ctx->hs);\
+	ssl_set_ciphersuites(s, ssl_default_ciphersuites);\
+	SSL_SET_SESSION(s, 1, 600, &ctx->ssn)
+#define TLS_setfd(s,fd)	ssl_set_bio(s, net_recv, &fd, net_send, &fd)
+#define TLS_connect(s)	ssl_handshake(s)
+#define TLS_accept(s)	ssl_handshake(s)
+#define TLS_read(s,b,l)	ssl_read(s,(unsigned char *)b,l)
+#define TLS_write(s,b,l)	ssl_write(s,(unsigned char *)b,l)
+#define TLS_shutdown(s)	ssl_close_notify(s)
+#define TLS_close(s)	ssl_free(s); free(s)
+
+
+#elif defined(USE_GNUTLS)
+#include <gnutls/gnutls.h>
+typedef struct tls_ctx
+{
+    gnutls_certificate_credentials_t cred;
+    gnutls_priority_t prios;
+} tls_ctx;
+#define TLS_CTX	tls_ctx *
+#define TLS_client(ctx,s)	gnutls_init((gnutls_session_t *)(&s), GNUTLS_CLIENT); gnutls_priority_set(s, ctx->prios); gnutls_credentials_set(s, GNUTLS_CRD_CERTIFICATE, ctx->cred)
+#define TLS_setfd(s,fd)	gnutls_transport_set_ptr(s, (gnutls_transport_ptr_t)(long)fd)
+#define TLS_connect(s)	gnutls_handshake(s)
+#define TLS_accept(s)	gnutls_handshake(s)
+#define TLS_read(s,b,l)	gnutls_record_recv(s,b,l)
+#define TLS_write(s,b,l)	gnutls_record_send(s,b,l)
+#define TLS_shutdown(s)	gnutls_bye(s, GNUTLS_SHUT_RDWR)
+#define TLS_close(s)	gnutls_deinit(s)
+
+#elif defined(USE_ONLY_MD5)
+#include "md5.h"
+#include "cencode.h"
+#define MD5_DIGEST_LENGTH 16
+
+#else	/* USE_OPENSSL */
+#define TLS_CTX	SSL_CTX *
+#define TLS_client(ctx,s)	s = SSL_new(ctx)
+#define TLS_setfd(s,fd)	SSL_set_fd(s,fd)
+#define TLS_connect(s)	SSL_connect(s)
+#define TLS_accept(s)	SSL_accept(s)
+#define TLS_read(s,b,l)	SSL_read(s,b,l)
+#define TLS_write(s,b,l)	SSL_write(s,b,l)
+#define TLS_shutdown(s)	SSL_shutdown(s)
+#define TLS_close(s)	SSL_free(s)
+
+#endif
+#endif
+
 #ifdef __cplusplus
 extern "C"
 {
@@ -326,6 +452,10 @@ extern "C"
         RTMP_LNK Link;
         int connect_time_ms;
         int last_error_code;
+
+#ifdef CRYPTO
+        TLS_CTX RTMP_TLS_ctx;
+#endif
     } RTMP;
 
     int RTMP_ParseURL(const char *url, int *protocol, AVal *host,
@@ -360,8 +490,6 @@ extern "C"
     struct sockaddr;
     int RTMP_Connect0(RTMP *r, struct sockaddr *svc, socklen_t addrlen);
     int RTMP_Connect1(RTMP *r, RTMPPacket *cp);
-    int RTMP_Serve(RTMP *r);
-    int RTMP_TLS_Accept(RTMP *r, void *ctx);
 
     int RTMP_ReadPacket(RTMP *r, RTMPPacket *packet);
     int RTMP_SendPacket(RTMP *r, RTMPPacket *packet, int queue);
@@ -381,13 +509,10 @@ extern "C"
     void RTMP_Init(RTMP *r);
     void RTMP_Close(RTMP *r);
     RTMP *RTMP_Alloc(void);
-    void RTMP_TLS_Free();
+    void RTMP_TLS_Free(RTMP *r);
     void RTMP_Free(RTMP *r);
     void RTMP_EnableWrite(RTMP *r);
 
-    void *RTMP_TLS_AllocServerContext(const char* cert, const char* key);
-    void RTMP_TLS_FreeServerContext(void *ctx);
-
     int RTMP_LibVersion(void);
     void RTMP_UserInterrupt(void);	/* user typed Ctrl-C */
 
@@ -415,10 +540,11 @@ extern "C"
     int RTMP_Read(RTMP *r, char *buf, int size);
     int RTMP_Write(RTMP *r, const char *buf, int size, int streamIdx);
 
+#ifdef USE_HASHSWF
     /* hashswf.c */
     int RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash,
                      int age);
-
+#endif
 #ifdef __cplusplus
 };
 #endif

+ 0 - 155
plugins/obs-outputs/librtmp/rtmp_sys.h

@@ -79,159 +79,4 @@
 #endif
 
 #include "rtmp.h"
-
-#if defined(USE_MBEDTLS)
-#include <mbedtls/version.h>
-#include <mbedtls/net_sockets.h>
-#include <mbedtls/ssl.h>
-#include <mbedtls/ctr_drbg.h>
-#include <mbedtls/entropy.h>
-
-#define my_dhm_P \
-    "E4004C1F94182000103D883A448B3F80" \
-    "2CE4B44A83301270002C20D0321CFD00" \
-    "11CCEF784C26A400F43DFB901BCA7538" \
-    "F2C6B176001CF5A0FD16D2C48B1D0C1C" \
-    "F6AC8E1DA6BCC3B4E1F96B0564965300" \
-    "FFA1D0B601EB2800F489AA512C4B248C" \
-    "01F76949A60BB7F00A40B1EAB64BDD48" \
-    "E8A700D60B7F1200FA8E77B0A979DABF"
-
-#define my_dhm_G "4"
-
-#define	SSL_SET_SESSION(S,resume,timeout,ctx)	mbedtls_ssl_set_session(S,ctx)
-
-typedef struct tls_ctx
-{
-    mbedtls_entropy_context entropy;
-    mbedtls_ctr_drbg_context ctr_drbg;
-    mbedtls_ssl_config conf;
-    mbedtls_ssl_session ssn;
-    mbedtls_x509_crt *cacert;
-    mbedtls_net_context net;
-} tls_ctx;
-
-typedef struct tls_server_ctx
-{
-  mbedtls_ssl_config *conf;
-  mbedtls_ctr_drbg_context *ctr_drbg;
-  mbedtls_pk_context key;
-  mbedtls_x509_crt cert;
-} tls_server_ctx;
-
-typedef tls_ctx *TLS_CTX;
-
-#define TLS_client(ctx,s)	\
-  s = malloc(sizeof(mbedtls_ssl_context));\
-  mbedtls_ssl_init(s);\
-  mbedtls_ssl_setup(s, &ctx->conf);\
-	mbedtls_ssl_config_defaults(&ctx->conf, MBEDTLS_SSL_IS_CLIENT, MBEDTLS_SSL_TRANSPORT_STREAM, MBEDTLS_SSL_PRESET_DEFAULT);\
-  mbedtls_ssl_conf_authmode(&ctx->conf, MBEDTLS_SSL_VERIFY_REQUIRED);\
-	mbedtls_ssl_conf_rng(&ctx->conf, mbedtls_ctr_drbg_random, &ctx->ctr_drbg)
-
-#define TLS_server(ctx,s)\
-  s = malloc(sizeof(mbedtls_ssl_context));\
-  mbedtls_ssl_init(s);\
-  mbedtls_ssl_setup(s, ctx->conf);\
-	mbedtls_ssl_conf_endpoint(ctx->conf, MBEDTLS_SSL_IS_SERVER);\
-  mbedtls_ssl_conf_authmode(ctx->conf, MBEDTLS_SSL_VERIFY_REQUIRED);\
-	mbedtls_ssl_conf_rng(ctx->conf, mbedtls_ctr_drbg_random, ctx->ctr_drbg);\
-	mbedtls_ssl_conf_own_cert(ctx->conf, &ctx->cert, &ctx->key);\
-	mbedtls_ssl_conf_dh_param_bin(ctx->conf,\
-    (const unsigned char *)my_dhm_P, strlen(my_dhm_P),\
-    (const unsigned char *)my_dhm_G, strlen(my_dhm_G))
-
-#define TLS_setfd(s,fd)	mbedtls_ssl_set_bio(s, fd, mbedtls_net_send, mbedtls_net_recv, NULL)
-#define TLS_connect(s)	mbedtls_ssl_handshake(s)
-#define TLS_accept(s)	mbedtls_ssl_handshake(s)
-#define TLS_read(s,b,l)	mbedtls_ssl_read(s,(unsigned char *)b,l)
-#define TLS_write(s,b,l)	mbedtls_ssl_write(s,(unsigned char *)b,l)
-#define TLS_shutdown(s)	mbedtls_ssl_close_notify(s)
-#define TLS_close(s)	mbedtls_ssl_free(s); free(s)
-
-#elif defined(USE_POLARSSL)
-#include <polarssl/version.h>
-#include <polarssl/net.h>
-#include <polarssl/ssl.h>
-#include <polarssl/havege.h>
-#if POLARSSL_VERSION_NUMBER < 0x01010000
-#define havege_random	havege_rand
-#endif
-#if POLARSSL_VERSION_NUMBER >= 0x01020000
-#define	SSL_SET_SESSION(S,resume,timeout,ctx)	ssl_set_session(S,ctx)
-#else
-#define	SSL_SET_SESSION(S,resume,timeout,ctx)	ssl_set_session(S,resume,timeout,ctx)
-#endif
-typedef struct tls_ctx
-{
-    havege_state hs;
-    ssl_session ssn;
-} tls_ctx;
-typedef struct tls_server_ctx
-{
-    havege_state *hs;
-    x509_cert cert;
-    rsa_context key;
-    ssl_session ssn;
-    const char *dhm_P, *dhm_G;
-} tls_server_ctx;
-
-#define TLS_CTX tls_ctx *
-#define TLS_client(ctx,s)	s = malloc(sizeof(ssl_context)); ssl_init(s);\
-	ssl_set_endpoint(s, SSL_IS_CLIENT); ssl_set_authmode(s, SSL_VERIFY_NONE);\
-	ssl_set_rng(s, havege_random, &ctx->hs);\
-	ssl_set_ciphersuites(s, ssl_default_ciphersuites);\
-	SSL_SET_SESSION(s, 1, 600, &ctx->ssn)
-#define TLS_server(ctx,s)	s = malloc(sizeof(ssl_context)); ssl_init(s);\
-	ssl_set_endpoint(s, SSL_IS_SERVER); ssl_set_authmode(s, SSL_VERIFY_NONE);\
-	ssl_set_rng(s, havege_random, ((tls_server_ctx*)ctx)->hs);\
-	ssl_set_ciphersuites(s, ssl_default_ciphersuites);\
-	SSL_SET_SESSION(s, 1, 600, &((tls_server_ctx*)ctx)->ssn);\
-	ssl_set_own_cert(s, &((tls_server_ctx*)ctx)->cert, &((tls_server_ctx*)ctx)->key);\
-	ssl_set_dh_param(s, ((tls_server_ctx*)ctx)->dhm_P, ((tls_server_ctx*)ctx)->dhm_G)
-#define TLS_setfd(s,fd)	ssl_set_bio(s, net_recv, &fd, net_send, &fd)
-#define TLS_connect(s)	ssl_handshake(s)
-#define TLS_accept(s)	ssl_handshake(s)
-#define TLS_read(s,b,l)	ssl_read(s,(unsigned char *)b,l)
-#define TLS_write(s,b,l)	ssl_write(s,(unsigned char *)b,l)
-#define TLS_shutdown(s)	ssl_close_notify(s)
-#define TLS_close(s)	ssl_free(s); free(s)
-
-
-#elif defined(USE_GNUTLS)
-#include <gnutls/gnutls.h>
-typedef struct tls_ctx
-{
-    gnutls_certificate_credentials_t cred;
-    gnutls_priority_t prios;
-} tls_ctx;
-#define TLS_CTX	tls_ctx *
-#define TLS_client(ctx,s)	gnutls_init((gnutls_session_t *)(&s), GNUTLS_CLIENT); gnutls_priority_set(s, ctx->prios); gnutls_credentials_set(s, GNUTLS_CRD_CERTIFICATE, ctx->cred)
-#define TLS_server(ctx,s)	gnutls_init((gnutls_session_t *)(&s), GNUTLS_SERVER); gnutls_priority_set_direct(s, "NORMAL", NULL); gnutls_credentials_set(s, GNUTLS_CRD_CERTIFICATE, ctx)
-#define TLS_setfd(s,fd)	gnutls_transport_set_ptr(s, (gnutls_transport_ptr_t)(long)fd)
-#define TLS_connect(s)	gnutls_handshake(s)
-#define TLS_accept(s)	gnutls_handshake(s)
-#define TLS_read(s,b,l)	gnutls_record_recv(s,b,l)
-#define TLS_write(s,b,l)	gnutls_record_send(s,b,l)
-#define TLS_shutdown(s)	gnutls_bye(s, GNUTLS_SHUT_RDWR)
-#define TLS_close(s)	gnutls_deinit(s)
-
-#elif defined(USE_ONLY_MD5)
-#include "md5.h"
-#include "cencode.h"
-#define MD5_DIGEST_LENGTH 16
-
-#else	/* USE_OPENSSL */
-#define TLS_CTX	SSL_CTX *
-#define TLS_client(ctx,s)	s = SSL_new(ctx)
-#define TLS_server(ctx,s)	s = SSL_new(ctx)
-#define TLS_setfd(s,fd)	SSL_set_fd(s,fd)
-#define TLS_connect(s)	SSL_connect(s)
-#define TLS_accept(s)	SSL_accept(s)
-#define TLS_read(s,b,l)	SSL_read(s,b,l)
-#define TLS_write(s,b,l)	SSL_write(s,b,l)
-#define TLS_shutdown(s)	SSL_shutdown(s)
-#define TLS_close(s)	SSL_free(s)
-
-#endif
 #endif

+ 13 - 4
plugins/obs-outputs/rtmp-stream.c

@@ -110,7 +110,7 @@ static void rtmp_stream_destroy(void *data)
 		}
 	}
 
-	RTMP_TLS_Free();
+	RTMP_TLS_Free(&stream->rtmp);
 	free_packets(stream);
 	dstr_free(&stream->path);
 	dstr_free(&stream->key);
@@ -145,8 +145,8 @@ static void *rtmp_stream_create(obs_data_t *settings, obs_output_t *output)
 	stream->output = output;
 	pthread_mutex_init_value(&stream->packets_mutex);
 
-	RTMP_Init(&stream->rtmp);
 	RTMP_LogSetCallback(log_rtmp);
+	RTMP_Init(&stream->rtmp);
 	RTMP_LogSetLevel(RTMP_LOGWARNING);
 
 	if (pthread_mutex_init(&stream->packets_mutex, NULL) != 0)
@@ -514,10 +514,18 @@ static void set_output_error(struct rtmp_stream *stream)
 		case -0x2700:
 			msg = obs_module_text("SSLCertVerifyFailed");
 			break;
+		case -0x7680:
+			msg = "Failed to load root certificates for a secure TLS connection."
+#if defined(__linux__)
+			      " Check you have an up to date root certificate bundle in /etc/ssl/certs."
+#endif
+				;
+			break;
 		}
 	}
 
-	obs_output_set_last_error(stream->output, msg);
+	if (msg)
+		obs_output_set_last_error(stream->output, msg);
 }
 
 static void dbr_add_frame(struct rtmp_stream *stream, struct dbr_frame *back)
@@ -934,7 +942,8 @@ static int try_connect(struct rtmp_stream *stream)
 
 	info("Connecting to RTMP URL %s...", stream->path.array);
 
-	RTMP_Init(&stream->rtmp);
+	// this should have been called already by rtmp_stream_create
+	//RTMP_Init(&stream->rtmp);
 	if (!RTMP_SetupURL(&stream->rtmp, stream->path.array))
 		return OBS_OUTPUT_BAD_PATH;