obs-outputs: Update librtmp with upstream patches

Mostly security / reliability related, fixing various access to
uninitialized data, integer overflows, etc from
https://git.ffmpeg.org/rtmpdump

plugins/obs-outputs/librtmp/amf.c

@@ -29,6 +29,7 @@
 #include "bytes.h"
 
 static const AMFObjectProperty AMFProp_Invalid = { {0, 0}, AMF_INVALID };
+static const AMFObject AMFObj_Invalid = { 0, 0 };
 static const AVal AV_empty = { 0, 0 };
 
 /* Data is Big-Endian */
@@ -336,13 +337,19 @@ AMFProp_GetBoolean(AMFObjectProperty *prop)
 void
 AMFProp_GetString(AMFObjectProperty *prop, AVal *str)
 {
-    *str = prop->p_vu.p_aval;
+    if (prop->p_type == AMF_STRING)
+        *str = prop->p_vu.p_aval;
+    else
+        *str = AV_empty;
 }
 
 void
 AMFProp_GetObject(AMFObjectProperty *prop, AMFObject *obj)
 {
-    *obj = prop->p_vu.p_object;
+    if (prop->p_type == AMF_OBJECT)
+        *obj = prop->p_vu.p_object;
+    else
+        *obj = AMFObj_Invalid;
 }
 
 int
@@ -472,6 +479,8 @@ AMF3ReadString(const char *data, AVal *str)
         RTMP_Log(RTMP_LOGDEBUG,
                  "%s, string reference, index: %d, not supported, ignoring!",
                  __FUNCTION__, refIndex);
+        str->av_val = NULL;
+        str->av_len = 0;
         return len;
     }
     else
@@ -511,9 +520,12 @@ AMF3Prop_Decode(AMFObjectProperty *prop, const char *pBuffer, int nSize,
         if (name.av_len <= 0)
             return nRes;
 
+        nSize -= nRes;
+        if (nSize <= 0)
+            return -1;
+
         prop->p_name = name;
         pBuffer += nRes;
-        nSize -= nRes;
     }
 
     /* decode */
@@ -601,6 +613,9 @@ AMF3Prop_Decode(AMFObjectProperty *prop, const char *pBuffer, int nSize,
         return -1;
     }
 
+    if (nSize < 0)
+        return -1;
+
     return nOriginalSize - nSize;
 }
 
@@ -994,9 +1009,18 @@ AMF_DecodeArray(AMFObject *obj, const char *pBuffer, int nSize,
         int nRes;
         nArrayLen--;
 
+        if (nSize <= 0)
+        {
+            bError = TRUE;
+            break;
+        }
+
         nRes = AMFProp_Decode(&prop, pBuffer, nSize, bDecodeName);
         if (nRes == -1)
+        {
             bError = TRUE;
+            break;
+        }
         else
         {
             nSize -= nRes;
@@ -1057,12 +1081,12 @@ AMF3_Decode(AMFObject *obj, const char *pBuffer, int nSize, int bAMFData)
         else
         {
             int32_t classExtRef = (classRef >> 1);
-            int i;
+            int i, cdnum;
 
             cd.cd_externalizable = (classExtRef & 0x1) == 1;
             cd.cd_dynamic = ((classExtRef >> 1) & 0x1) == 1;
 
-            cd.cd_num = classExtRef >> 2;
+            cdnum = classExtRef >> 2;
 
             /* class name */
 
@@ -1077,9 +1101,16 @@ AMF3_Decode(AMFObject *obj, const char *pBuffer, int nSize, int bAMFData)
                      cd.cd_name.av_val, cd.cd_externalizable, cd.cd_dynamic,
                      cd.cd_num);
 
-            for (i = 0; i < cd.cd_num; i++)
+            for (i = 0; i < cdnum; i++)
             {
                 AVal memberName = AV_empty;
+                if (nSize <= 0)
+                {
+                invalid:
+                    RTMP_Log(RTMP_LOGDEBUG, "%s, invalid class encoding!",
+                        __FUNCTION__);
+                    return nOriginalSize;
+                }
                 len = AMF3ReadString(pBuffer, &memberName);
                 RTMP_Log(RTMP_LOGDEBUG, "Member: %s", memberName.av_val);
                 AMF3CD_AddProp(&cd, &memberName);
@@ -1115,6 +1146,8 @@ AMF3_Decode(AMFObject *obj, const char *pBuffer, int nSize, int bAMFData)
             int nRes, i;
             for (i = 0; i < cd.cd_num; i++)	/* non-dynamic */
             {
+                if (nSize <= 0)
+                    goto invalid;
                 nRes = AMF3Prop_Decode(&prop, pBuffer, nSize, FALSE);
                 if (nRes == -1)
                     RTMP_Log(RTMP_LOGDEBUG, "%s, failed to decode AMF3 property!",
@@ -1132,6 +1165,8 @@ AMF3_Decode(AMFObject *obj, const char *pBuffer, int nSize, int bAMFData)
 
                 do
                 {
+                    if (nSize <= 0)
+                        goto invalid;
                     nRes = AMF3Prop_Decode(&prop, pBuffer, nSize, TRUE);
                     AMF_AddProp(obj, &prop);
 
@@ -1179,10 +1214,18 @@ AMF_Decode(AMFObject *obj, const char *pBuffer, int nSize, int bDecodeName)
 
         nRes = AMFProp_Decode(&prop, pBuffer, nSize, bDecodeName);
         if (nRes == -1)
+        {
             bError = TRUE;
+            break;
+        }
         else
         {
             nSize -= nRes;
+            if (nSize < 0)
+            {
+                bError = TRUE;
+                break;
+            }
             pBuffer += nRes;
             AMF_AddProp(obj, &prop);
         }

plugins/obs-outputs/librtmp/rtmp.c

@@ -226,9 +226,13 @@ RTMPPacket_Reset(RTMPPacket *p)
 }
 
 int
-RTMPPacket_Alloc(RTMPPacket *p, int nSize)
+RTMPPacket_Alloc(RTMPPacket *p, uint32_t nSize)
 {
-    char *ptr = calloc(1, nSize + RTMP_MAX_HEADER_SIZE);
+    char *ptr;
+    if (nSize > SIZE_MAX - RTMP_MAX_HEADER_SIZE)
+        return FALSE;
+
+    ptr = calloc(1, nSize + RTMP_MAX_HEADER_SIZE);
     if (!ptr)
         return FALSE;
     p->m_body = ptr + RTMP_MAX_HEADER_SIZE;
@@ -1108,7 +1112,7 @@ RTMP_GetNextMediaPacket(RTMP *r, RTMPPacket *packet)
     while (!bHasMediaPacket && RTMP_IsConnected(r)
             && RTMP_ReadPacket(r, packet))
     {
-        if (!RTMPPacket_IsReady(packet))
+        if (!RTMPPacket_IsReady(packet) || !packet->m_nBodySize)
         {
             continue;
         }
@@ -3713,7 +3717,6 @@ RTMP_ReadPacket(RTMP *r, RTMPPacket *packet)
         {
             packet->m_nBodySize = AMF_DecodeInt24(header + 3);
             packet->m_nBytesRead = 0;
-            RTMPPacket_Free(packet);
 
             if (nSize > 6)
             {

plugins/obs-outputs/librtmp/rtmp.h

@@ -150,7 +150,7 @@ extern "C"
 
     void RTMPPacket_Reset(RTMPPacket *p);
     void RTMPPacket_Dump(RTMPPacket *p);
-    int RTMPPacket_Alloc(RTMPPacket *p, int nSize);
+    int RTMPPacket_Alloc(RTMPPacket *p, uint32_t nSize);
     void RTMPPacket_Free(RTMPPacket *p);
 
 #define RTMPPacket_IsReady(a)	((a)->m_nBytesRead == (a)->m_nBodySize)