Memory Optimization - Rolling Buffer

CMakeLists.txt

@@ -386,6 +386,7 @@ endif()
 file(GLOB WEBRTC_SIGNALING_CLIENT_SOURCE_FILES "src/source/Signaling/*.c")
 
 
+message(STATUS "OPEN_SRC_INSTALL_PREFIX: ${OPEN_SRC_INSTALL_PREFIX}")
 include_directories(${OPEN_SRC_INCLUDE_DIRS})
 include_directories(${OPEN_SRC_INSTALL_PREFIX}/include)
 include_directories(${KINESIS_VIDEO_WEBRTC_CLIENT_SRC}/src/include)

README.md

@@ -369,6 +369,43 @@ If using the WebRTC SDK Test Page, set the following values using the same AWS c
 
 Then choose Start Viewer to start live video streaming of the sample H264/Opus frames.
 
+## Memory optimization switches
+
+Starting with v1.11.0, the SDK provides some knobs to optimize memory usage to tailor to platform needs and resources
+
+### Controlling RTP rolling buffer capacity
+
+The SDK maintains an RTP rolling buffer to hold the RTP packets. This is useful to respond to NACKs and even in case of JitterBuffer. The rolling buffer size is controlled by 3 parameters:
+1. MTU: This is set to a default of 1200 bytes
+2. Buffer duration: This is the amount of time of media that you would like the rolling buffer to accommodate before it is overwritten due to buffer overflow. By default, the SDK sets this to 3 seconds
+3. Highest expected bitrate: This is the expected bitrate of the media in question. The typical bitrates could vary based on resolution and codec. By default, the SDK sets this to 5 mibps for video and 1 mibps for audio
+
+The rolling buffer capacity is calculated as follows:
+```
+Capacity = Buffer duration * highest expected bitrate (in bips) / 8 / MTU
+
+With buffer duration = 1 second,  Highest expected bitrate = 5 mibps and MTU 1200 bytes, capacity = 546 RTP packets
+```
+
+The rolling buffer size can be configured per transceiver using the `configureTransceiverRollingBuffer` API. Make sure to use the API after the addTransceiver call to ensure the `RtcMediaStreamTrack` and `KvsRtpTransceiver` objects are created. By default, the rolling buffer duration is set to 3 sec and bitrate is set to 5mibps for video and 1mibps for audio.
+
+The rolling buffer config parameters are as follows:
+```
+rollingBufferDurationSec = <duration in seconds>, must be more than 100ms and less than 10 seconds
+rollingBufferBitratebps = <bitrate in bits/sec>, must be more than 100kibits/sec and less than 240 mibps
+```
+
+For example, if we want to set duration to 200ms and bitrate to 150kibps,
+```c
+PRtcRtpTransceiver pVideoRtcRtpTransceiver;
+RtcMediaStreamTrack videoTrack;
+videoTrack.kind = MEDIA_STREAM_TRACK_KIND_VIDEO;
+videoTrack.codec = RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE;
+CHK_STATUS(configureTransceiverRollingBuffer(pVideoRtcRtpTransceiver, &videoTrack, 0.2, 150 * 1024));
+```
+By setting these up, applications can have better control over the amount of memory that the application consumes. However, note, if the allocation is too small and the network bad leading to multiple nacks, it can lead to choppy media / dropped frames. Hence, care must be taken while deciding on the values to ensure the parameters satisfy necessary performance requirements.
+For more information, check the sample to see how these values are set up.
+
 ## Setup IoT
 * To use IoT certificate to authenticate with KVS signaling, please refer to [Controlling Access to Kinesis Video Streams Resources Using AWS IoT](https://docs.aws.amazon.com/kinesisvideostreams/latest/dg/how-iot.html) for provisioning details.
 * A sample IAM policy for the IoT role looks like below, policy can be modified based on your permission requirement.

src/include/com/amazonaws/kinesis/video/webrtcclient/Include.h

@@ -1172,7 +1172,7 @@ typedef struct {
     UINT16 maximumTransmissionUnit; //!< Controls the size of the largest packet the WebRTC SDK will send
                                     //!< Some networks may drop packets if they exceed a certain size, and is useful in those conditions.
                                     //!< A smaller MTU will incur higher bandwidth usage however since more packets will be generated with
-                                    //!< smaller payloads. If unset DEFAULT_MTU_SIZE will be used
+                                    //!< smaller payloads. If unset DEFAULT_MTU_SIZE_BYTES will be used
 
     UINT32 iceLocalCandidateGatheringTimeout; //!< Maximum time ice will wait for gathering STUN and RELAY candidates. Once
                                               //!< it's reached, ice will proceed with whatever candidate it current has. Use default value if 0.
@@ -1610,6 +1610,19 @@ typedef struct {
  * @{
  */
 
+/**
+ * @brief Set up rolling buffer configuration - max duration of media to store (sec) and expected max bitrate (bips) of the encoded media
+ *
+ *
+ * @param[in] PRtcRtpTransceiver IN/Initialized and configured RtcRtpTransceiver
+ * @param[in] PRtcMediaStreamTrack IN/Initialized media stream track information
+ * @param[in] DOUBLE IN/Rolling buffer duration in seconds
+ * @param[in] DOUBLE IN/Rolling buffer bitrate in bits/second
+ *
+ * @return STATUS code of the execution. STATUS_SUCCESS on success
+ */
+STATUS configureTransceiverRollingBuffer(PRtcRtpTransceiver, PRtcMediaStreamTrack, DOUBLE, DOUBLE);
+
 /**
  * @brief Initialize a RtcPeerConnection with the provided Configuration
  *

src/source/Crypto/Dtls_mbedtls.c

@@ -31,7 +31,7 @@ STATUS createDtlsSession(PDtlsSessionCallbacks pDtlsSessionCallbacks, TIMER_QUEU
     mbedtls_ctr_drbg_set_prediction_resistance(&pDtlsSession->ctrDrbg, MBEDTLS_CTR_DRBG_PR_ON);
     CHK(mbedtls_ctr_drbg_seed(&pDtlsSession->ctrDrbg, mbedtls_entropy_func, &pDtlsSession->entropy, NULL, 0) == 0, STATUS_CREATE_SSL_FAILED);
 
-    CHK_STATUS(createIOBuffer(DEFAULT_MTU_SIZE, &pDtlsSession->pReadBuffer));
+    CHK_STATUS(createIOBuffer(DEFAULT_MTU_SIZE_BYTES, &pDtlsSession->pReadBuffer));
     pDtlsSession->timerQueueHandle = timerQueueHandle;
     pDtlsSession->timerId = MAX_UINT32;
     pDtlsSession->sslLock = MUTEX_CREATE(TRUE);
@@ -290,7 +290,7 @@ STATUS dtlsSessionStart(PDtlsSession pDtlsSession, BOOL isServer)
     mbedtls_ssl_conf_export_keys_ext_cb(&pDtlsSession->sslCtxConfig, dtlsSessionKeyDerivationCallback, pDtlsSession);
 
     CHK(mbedtls_ssl_setup(&pDtlsSession->sslCtx, &pDtlsSession->sslCtxConfig) == 0, STATUS_SSL_CTX_CREATION_FAILED);
-    mbedtls_ssl_set_mtu(&pDtlsSession->sslCtx, DEFAULT_MTU_SIZE);
+    mbedtls_ssl_set_mtu(&pDtlsSession->sslCtx, DEFAULT_MTU_SIZE_BYTES);
     mbedtls_ssl_set_bio(&pDtlsSession->sslCtx, pDtlsSession, dtlsSessionSendCallback, dtlsSessionReceiveCallback, NULL);
     mbedtls_ssl_set_timer_cb(&pDtlsSession->sslCtx, &pDtlsSession->transmissionTimer, dtlsSessionSetTimerCallback, dtlsSessionGetTimerCallback);
 

src/source/Crypto/Tls_mbedtls.c

@@ -15,7 +15,7 @@ STATUS createTlsSession(PTlsSessionCallbacks pCallbacks, PTlsSession* ppTlsSessi
     pTlsSession = (PTlsSession) MEMCALLOC(1, SIZEOF(TlsSession));
     CHK(pTlsSession != NULL, STATUS_NOT_ENOUGH_MEMORY);
 
-    CHK_STATUS(createIOBuffer(DEFAULT_MTU_SIZE, &pTlsSession->pReadBuffer));
+    CHK_STATUS(createIOBuffer(DEFAULT_MTU_SIZE_BYTES, &pTlsSession->pReadBuffer));
     pTlsSession->callbacks = *pCallbacks;
     pTlsSession->state = TLS_SESSION_STATE_NEW;
 
@@ -117,7 +117,7 @@ STATUS tlsSessionStart(PTlsSession pTlsSession, BOOL isServer)
     mbedtls_ssl_conf_authmode(&pTlsSession->sslCtxConfig, MBEDTLS_SSL_VERIFY_REQUIRED);
     mbedtls_ssl_conf_rng(&pTlsSession->sslCtxConfig, mbedtls_ctr_drbg_random, &pTlsSession->ctrDrbg);
     CHK(mbedtls_ssl_setup(&pTlsSession->sslCtx, &pTlsSession->sslCtxConfig) == 0, STATUS_SSL_CTX_CREATION_FAILED);
-    mbedtls_ssl_set_mtu(&pTlsSession->sslCtx, DEFAULT_MTU_SIZE);
+    mbedtls_ssl_set_mtu(&pTlsSession->sslCtx, DEFAULT_MTU_SIZE_BYTES);
     mbedtls_ssl_set_bio(&pTlsSession->sslCtx, pTlsSession, tlsSessionSendCallback, tlsSessionReceiveCallback, NULL);
 
     /* init and send handshake */

src/source/PeerConnection/PeerConnection.c

@@ -961,6 +961,7 @@ STATUS createPeerConnection(PRtcConfiguration pConfiguration, PRtcPeerConnection
     CHK_STATUS(timerQueueCreate(&pKvsPeerConnection->timerQueueHandle));
 
     pKvsPeerConnection->peerConnection.version = PEER_CONNECTION_CURRENT_VERSION;
+
     CHK_STATUS(generateJSONSafeString(pKvsPeerConnection->localIceUfrag, LOCAL_ICE_UFRAG_LEN));
     CHK_STATUS(generateJSONSafeString(pKvsPeerConnection->localIcePwd, LOCAL_ICE_PWD_LEN));
     CHK_STATUS(generateJSONSafeString(pKvsPeerConnection->localCNAME, LOCAL_CNAME_LEN));
@@ -983,7 +984,7 @@ STATUS createPeerConnection(PRtcConfiguration pConfiguration, PRtcPeerConnection
     pKvsPeerConnection->peerConnectionObjLock = MUTEX_CREATE(FALSE);
     pKvsPeerConnection->connectionState = RTC_PEER_CONNECTION_STATE_NONE;
     pKvsPeerConnection->MTU = pConfiguration->kvsRtcConfiguration.maximumTransmissionUnit == 0
-        ? DEFAULT_MTU_SIZE
+        ? DEFAULT_MTU_SIZE_BYTES
         : pConfiguration->kvsRtcConfiguration.maximumTransmissionUnit;
     ATOMIC_STORE_BOOL(&pKvsPeerConnection->sctpIsEnabled, FALSE);
 
@@ -1521,6 +1522,19 @@ STATUS setLocalDescription(PRtcPeerConnection pPeerConnection, PRtcSessionDescri
     return retStatus;
 }
 
+STATUS configureTransceiverRollingBuffer(PRtcRtpTransceiver pRtcRtpTransceiver, PRtcMediaStreamTrack pRtcMediaStreamTrack,
+                                         DOUBLE rollingBufferDurationSec, DOUBLE rollingBufferBitratebps)
+{
+    STATUS retStatus = STATUS_SUCCESS;
+    PKvsRtpTransceiver pKvsRtpTransceiver = (PKvsRtpTransceiver) pRtcRtpTransceiver;
+    CHK_WARN(pKvsRtpTransceiver != NULL || pRtcMediaStreamTrack != NULL, STATUS_NULL_ARG,
+             "Transceiver is not created. This needs to be invoked after addTransceiver is invoked");
+
+    CHK_STATUS(setUpRollingBufferConfigInternal(pKvsRtpTransceiver, pRtcMediaStreamTrack, rollingBufferDurationSec, rollingBufferBitratebps));
+CleanUp:
+    return retStatus;
+}
+
 STATUS addTransceiver(PRtcPeerConnection pPeerConnection, PRtcMediaStreamTrack pRtcMediaStreamTrack, PRtcRtpTransceiverInit pRtcRtpTransceiverInit,
                       PRtcRtpTransceiver* ppRtcRtpTransceiver)
 {
@@ -1535,19 +1549,22 @@ STATUS addTransceiver(PRtcPeerConnection pPeerConnection, PRtcMediaStreamTrack p
     UINT32 ssrc = (UINT32) RAND(), rtxSsrc = (UINT32) RAND();
     RTC_RTP_TRANSCEIVER_DIRECTION direction = RTC_RTP_TRANSCEIVER_DIRECTION_SENDRECV;
     RtcMediaStreamTrack videoTrack;
+
+    CHK(pKvsPeerConnection != NULL, STATUS_NULL_ARG);
+
     if (pRtcRtpTransceiverInit != NULL) {
         direction = pRtcRtpTransceiverInit->direction;
     }
 
-    CHK(pKvsPeerConnection != NULL, STATUS_NULL_ARG);
-
     if (direction == RTC_RTP_TRANSCEIVER_DIRECTION_RECVONLY && pRtcMediaStreamTrack == NULL) {
         MEMSET(&videoTrack, 0x00, SIZEOF(RtcMediaStreamTrack));
         videoTrack.kind = MEDIA_STREAM_TRACK_KIND_VIDEO;
         videoTrack.codec = RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE;
         STRCPY(videoTrack.streamId, "myKvsVideoStream");
         STRCPY(videoTrack.trackId, "myVideoTrack");
         pRtcMediaStreamTrack = &videoTrack;
+        // rollingBufferDurationSec will be DEFAULT_ROLLING_BUFFER_DURATION_IN_SECONDS
+        // rollingBufferBitratebps will be DEFAULT_EXPECTED_VIDEO_BIT_RATE
     }
 
     switch (pRtcMediaStreamTrack->codec) {

src/source/PeerConnection/Rtp.c

@@ -49,12 +49,73 @@ STATUS createKvsRtpTransceiver(RTC_RTP_TRANSCEIVER_DIRECTION direction, PKvsPeer
     return retStatus;
 }
 
+STATUS setUpRollingBufferConfigInternal(PKvsRtpTransceiver pKvsRtpTransceiver, PRtcMediaStreamTrack pRtcMediaStreamTrack,
+                                        DOUBLE rollingBufferDurationSec, DOUBLE rollingBufferBitratebps)
+{
+    STATUS retStatus = STATUS_SUCCESS;
+    CHK_ERR(pKvsRtpTransceiver != NULL || pRtcMediaStreamTrack != NULL, STATUS_NULL_ARG,
+            "Media track and transceiver not set. Make sure to set up transceiver with addTransceiver()");
+
+    //  Do not attempt to alloc for a new RollingBufferConfig if one is still not freed.
+    if (pKvsRtpTransceiver->pRollingBufferConfig == NULL) {
+        pKvsRtpTransceiver->pRollingBufferConfig = (PRollingBufferConfig) MEMCALLOC(1, SIZEOF(RollingBufferConfig));
+        CHK(pKvsRtpTransceiver->pRollingBufferConfig != NULL, STATUS_NOT_ENOUGH_MEMORY);
+    }
+
+    // Validate configured buffer duration is within acceptable range, else set to default duration.
+    if (rollingBufferDurationSec >= MIN_ROLLING_BUFFER_DURATION_IN_SECONDS && rollingBufferDurationSec <= MAX_ROLLING_BUFFER_DURATION_IN_SECONDS) {
+        DLOGI("Rolling buffer duration set to %lf seconds.", rollingBufferDurationSec);
+        pKvsRtpTransceiver->pRollingBufferConfig->rollingBufferDurationSec = rollingBufferDurationSec;
+    } else {
+        DLOGW("Rolling buffer duration does not fit range (%lf sec - %lf sec). Setting to default %lf sec", MIN_ROLLING_BUFFER_DURATION_IN_SECONDS,
+              MAX_ROLLING_BUFFER_DURATION_IN_SECONDS, DEFAULT_ROLLING_BUFFER_DURATION_IN_SECONDS);
+        pKvsRtpTransceiver->pRollingBufferConfig->rollingBufferDurationSec = DEFAULT_ROLLING_BUFFER_DURATION_IN_SECONDS;
+    }
+
+    // Validate configured expected bitrate is within acceptable range, else set to default bitrate.
+    if (rollingBufferBitratebps >= MIN_EXPECTED_BIT_RATE && rollingBufferBitratebps <= MAX_EXPECTED_BIT_RATE) {
+        if (pRtcMediaStreamTrack->kind == MEDIA_STREAM_TRACK_KIND_VIDEO) {
+            DLOGI("Rolling buffer expected bitrate set to %lf bps for video.", rollingBufferBitratebps);
+        } else if (pRtcMediaStreamTrack->kind == MEDIA_STREAM_TRACK_KIND_AUDIO) {
+            DLOGI("Rolling buffer expected bitrate set to %lf bps for audio.", rollingBufferBitratebps);
+        } else {
+            DLOGI("Rolling buffer expected bitrate set to %lf bps for unkown codec", rollingBufferBitratebps);
+        }
+        pKvsRtpTransceiver->pRollingBufferConfig->rollingBufferBitratebps = rollingBufferBitratebps;
+    } else {
+        if (pRtcMediaStreamTrack->kind == MEDIA_STREAM_TRACK_KIND_VIDEO) {
+            DLOGI("Rolling buffer bitrate does not fit range (%lf bps - %lf bps) for video. Setting to default %lf bps", MIN_EXPECTED_BIT_RATE,
+                  MAX_EXPECTED_BIT_RATE, DEFAULT_EXPECTED_VIDEO_BIT_RATE);
+            pKvsRtpTransceiver->pRollingBufferConfig->rollingBufferBitratebps = DEFAULT_EXPECTED_VIDEO_BIT_RATE;
+        } else if (pRtcMediaStreamTrack->kind == MEDIA_STREAM_TRACK_KIND_AUDIO) {
+            DLOGI("Rolling buffer bitrate does not fit range (%lf bps - %lf bps) for audio. Setting to default %lf bps", MIN_EXPECTED_BIT_RATE,
+                  MAX_EXPECTED_BIT_RATE, DEFAULT_EXPECTED_AUDIO_BIT_RATE);
+            pKvsRtpTransceiver->pRollingBufferConfig->rollingBufferBitratebps = DEFAULT_EXPECTED_AUDIO_BIT_RATE;
+        } else {
+            DLOGI("Rolling buffer bitrate does not fit range (%lf bps - %lf bps) for unknown codec. Setting to default %lf bps",
+                  MIN_EXPECTED_BIT_RATE, MAX_EXPECTED_BIT_RATE, DEFAULT_EXPECTED_VIDEO_BIT_RATE);
+            pKvsRtpTransceiver->pRollingBufferConfig->rollingBufferBitratebps = DEFAULT_EXPECTED_VIDEO_BIT_RATE;
+        }
+    }
+
+CleanUp:
+    CHK_LOG_ERR(retStatus);
+
+    return retStatus;
+}
+
 STATUS freeTransceiver(PRtcRtpTransceiver* pRtcRtpTransceiver)
 {
     UNUSED_PARAM(pRtcRtpTransceiver);
     return STATUS_NOT_IMPLEMENTED;
 }
 
+STATUS freeRollingBufferConfig(PRollingBufferConfig pRollingBufferConfig)
+{
+    SAFE_MEMFREE(pRollingBufferConfig);
+    return STATUS_SUCCESS;
+}
+
 STATUS freeKvsRtpTransceiver(PKvsRtpTransceiver* ppKvsRtpTransceiver)
 {
     STATUS retStatus = STATUS_SUCCESS;
@@ -76,6 +137,9 @@ STATUS freeKvsRtpTransceiver(PKvsRtpTransceiver* ppKvsRtpTransceiver)
     if (pKvsRtpTransceiver->sender.retransmitter != NULL) {
         freeRetransmitter(&pKvsRtpTransceiver->sender.retransmitter);
     }
+
+    freeRollingBufferConfig(pKvsRtpTransceiver->pRollingBufferConfig);
+
     MUTEX_FREE(pKvsRtpTransceiver->statsLock);
 
     SAFE_MEMFREE(pKvsRtpTransceiver->peerFrameBuffer);

src/source/PeerConnection/Rtp.h

@@ -9,13 +9,18 @@ extern "C" {
 
 // Default MTU comes from libwebrtc
 // https://groups.google.com/forum/#!topic/discuss-webrtc/gH5ysR3SoZI
-#define DEFAULT_MTU_SIZE                           1200
-#define DEFAULT_ROLLING_BUFFER_DURATION_IN_SECONDS 3
-#define HIGHEST_EXPECTED_BIT_RATE                  (10 * 1024 * 1024)
+#define DEFAULT_MTU_SIZE_BYTES                     1200
+#define DEFAULT_ROLLING_BUFFER_DURATION_IN_SECONDS (DOUBLE) 3
+#define DEFAULT_EXPECTED_VIDEO_BIT_RATE            (DOUBLE)(10 * 1024 * 1024)
+#define DEFAULT_EXPECTED_AUDIO_BIT_RATE            (DOUBLE)(10 * 1024 * 1024)
 #define DEFAULT_SEQ_NUM_BUFFER_SIZE                1000
 #define DEFAULT_VALID_INDEX_BUFFER_SIZE            1000
 #define DEFAULT_PEER_FRAME_BUFFER_SIZE             (5 * 1024)
 #define SRTP_AUTH_TAG_OVERHEAD                     10
+#define MIN_ROLLING_BUFFER_DURATION_IN_SECONDS     (DOUBLE) 0.1
+#define MIN_EXPECTED_BIT_RATE                      (DOUBLE)(102.4 * 1024) // Considering 1Kib = 1024 bits
+#define MAX_ROLLING_BUFFER_DURATION_IN_SECONDS     (DOUBLE) 10
+#define MAX_EXPECTED_BIT_RATE                      (DOUBLE)(240 * 1024 * 1024) // Considering 1Kib = 1024 bits
 
 // https://www.w3.org/TR/webrtc-stats/#dom-rtcoutboundrtpstreamstats-huge
 // Huge frames, by definition, are frames that have an encoded size at least 2.5 times the average size of the frames.
@@ -43,6 +48,12 @@ typedef struct {
 
 } RtcRtpSender, *PRtcRtpSender;
 
+typedef struct {
+    DOUBLE rollingBufferDurationSec; //!< Maximum duration of media that needs to be buffered (in seconds). The lowest allowed is 0.1 seconds (100ms)
+    DOUBLE rollingBufferBitratebps;  //!< Maximum expected bitrate of media (In bits/second). It is used to determine the buffer capacity. The lowest
+                                     //!< allowed is 100 Kbps
+} RollingBufferConfig, *PRollingBufferConfig;
+
 typedef struct {
     RtcRtpTransceiver transceiver;
     RtcRtpSender sender;
@@ -52,6 +63,8 @@ typedef struct {
     UINT32 jitterBufferSsrc;
     PJitterBuffer pJitterBuffer;
 
+    PRollingBufferConfig pRollingBufferConfig;
+
     UINT64 onFrameCustomData;
     RtcOnFrame onFrame;
 
@@ -84,6 +97,9 @@ STATUS writeRtpPacket(PKvsPeerConnection pKvsPeerConnection, PRtpPacket pRtpPack
 STATUS hasTransceiverWithSsrc(PKvsPeerConnection pKvsPeerConnection, UINT32 ssrc);
 STATUS findTransceiverBySsrc(PKvsPeerConnection pKvsPeerConnection, PKvsRtpTransceiver* ppTransceiver, UINT32 ssrc);
 
+STATUS setUpRollingBufferConfigInternal(PKvsRtpTransceiver, PRtcMediaStreamTrack, DOUBLE, DOUBLE);
+STATUS freeRollingBufferConfig(PRollingBufferConfig);
+
 #ifdef __cplusplus
 }
 #endif