Make DTLS fragment stay within MTU size range

- Fixes #1100
- All credits to @pnts-se and his PR #1143

worker/include/RTC/DtlsTransport.hpp

@@ -152,6 +152,8 @@ namespace RTC
 			return this->localRole;
 		}
 		void SendApplicationData(const uint8_t* data, size_t len);
+		// This method must be public since it's called within an OpenSSL callback.
+		void SendDtlsData(const uint8_t* data, size_t len);
 
 	private:
 		bool IsRunning() const
@@ -173,7 +175,6 @@ namespace RTC
 		}
 		void Reset();
 		bool CheckStatus(int returnCode);
-		void SendPendingOutgoingDtlsData();
 		bool SetTimeout();
 		bool ProcessHandshake();
 		bool CheckRemoteFingerprint();

worker/src/RTC/DtlsTransport.cpp

@@ -53,6 +53,31 @@ inline static unsigned int onSslDtlsTimer(SSL* /*ssl*/, unsigned int timerUs)
 		return 2 * timerUs;
 }
 
+inline static long onSslBioOut(
+  BIO* bio,
+  int operationType,
+  const char* argp,
+  size_t len,
+  int /*argi*/,
+  long /*argl*/,
+  int ret,
+  size_t* /*processed*/)
+{
+	long resultOfcallback = (operationType == BIO_CB_RETURN) ? static_cast<long>(ret) : 1;
+
+	if (operationType == BIO_CB_WRITE && argp && len > 0)
+	{
+		MS_DEBUG_DEV("%zu bytes of DTLS data ready to be sent", len);
+
+		RTC::DtlsTransport* dtlsTransport =
+		  reinterpret_cast<RTC::DtlsTransport*>(BIO_get_callback_arg(bio));
+
+		dtlsTransport->SendDtlsData(reinterpret_cast<const uint8_t*>(argp), len);
+	}
+
+	return resultOfcallback;
+}
+
 namespace RTC
 {
 	/* Static. */
@@ -633,6 +658,8 @@ namespace RTC
 			goto error;
 		}
 
+		BIO_set_callback_ex(this->sslBioToNetwork, onSslBioOut);
+		BIO_set_callback_arg(this->sslBioToNetwork, reinterpret_cast<char*>(this));
 		SSL_set_bio(this->ssl, this->sslBioFromNetwork, this->sslBioToNetwork);
 
 		// Set the MTU so that we don't send packets that are too large with no fragmentation.
@@ -673,7 +700,6 @@ namespace RTC
 		{
 			// Send close alert to the peer.
 			SSL_shutdown(this->ssl);
-			SendPendingOutgoingDtlsData();
 		}
 
 		if (this->ssl)
@@ -776,7 +802,6 @@ namespace RTC
 
 				SSL_set_connect_state(this->ssl);
 				SSL_do_handshake(this->ssl);
-				SendPendingOutgoingDtlsData();
 				SetTimeout();
 
 				break;
@@ -847,9 +872,6 @@ namespace RTC
 		// Must call SSL_read() to process received DTLS data.
 		read = SSL_read(this->ssl, static_cast<void*>(DtlsTransport::sslReadBuffer), SslReadBufferSize);
 
-		// Send data if it's ready.
-		SendPendingOutgoingDtlsData();
-
 		// Check SSL status and return if it is bad/closed.
 		if (!CheckStatus(read))
 			return;
@@ -910,9 +932,14 @@ namespace RTC
 			MS_WARN_TAG(
 			  dtls, "OpenSSL SSL_write() wrote less (%d bytes) than given data (%zu bytes)", written, len);
 		}
+	}
 
-		// Send data.
-		SendPendingOutgoingDtlsData();
+	void DtlsTransport::SendDtlsData(const uint8_t* data, size_t len)
+	{
+		MS_TRACE();
+
+		// Notify the listener.
+		this->listener->OnDtlsTransportSendData(this, data, len);
 	}
 
 	void DtlsTransport::Reset()
@@ -929,9 +956,8 @@ namespace RTC
 		// Stop the DTLS timer.
 		this->timer->Stop();
 
-		// We need to reset the SSL instance so we need to "shutdown" it, but we
-		// don't want to send a Close Alert to the peer, so just don't call
-		// SendPendingOutgoingDTLSData().
+		// NOTE: We need to reset the SSL instance so we need to "shutdown" it, but we
+		// don't want to send a Close Alert to the peer. However this is gonna happen.
 		SSL_shutdown(this->ssl);
 
 		this->localRole.reset();
@@ -1044,32 +1070,6 @@ namespace RTC
 		}
 	}
 
-	inline void DtlsTransport::SendPendingOutgoingDtlsData()
-	{
-		MS_TRACE();
-
-		if (BIO_eof(this->sslBioToNetwork))
-			return;
-
-		int64_t read;
-		char* data{ nullptr };
-
-		read = BIO_get_mem_data(this->sslBioToNetwork, &data); // NOLINT
-
-		if (read <= 0)
-			return;
-
-		MS_DEBUG_DEV("%" PRIu64 " bytes of DTLS data ready to sent to the peer", read);
-
-		// Notify the listener.
-		this->listener->OnDtlsTransportSendData(
-		  this, reinterpret_cast<uint8_t*>(data), static_cast<size_t>(read));
-
-		// Clear the BIO buffer.
-		// NOTE: the (void) avoids the -Wunused-value warning.
-		(void)BIO_reset(this->sslBioToNetwork);
-	}
-
 	inline bool DtlsTransport::SetTimeout()
 	{
 		MS_TRACE();
@@ -1512,9 +1512,6 @@ namespace RTC
 
 		if (ret == 1)
 		{
-			// If required, send DTLS data.
-			SendPendingOutgoingDtlsData();
-
 			// Set the DTLS timer again.
 			SetTimeout();
 		}