Add a public api to set max frame and message size (CVE-2018-21035)

This change allows the user to set a lower allowed frame/message size
for reception. The purpose is to avoid an attacker to exhaust the
virtual memory of the peer.
Fixes CVE-2018-21035

[ChangeLog] Added public API to set the maximum frame size and message size

Task-number: QTBUG-70693
Change-Id: I5dc5918badc99166afdcc8d9c6106247a9f8666f
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

1 files changed, 83 insertions, 6 deletions

diff --git a/src/websockets/qwebsocket_p.cpp b/src/websockets/qwebsocket_p.cpp
index ed09278..d50ccd2 100644
--- a/src/websockets/qwebsocket_p.cpp
+++ b/src/websockets/qwebsocket_p.cpp
@@ -69,7 +69,8 @@
 
 QT_BEGIN_NAMESPACE
 
-const quint64 FRAME_SIZE_IN_BYTES = 512 * 512 * 2;	//maximum size of a frame when sending a message
+const quint64 MAX_OUTGOING_FRAME_SIZE_IN_BYTES = std::numeric_limits<int>::max() - 1;
+const quint64 DEFAULT_OUTGOING_FRAME_SIZE_IN_BYTES = 512 * 512 * 2; //default size of a frame when sending a message
 
 QWebSocketConfiguration::QWebSocketConfiguration() :
 #ifndef QT_NO_SSL
@@ -111,7 +112,8 @@ QWebSocketPrivate::QWebSocketPrivate(const QString &origin, QWebSocketProtocol::
     m_configuration(),
     m_pMaskGenerator(&m_defaultMaskGenerator),
     m_defaultMaskGenerator(),
-    m_handshakeState(NothingDoneState)
+    m_handshakeState(NothingDoneState),
+    m_outgoingFrameSize(DEFAULT_OUTGOING_FRAME_SIZE_IN_BYTES)
 {
     m_pingTimer.start();
 }
@@ -143,7 +145,8 @@ QWebSocketPrivate::QWebSocketPrivate(QTcpSocket *pTcpSocket, QWebSocketProtocol:
     m_configuration(),
     m_pMaskGenerator(&m_defaultMaskGenerator),
     m_defaultMaskGenerator(),
-    m_handshakeState(NothingDoneState)
+    m_handshakeState(NothingDoneState),
+    m_outgoingFrameSize(DEFAULT_OUTGOING_FRAME_SIZE_IN_BYTES)
 {
     m_pingTimer.start();
 }
@@ -776,11 +779,11 @@ qint64 QWebSocketPrivate::doWriteFrames(const QByteArray &data, bool isBinary)
     const QWebSocketProtocol::OpCode firstOpCode = isBinary ?
                 QWebSocketProtocol::OpCodeBinary : QWebSocketProtocol::OpCodeText;
 
-    int numFrames = data.size() / int(FRAME_SIZE_IN_BYTES);
+    int numFrames = data.size() / int(outgoingFrameSize());
     QByteArray tmpData(data);
     tmpData.detach();
     char *payload = tmpData.data();
-    quint64 sizeLeft = quint64(data.size()) % FRAME_SIZE_IN_BYTES;
+    quint64 sizeLeft = quint64(data.size()) % outgoingFrameSize();
     if (Q_LIKELY(sizeLeft))
         ++numFrames;
 
@@ -799,7 +802,7 @@ qint64 QWebSocketPrivate::doWriteFrames(const QByteArray &data, bool isBinary)
         const bool isLastFrame = (i == (numFrames - 1));
         const bool isFirstFrame = (i == 0);
 
-        const quint64 size = qMin(bytesLeft, FRAME_SIZE_IN_BYTES);
+        const quint64 size = qMin(bytesLeft, outgoingFrameSize());
         const QWebSocketProtocol::OpCode opcode = isFirstFrame ? firstOpCode
                                                                : QWebSocketProtocol::OpCodeContinue;
 
@@ -1307,6 +1310,80 @@ void QWebSocketPrivate::setSocketState(QAbstractSocket::SocketState state)
 /*!
     \internal
  */
+void QWebSocketPrivate::setMaxAllowedIncomingFrameSize(quint64 maxAllowedIncomingFrameSize)
+{
+    m_dataProcessor.setMaxAllowedFrameSize(maxAllowedIncomingFrameSize);
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketPrivate::maxAllowedIncomingFrameSize() const
+{
+    return m_dataProcessor.maxAllowedFrameSize();
+}
+
+/*!
+    \internal
+ */
+void QWebSocketPrivate::setMaxAllowedIncomingMessageSize(quint64 maxAllowedIncomingMessageSize)
+{
+    m_dataProcessor.setMaxAllowedMessageSize(maxAllowedIncomingMessageSize);
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketPrivate::maxAllowedIncomingMessageSize() const
+{
+    return m_dataProcessor.maxAllowedMessageSize();
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketPrivate::maxIncomingMessageSize()
+{
+    return QWebSocketDataProcessor::maxMessageSize();
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketPrivate::maxIncomingFrameSize()
+{
+    return QWebSocketDataProcessor::maxFrameSize();
+}
+
+/*!
+    \internal
+ */
+void QWebSocketPrivate::setOutgoingFrameSize(quint64 outgoingFrameSize)
+{
+    if (outgoingFrameSize <= maxOutgoingFrameSize())
+        m_outgoingFrameSize = outgoingFrameSize;
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketPrivate::outgoingFrameSize() const
+{
+    return m_outgoingFrameSize;
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketPrivate::maxOutgoingFrameSize()
+{
+    return MAX_OUTGOING_FRAME_SIZE_IN_BYTES;
+}
+
+
+/*!
+    \internal
+ */
 void QWebSocketPrivate::setErrorString(const QString &errorString)
 {
     if (m_errorString != errorString)