Add a public api to set max frame and message size (CVE-2018-21035)

This change allows the user to set a lower allowed frame/message size
for reception. The purpose is to avoid an attacker to exhaust the
virtual memory of the peer.
Fixes CVE-2018-21035

[ChangeLog] Added public API to set the maximum frame size and message size

Task-number: QTBUG-70693
Change-Id: I5dc5918badc99166afdcc8d9c6106247a9f8666f
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

8 files changed, 285 insertions, 10 deletions

diff --git a/src/websockets/qwebsocket.cpp b/src/websockets/qwebsocket.cpp
index ade1eb4..144268f 100644
--- a/src/websockets/qwebsocket.cpp
+++ b/src/websockets/qwebsocket.cpp
@@ -788,4 +788,115 @@ qint64 QWebSocket::bytesToWrite() const
     return d->m_pSocket ? d->m_pSocket->bytesToWrite() : 0;
 }
 
+/*!
+    \since 5.15
+    Sets the maximum allowed size of an incoming websocket frame to \a maxAllowedIncomingFrameSize.
+    If an incoming frame exceeds this limit, the peer gets disconnected.
+    The accepted range is between 0 and maxIncomingFrameSize(), default is maxIncomingFrameSize().
+    The purpose of this function is to avoid exhausting virtual memory.
+
+    \sa maxAllowedIncomingFrameSize()
+ */
+void QWebSocket::setMaxAllowedIncomingFrameSize(quint64 maxAllowedIncomingFrameSize)
+{
+    Q_D(QWebSocket);
+    d->setMaxAllowedIncomingFrameSize(maxAllowedIncomingFrameSize);
+}
+
+/*!
+    \since 5.15
+    Returns the maximum allowed size of an incoming websocket frame.
+
+    \sa setMaxAllowedIncomingFrameSize()
+ */
+quint64 QWebSocket::maxAllowedIncomingFrameSize() const
+{
+    Q_D(const QWebSocket);
+    return d->maxAllowedIncomingFrameSize();
+}
+
+/*!
+    \since 5.15
+    Sets the maximum allowed size of an incoming websocket message to \a maxAllowedIncomingMessageSize.
+    If an incoming message exceeds this limit, the peer gets disconnected.
+    The accepted range is between 0 and maxIncomingMessageSize(), default is maxIncomingMessageSize().
+    The purpose of this function is to avoid exhausting virtual memory.
+
+    \sa maxAllowedIncomingMessageSize()
+ */
+void QWebSocket::setMaxAllowedIncomingMessageSize(quint64 maxAllowedIncomingMessageSize)
+{
+    Q_D(QWebSocket);
+    d->setMaxAllowedIncomingMessageSize(maxAllowedIncomingMessageSize);
+}
+
+/*!
+    \since 5.15
+    Returns the maximum allowed size of an incoming websocket message.
+
+    \sa setMaxAllowedIncomingMessageSize()
+ */
+quint64 QWebSocket::maxAllowedIncomingMessageSize() const
+{
+    Q_D(const QWebSocket);
+    return d->maxAllowedIncomingMessageSize();
+}
+
+/*!
+    \since 5.15
+    Returns the maximum supported size of an incoming websocket message for this websocket
+    implementation.
+ */
+quint64 QWebSocket::maxIncomingMessageSize()
+{
+    return QWebSocketPrivate::maxIncomingMessageSize();
+}
+
+/*!
+    \since 5.15
+    Returns the maximum supported size of an incoming websocket frame for this websocket
+    implementation.
+ */
+quint64 QWebSocket::maxIncomingFrameSize()
+{
+    return QWebSocketPrivate::maxIncomingFrameSize();
+}
+
+/*!
+    \since 5.15
+    Sets the maximum size of an outgoing websocket frame to \a outgoingFrameSize.
+    The accepted range is between 0 and maxOutgoingFrameSize(), default is 512kB.
+    The purpose of this function is to adapt to the maximum allowed frame size
+    of the receiver.
+
+    \sa outgoingFrameSize()
+ */
+void QWebSocket::setOutgoingFrameSize(quint64 outgoingFrameSize)
+{
+    Q_D(QWebSocket);
+    d->setOutgoingFrameSize(outgoingFrameSize);
+}
+
+/*!
+    \since 5.15
+    Returns the maximum size of an outgoing websocket frame.
+
+    \sa setOutgoingFrameSize()
+ */
+quint64 QWebSocket::outgoingFrameSize() const
+{
+    Q_D(const QWebSocket);
+    return d->outgoingFrameSize();
+}
+
+/*!
+    \since 5.15
+    Returns the maximum supported size of an outgoing websocket frame for this websocket
+    implementation.
+ */
+quint64 QWebSocket::maxOutgoingFrameSize()
+{
+    return QWebSocketPrivate::maxOutgoingFrameSize();
+}
+
 QT_END_NAMESPACE
diff --git a/src/websockets/qwebsocket.h b/src/websockets/qwebsocket.h
index 4944689..fad565c 100644
--- a/src/websockets/qwebsocket.h
+++ b/src/websockets/qwebsocket.h
@@ -115,6 +115,17 @@ public:
 
     qint64 bytesToWrite() const;
 
+    void setMaxAllowedIncomingFrameSize(quint64 maxAllowedIncomingFrameSize);
+    quint64 maxAllowedIncomingFrameSize() const;
+    void setMaxAllowedIncomingMessageSize(quint64 maxAllowedIncomingMessageSize);
+    quint64 maxAllowedIncomingMessageSize() const;
+    static quint64 maxIncomingMessageSize();
+    static quint64 maxIncomingFrameSize();
+
+    void setOutgoingFrameSize(quint64 outgoingFrameSize);
+    quint64 outgoingFrameSize() const;
+    static quint64 maxOutgoingFrameSize();
+
 public Q_SLOTS:
     void close(QWebSocketProtocol::CloseCode closeCode = QWebSocketProtocol::CloseCodeNormal,
                const QString &reason = QString());
diff --git a/src/websockets/qwebsocket_p.cpp b/src/websockets/qwebsocket_p.cpp
index ed09278..d50ccd2 100644
--- a/src/websockets/qwebsocket_p.cpp
+++ b/src/websockets/qwebsocket_p.cpp
@@ -69,7 +69,8 @@
 
 QT_BEGIN_NAMESPACE
 
-const quint64 FRAME_SIZE_IN_BYTES = 512 * 512 * 2;	//maximum size of a frame when sending a message
+const quint64 MAX_OUTGOING_FRAME_SIZE_IN_BYTES = std::numeric_limits<int>::max() - 1;
+const quint64 DEFAULT_OUTGOING_FRAME_SIZE_IN_BYTES = 512 * 512 * 2; //default size of a frame when sending a message
 
 QWebSocketConfiguration::QWebSocketConfiguration() :
 #ifndef QT_NO_SSL
@@ -111,7 +112,8 @@ QWebSocketPrivate::QWebSocketPrivate(const QString &origin, QWebSocketProtocol::
     m_configuration(),
     m_pMaskGenerator(&m_defaultMaskGenerator),
     m_defaultMaskGenerator(),
-    m_handshakeState(NothingDoneState)
+    m_handshakeState(NothingDoneState),
+    m_outgoingFrameSize(DEFAULT_OUTGOING_FRAME_SIZE_IN_BYTES)
 {
     m_pingTimer.start();
 }
@@ -143,7 +145,8 @@ QWebSocketPrivate::QWebSocketPrivate(QTcpSocket *pTcpSocket, QWebSocketProtocol:
     m_configuration(),
     m_pMaskGenerator(&m_defaultMaskGenerator),
     m_defaultMaskGenerator(),
-    m_handshakeState(NothingDoneState)
+    m_handshakeState(NothingDoneState),
+    m_outgoingFrameSize(DEFAULT_OUTGOING_FRAME_SIZE_IN_BYTES)
 {
     m_pingTimer.start();
 }
@@ -776,11 +779,11 @@ qint64 QWebSocketPrivate::doWriteFrames(const QByteArray &data, bool isBinary)
     const QWebSocketProtocol::OpCode firstOpCode = isBinary ?
                 QWebSocketProtocol::OpCodeBinary : QWebSocketProtocol::OpCodeText;
 
-    int numFrames = data.size() / int(FRAME_SIZE_IN_BYTES);
+    int numFrames = data.size() / int(outgoingFrameSize());
     QByteArray tmpData(data);
     tmpData.detach();
     char *payload = tmpData.data();
-    quint64 sizeLeft = quint64(data.size()) % FRAME_SIZE_IN_BYTES;
+    quint64 sizeLeft = quint64(data.size()) % outgoingFrameSize();
     if (Q_LIKELY(sizeLeft))
         ++numFrames;
 
@@ -799,7 +802,7 @@ qint64 QWebSocketPrivate::doWriteFrames(const QByteArray &data, bool isBinary)
         const bool isLastFrame = (i == (numFrames - 1));
         const bool isFirstFrame = (i == 0);
 
-        const quint64 size = qMin(bytesLeft, FRAME_SIZE_IN_BYTES);
+        const quint64 size = qMin(bytesLeft, outgoingFrameSize());
         const QWebSocketProtocol::OpCode opcode = isFirstFrame ? firstOpCode
                                                                : QWebSocketProtocol::OpCodeContinue;
 
@@ -1307,6 +1310,80 @@ void QWebSocketPrivate::setSocketState(QAbstractSocket::SocketState state)
 /*!
     \internal
  */
+void QWebSocketPrivate::setMaxAllowedIncomingFrameSize(quint64 maxAllowedIncomingFrameSize)
+{
+    m_dataProcessor.setMaxAllowedFrameSize(maxAllowedIncomingFrameSize);
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketPrivate::maxAllowedIncomingFrameSize() const
+{
+    return m_dataProcessor.maxAllowedFrameSize();
+}
+
+/*!
+    \internal
+ */
+void QWebSocketPrivate::setMaxAllowedIncomingMessageSize(quint64 maxAllowedIncomingMessageSize)
+{
+    m_dataProcessor.setMaxAllowedMessageSize(maxAllowedIncomingMessageSize);
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketPrivate::maxAllowedIncomingMessageSize() const
+{
+    return m_dataProcessor.maxAllowedMessageSize();
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketPrivate::maxIncomingMessageSize()
+{
+    return QWebSocketDataProcessor::maxMessageSize();
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketPrivate::maxIncomingFrameSize()
+{
+    return QWebSocketDataProcessor::maxFrameSize();
+}
+
+/*!
+    \internal
+ */
+void QWebSocketPrivate::setOutgoingFrameSize(quint64 outgoingFrameSize)
+{
+    if (outgoingFrameSize <= maxOutgoingFrameSize())
+        m_outgoingFrameSize = outgoingFrameSize;
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketPrivate::outgoingFrameSize() const
+{
+    return m_outgoingFrameSize;
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketPrivate::maxOutgoingFrameSize()
+{
+    return MAX_OUTGOING_FRAME_SIZE_IN_BYTES;
+}
+
+
+/*!
+    \internal
+ */
 void QWebSocketPrivate::setErrorString(const QString &errorString)
 {
     if (m_errorString != errorString)
diff --git a/src/websockets/qwebsocket_p.h b/src/websockets/qwebsocket_p.h
index 2d56f8a..640e7b2 100644
--- a/src/websockets/qwebsocket_p.h
+++ b/src/websockets/qwebsocket_p.h
@@ -160,6 +160,17 @@ public:
     void ping(const QByteArray &payload);
     void setSocketState(QAbstractSocket::SocketState state);
 
+    void setMaxAllowedIncomingFrameSize(quint64 maxAllowedIncomingFrameSize);
+    quint64 maxAllowedIncomingFrameSize() const;
+    void setMaxAllowedIncomingMessageSize(quint64 maxAllowedIncomingMessageSize);
+    quint64 maxAllowedIncomingMessageSize() const;
+    static quint64 maxIncomingMessageSize();
+    static quint64 maxIncomingFrameSize();
+
+    void setOutgoingFrameSize(quint64 outgoingFrameSize);
+    quint64 outgoingFrameSize() const;
+    static quint64 maxOutgoingFrameSize();
+
 private:
     QWebSocketPrivate(QTcpSocket *pTcpSocket, QWebSocketProtocol::Version version);
     void setVersion(QWebSocketProtocol::Version version);
@@ -250,6 +261,8 @@ private:
     QString m_httpStatusMessage;
     QMultiMap<QString, QString> m_headers;
 
+    quint64 m_outgoingFrameSize;
+
     friend class QWebSocketServerPrivate;
 #ifdef Q_OS_WASM
     emscripten::val socketContext = emscripten::val::null();
diff --git a/src/websockets/qwebsocketdataprocessor.cpp b/src/websockets/qwebsocketdataprocessor.cpp
index 191b992..4110f2a 100644
--- a/src/websockets/qwebsocketdataprocessor.cpp
+++ b/src/websockets/qwebsocketdataprocessor.cpp
@@ -105,6 +105,33 @@ QWebSocketDataProcessor::~QWebSocketDataProcessor()
     }
 }
 
+void QWebSocketDataProcessor::setMaxAllowedFrameSize(quint64 maxAllowedFrameSize)
+{
+    frame.setMaxAllowedFrameSize(maxAllowedFrameSize);
+}
+
+quint64 QWebSocketDataProcessor::maxAllowedFrameSize() const
+{
+    return frame.maxAllowedFrameSize();
+}
+
+/*!
+    \internal
+ */
+void QWebSocketDataProcessor::setMaxAllowedMessageSize(quint64 maxAllowedMessageSize)
+{
+    if (maxAllowedMessageSize <= maxMessageSize())
+        m_maxAllowedMessageSize = maxAllowedMessageSize;
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketDataProcessor::maxAllowedMessageSize() const
+{
+    return m_maxAllowedMessageSize;
+}
+
 /*!
     \internal
  */
@@ -118,7 +145,7 @@ quint64 QWebSocketDataProcessor::maxMessageSize()
  */
 quint64 QWebSocketDataProcessor::maxFrameSize()
 {
-    return MAX_FRAME_SIZE_IN_BYTES;
+   return QWebSocketFrame::maxFrameSize();
 }
 
 /*!
@@ -167,7 +194,7 @@ bool QWebSocketDataProcessor::process(QIODevice *pIoDevice)
                         ? quint64(m_textMessage.length())
                         : quint64(m_binaryMessage.length());
                 if (Q_UNLIKELY((messageLength + quint64(frame.payload().length())) >
-                               MAX_MESSAGE_SIZE_IN_BYTES)) {
+                               maxAllowedMessageSize())) {
                     clear();
                     Q_EMIT errorEncountered(QWebSocketProtocol::CloseCodeTooMuchData,
                                             tr("Received message is too big."));
diff --git a/src/websockets/qwebsocketdataprocessor_p.h b/src/websockets/qwebsocketdataprocessor_p.h
index 03635b1..62a2dc0 100644
--- a/src/websockets/qwebsocketdataprocessor_p.h
+++ b/src/websockets/qwebsocketdataprocessor_p.h
@@ -65,6 +65,8 @@ QT_BEGIN_NAMESPACE
 class QIODevice;
 class QWebSocketFrame;
 
+const quint64 MAX_MESSAGE_SIZE_IN_BYTES = std::numeric_limits<int>::max() - 1;
+
 class Q_AUTOTEST_EXPORT QWebSocketDataProcessor : public QObject
 {
     Q_OBJECT
@@ -74,6 +76,10 @@ public:
     explicit QWebSocketDataProcessor(QObject *parent = nullptr);
     ~QWebSocketDataProcessor() override;
 
+    void setMaxAllowedFrameSize(quint64 maxAllowedFrameSize);
+    quint64 maxAllowedFrameSize() const;
+    void setMaxAllowedMessageSize(quint64 maxAllowedMessageSize);
+    quint64 maxAllowedMessageSize() const;
     static quint64 maxMessageSize();
     static quint64 maxFrameSize();
 
@@ -115,6 +121,7 @@ private:
     QTextCodec *m_pTextCodec;
     QWebSocketFrame frame;
     QTimer waitTimer;
+    quint64 m_maxAllowedMessageSize = MAX_MESSAGE_SIZE_IN_BYTES;
 
     bool processControlFrame(const QWebSocketFrame &frame);
     void timeout();
diff --git a/src/websockets/qwebsocketframe.cpp b/src/websockets/qwebsocketframe.cpp
index cfa63ed..716aebd 100644
--- a/src/websockets/qwebsocketframe.cpp
+++ b/src/websockets/qwebsocketframe.cpp
@@ -64,6 +64,31 @@ QT_BEGIN_NAMESPACE
 /*!
     \internal
  */
+void QWebSocketFrame::setMaxAllowedFrameSize(quint64 maxAllowedFrameSize)
+{
+    if (maxAllowedFrameSize <= maxFrameSize())
+        m_maxAllowedFrameSize = maxAllowedFrameSize;
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketFrame::maxAllowedFrameSize() const
+{
+    return m_maxAllowedFrameSize;
+}
+
+/*!
+    \internal
+ */
+quint64 QWebSocketFrame::maxFrameSize()
+{
+    return MAX_FRAME_SIZE_IN_BYTES;
+}
+
+/*!
+    \internal
+ */
 QWebSocketProtocol::CloseCode QWebSocketFrame::closeCode() const
 {
     return isDone() ? m_closeCode : QWebSocketProtocol::CloseCodeGoingAway;
@@ -354,7 +379,7 @@ QWebSocketFrame::ProcessingState QWebSocketFrame::readFramePayload(QIODevice *pI
     if (!m_length)
         return PS_DISPATCH_RESULT;
 
-    if (Q_UNLIKELY(m_length > MAX_FRAME_SIZE_IN_BYTES)) {
+    if (Q_UNLIKELY(m_length > maxAllowedFrameSize())) {
         setError(QWebSocketProtocol::CloseCodeTooMuchData, tr("Maximum framesize exceeded."));
         return PS_DISPATCH_RESULT;
     }
diff --git a/src/websockets/qwebsocketframe_p.h b/src/websockets/qwebsocketframe_p.h
index a8b9684..992379a 100644
--- a/src/websockets/qwebsocketframe_p.h
+++ b/src/websockets/qwebsocketframe_p.h
@@ -65,7 +65,6 @@ QT_BEGIN_NAMESPACE
 class QIODevice;
 
 const quint64 MAX_FRAME_SIZE_IN_BYTES = std::numeric_limits<int>::max() - 1;
-const quint64 MAX_MESSAGE_SIZE_IN_BYTES = std::numeric_limits<int>::max() - 1;
 
 class Q_AUTOTEST_EXPORT QWebSocketFrame
 {
@@ -74,6 +73,10 @@ class Q_AUTOTEST_EXPORT QWebSocketFrame
 public:
     QWebSocketFrame() = default;
 
+    void setMaxAllowedFrameSize(quint64 maxAllowedFrameSize);
+    quint64 maxAllowedFrameSize() const;
+    static quint64 maxFrameSize();
+
     QWebSocketProtocol::CloseCode closeCode() const;
     QString closeReason() const;
     bool isFinalFrame() const;
@@ -118,6 +121,7 @@ private:
     bool m_rsv2 = false;
     bool m_rsv3 = false;
     bool m_isValid = false;
+    quint64 m_maxAllowedFrameSize = MAX_FRAME_SIZE_IN_BYTES;
 
     ProcessingState readFrameHeader(QIODevice *pIoDevice);
     ProcessingState readFramePayloadLength(QIODevice *pIoDevice);