QNetworkReply: Decompress when reading

Rather than when the data is received. Source compatibility is
achieved through double-decompressing the data. This lets us know
how many bytes are available just as before but without having the
uncompressed data left in memory.

Fixes: QTBUG-83269
Change-Id: I352bd09581614c582e4628243e2a0e895ba4946b
Reviewed-by: Edward Welbourne <[email protected]>

src/network/access/qhttp2protocolhandler.cpp

@@ -1233,13 +1233,8 @@ void QHttp2ProtocolHandler::updateStream(Stream &stream, const HPack::HttpHeader
     if (QHttpNetworkReply::isHttpRedirect(statusCode) && redirectUrl.isValid())
         httpReply->setRedirectUrl(redirectUrl);
 
-    if (httpReplyPrivate->isCompressed() && httpRequest.d->autoDecompress) {
+    if (httpReplyPrivate->isCompressed() && httpRequest.d->autoDecompress)
         httpReplyPrivate->removeAutoDecompressHeader();
-        httpReplyPrivate->decompressHelper.setEncoding(
-                httpReplyPrivate->headerField("content-encoding"));
-        httpReplyPrivate->decompressHelper.setMinimumArchiveBombSize(
-                httpReplyPrivate->request.minimumArchiveBombSize());
-    }
 
     if (QHttpNetworkReply::isHttpRedirect(statusCode)) {
         // Note: This status code can trigger uploadByteDevice->reset() in
@@ -1276,27 +1271,12 @@ void QHttp2ProtocolHandler::updateStream(Stream &stream, const Frame &frame,
 
     if (const auto length = frame.dataSize()) {
         const char *data = reinterpret_cast<const char *>(frame.dataBegin());
-        auto &httpRequest = stream.request();
         auto replyPrivate = httpReply->d_func();
 
         replyPrivate->totalProgress += length;
 
         const QByteArray wrapped(data, length);
-        if (httpRequest.d->autoDecompress && replyPrivate->isCompressed()) {
-            Q_ASSERT(replyPrivate->decompressHelper.isValid());
-
-            replyPrivate->decompressHelper.feed(wrapped);
-            while (replyPrivate->decompressHelper.hasData()) {
-                QByteArray output(4 * 1024, Qt::Uninitialized);
-                qint64 read = replyPrivate->decompressHelper.read(output.data(), output.size());
-                if (read > 0) {
-                    output.resize(read);
-                    replyPrivate->responseData.append(std::move(output));
-                }
-            }
-        } else {
-            replyPrivate->responseData.append(wrapped);
-        }
+        replyPrivate->responseData.append(wrapped);
 
         if (replyPrivate->shouldEmitSignals()) {
             if (connectionType == Qt::DirectConnection) {

src/network/access/qhttpnetworkconnection.cpp

@@ -48,6 +48,7 @@
 #include <qnetworkproxy.h>
 #include <qauthenticator.h>
 #include <qcoreapplication.h>
+#include <private/qdecompresshelper_p.h>
 
 #include <qbuffer.h>
 #include <qpair.h>

src/network/access/qhttpnetworkreply.cpp

@@ -46,6 +46,8 @@
 #    include <QtNetwork/qsslconfiguration.h>
 #endif
 
+#include <private/qdecompresshelper_p.h>
+
 QT_BEGIN_NAMESPACE
 
 QHttpNetworkReply::QHttpNetworkReply(const QUrl &url, QObject *parent)
@@ -348,7 +350,6 @@ void QHttpNetworkReplyPrivate::clearHttpLayerInformation()
     currentChunkRead = 0;
     lastChunkRead = false;
     connectionCloseEnabled = true;
-    decompressHelper.clear();
     fields.clear();
 }
 
@@ -554,11 +555,6 @@ qint64 QHttpNetworkReplyPrivate::readHeader(QAbstractSocket *socket)
             headerField("proxy-connection").toLower().contains("close")) ||
             (majorVersion == 1 && minorVersion == 0 &&
             (connectionHeaderField.isEmpty() && !headerField("proxy-connection").toLower().contains("keep-alive")));
-        if (autoDecompress && isCompressed()) {
-            if (!decompressHelper.setEncoding(headerField("content-encoding")))
-                return -1; // Either the encoding was unsupported or the decoder could not be set up
-            decompressHelper.setMinimumArchiveBombSize(request.minimumArchiveBombSize());
-        }
     }
     return bytes;
 }
@@ -660,42 +656,18 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QByteDataBuff
 {
     qint64 bytes = 0;
 
-    // for compressed data we'll allocate a temporary one that we then decompress
-    QByteDataBuffer *tempOutDataBuffer = (autoDecompress ? new QByteDataBuffer : out);
-
-
     if (isChunked()) {
         // chunked transfer encoding (rfc 2616, sec 3.6)
-        bytes += readReplyBodyChunked(socket, tempOutDataBuffer);
+        bytes += readReplyBodyChunked(socket, out);
     } else if (bodyLength > 0) {
         // we have a Content-Length
-        bytes += readReplyBodyRaw(socket, tempOutDataBuffer, bodyLength - contentRead);
+        bytes += readReplyBodyRaw(socket, out, bodyLength - contentRead);
         if (contentRead + bytes == bodyLength)
             state = AllDoneState;
     } else {
         // no content length. just read what's possible
-        bytes += readReplyBodyRaw(socket, tempOutDataBuffer, socket->bytesAvailable());
-    }
-
-    // This is true if there is compressed encoding and we're supposed to use it.
-    if (autoDecompress) {
-        QScopedPointer holder(tempOutDataBuffer);
-        if (!decompressHelper.isValid())
-            return -1;
-
-        decompressHelper.feed(std::move(*tempOutDataBuffer));
-        while (decompressHelper.hasData()) {
-            QByteArray output(4 * 1024, Qt::Uninitialized);
-            qint64 read = decompressHelper.read(output.data(), output.size());
-            if (read < 0) {
-                return -1;
-            } else if (read > 0) {
-                output.resize(read);
-                out->append(std::move(output));
-            }
-        }
+        bytes += readReplyBodyRaw(socket, out, socket->bytesAvailable());
     }
-
     contentRead += bytes;
     return bytes;
 }

src/network/access/qhttpnetworkreply_p.h

@@ -274,8 +274,6 @@ class Q_AUTOTEST_EXPORT QHttpNetworkReplyPrivate : public QObjectPrivate, public
 
     char* userProvidedDownloadBuffer;
     QUrl redirectUrl;
-
-    QDecompressHelper decompressHelper;
 };
 
 

src/network/access/qhttpnetworkrequest.cpp

@@ -57,7 +57,6 @@ QHttpNetworkRequestPrivate::QHttpNetworkRequestPrivate(const QHttpNetworkRequest
       customVerb(other.customVerb),
       priority(other.priority),
       uploadByteDevice(other.uploadByteDevice),
-      minimumArchiveBombSize(other.minimumArchiveBombSize),
       autoDecompress(other.autoDecompress),
       pipeliningAllowed(other.pipeliningAllowed),
       http2Allowed(other.http2Allowed),
@@ -94,7 +93,7 @@ bool QHttpNetworkRequestPrivate::operator==(const QHttpNetworkRequestPrivate &ot
         && (redirectPolicy == other.redirectPolicy)
         && (peerVerifyName == other.peerVerifyName)
         && (needResendWithCredentials == other.needResendWithCredentials)
-        && (minimumArchiveBombSize == other.minimumArchiveBombSize);
+        ;
 }
 
 QByteArray QHttpNetworkRequest::methodName() const
@@ -406,15 +405,5 @@ void QHttpNetworkRequest::setPeerVerifyName(const QString &peerName)
     d->peerVerifyName = peerName;
 }
 
-qint64 QHttpNetworkRequest::minimumArchiveBombSize() const
-{
-    return d->minimumArchiveBombSize;
-}
-
-void QHttpNetworkRequest::setMinimumArchiveBombSize(qint64 threshold)
-{
-    d->minimumArchiveBombSize = threshold;
-}
-
 QT_END_NAMESPACE
 

src/network/access/qhttpnetworkrequest_p.h

@@ -150,9 +150,6 @@ class Q_AUTOTEST_EXPORT QHttpNetworkRequest: public QHttpNetworkHeader
     QString peerVerifyName() const;
     void setPeerVerifyName(const QString &peerName);
 
-    qint64 minimumArchiveBombSize() const;
-    void setMinimumArchiveBombSize(qint64 threshold);
-
 private:
     QSharedDataPointer<QHttpNetworkRequestPrivate> d;
     friend class QHttpNetworkRequestPrivate;
@@ -178,7 +175,6 @@ class QHttpNetworkRequestPrivate : public QHttpNetworkHeaderPrivate
     QByteArray customVerb;
     QHttpNetworkRequest::Priority priority;
     mutable QNonContiguousByteDevice* uploadByteDevice;
-    qint64 minimumArchiveBombSize = 0;
     bool autoDecompress;
     bool pipeliningAllowed;
     bool http2Allowed;

src/network/access/qhttpprotocolhandler.cpp

@@ -177,8 +177,7 @@ void QHttpProtocolHandler::_q_receiveReply()
                    m_connection->d_func()->emitReplyError(m_socket, m_reply, QNetworkReply::RemoteHostClosedError);
                    break;
                }
-           } else if (!replyPrivate->isChunked() && !replyPrivate->autoDecompress
-                 && replyPrivate->bodyLength > 0) {
+           } else if (!replyPrivate->isChunked() && replyPrivate->bodyLength > 0) {
                  // bulk files like images should fulfill these properties and
                  // we can therefore save on memory copying
                 qint64 haveRead = replyPrivate->readBodyFast(m_socket, &replyPrivate->responseData);

src/network/access/qhttpthreaddelegate.cpp

@@ -545,6 +545,7 @@ void QHttpThreadDelegate::synchronousFinishedSlot()
             incomingErrorCode = statusCodeFromHttp(httpReply->statusCode(), httpRequest.url());
     }
 
+    isCompressed = httpReply->isCompressed();
     synchronousDownloadData = httpReply->readAll();
 
     QMetaObject::invokeMethod(httpReply, "deleteLater", Qt::QueuedConnection);
@@ -634,6 +635,7 @@ void QHttpThreadDelegate::headerChangedSlot()
     incomingContentLength = httpReply->contentLength();
     removedContentLength = httpReply->removedContentLength();
     isHttp2Used = httpReply->isHttp2Used();
+    isCompressed = httpReply->isCompressed();
 
     emit downloadMetaData(incomingHeaders,
                           incomingStatusCode,
@@ -642,7 +644,8 @@ void QHttpThreadDelegate::headerChangedSlot()
                           downloadBuffer,
                           incomingContentLength,
                           removedContentLength,
-                          isHttp2Used);
+                          isHttp2Used,
+                          isCompressed);
 }
 
 void QHttpThreadDelegate::synchronousHeaderChangedSlot()

src/network/access/qhttpthreaddelegate_p.h

@@ -119,6 +119,8 @@ class QHttpThreadDelegate : public QObject
     QString incomingErrorDetail;
     QHttp2Configuration http2Parameters;
 
+    bool isCompressed;
+
 protected:
     // The zerocopy download buffer, if used:
     QSharedPointer<char> downloadBuffer;
@@ -142,7 +144,7 @@ class QHttpThreadDelegate : public QObject
     void preSharedKeyAuthenticationRequired(QSslPreSharedKeyAuthenticator *);
 #endif
     void downloadMetaData(const QList<QPair<QByteArray,QByteArray> > &, int, const QString &, bool,
-                          QSharedPointer<char>, qint64, qint64, bool);
+                          QSharedPointer<char>, qint64, qint64, bool, bool);
     void downloadProgress(qint64, qint64);
     void downloadData(const QByteArray &);
     void error(QNetworkReply::NetworkError, const QString &);