bug 366559 - patch 7, content-encoding brotli for https r=bagder

--HG--
extra : rebase_source : 7723eb77ac22aa5000d8e00bf68c83427e3411bb

modules/libpref/init/all.js

@@ -1322,7 +1322,7 @@ pref("network.http.redirection-limit", 20);
 // NOTE: support for "compress" has been disabled per bug 196406.
 // NOTE: separate values with comma+space (", "): see bug 576033
 pref("network.http.accept-encoding", "gzip, deflate");
-pref("network.http.accept-encoding.secure", "gzip, deflate");
+pref("network.http.accept-encoding.secure", "gzip, deflate, brotli");
 
 pref("network.http.pipelining"      , false);
 pref("network.http.pipelining.ssl"  , false); // disable pipelining over SSL

netwerk/build/nsNetModule.cpp

@@ -443,6 +443,7 @@ nsresult NS_NewStreamConv(nsStreamConverterService **aStreamConv);
 #define UNKNOWN_CONTENT              "?from=" UNKNOWN_CONTENT_TYPE "&to=*/*"
 #define GZIP_TO_UNCOMPRESSED         "?from=gzip&to=uncompressed"
 #define XGZIP_TO_UNCOMPRESSED        "?from=x-gzip&to=uncompressed"
+#define BROTLI_TO_UNCOMPRESSED       "?from=brotli&to=uncompressed"
 #define COMPRESS_TO_UNCOMPRESSED     "?from=compress&to=uncompressed"
 #define XCOMPRESS_TO_UNCOMPRESSED    "?from=x-compress&to=uncompressed"
 #define DEFLATE_TO_UNCOMPRESSED      "?from=deflate&to=uncompressed"
@@ -462,6 +463,7 @@ static const mozilla::Module::CategoryEntry kNeckoCategories[] = {
     { NS_ISTREAMCONVERTER_KEY, UNKNOWN_CONTENT, "" },
     { NS_ISTREAMCONVERTER_KEY, GZIP_TO_UNCOMPRESSED, "" },
     { NS_ISTREAMCONVERTER_KEY, XGZIP_TO_UNCOMPRESSED, "" },
+    { NS_ISTREAMCONVERTER_KEY, BROTLI_TO_UNCOMPRESSED, "" },
     { NS_ISTREAMCONVERTER_KEY, COMPRESS_TO_UNCOMPRESSED, "" },
     { NS_ISTREAMCONVERTER_KEY, XCOMPRESS_TO_UNCOMPRESSED, "" },
     { NS_ISTREAMCONVERTER_KEY, DEFLATE_TO_UNCOMPRESSED, "" },
@@ -1050,6 +1052,7 @@ static const mozilla::Module::ContractIDEntry kNeckoContracts[] = {
     { NS_BINARYDETECTOR_CONTRACTID, &kNS_BINARYDETECTOR_CID },
     { NS_ISTREAMCONVERTER_KEY GZIP_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY XGZIP_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
+    { NS_ISTREAMCONVERTER_KEY BROTLI_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY COMPRESS_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY XCOMPRESS_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY DEFLATE_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },

netwerk/mime/nsMimeTypes.h

@@ -30,6 +30,7 @@
 #define APPLICATION_GZIP                    "application/x-gzip"
 #define APPLICATION_GZIP2                   "application/gzip"
 #define APPLICATION_GZIP3                   "application/x-gunzip"
+#define APPLICATION_BROTLI                  "application/brotli"
 #define APPLICATION_ZIP                     "application/zip"
 #define APPLICATION_HTTP_INDEX_FORMAT       "application/http-index-format"
 #define APPLICATION_ECMASCRIPT              "application/ecmascript"

netwerk/protocol/http/HttpBaseChannel.cpp

@@ -836,6 +836,17 @@ HttpBaseChannel::DoApplyContentConversions(nsIStreamListener* aNextListener,
       }
 
       LOG(("converter removed '%s' content-encoding\n", val));
+      if (gHttpHandler->IsTelemetryEnabled()) {
+        int mode = 0;
+        if (from.Equals("gzip") || from.Equals("x-gzip")) {
+          mode = 1;
+        } else if (from.Equals("deflate") || from.Equals("x-deflate")) {
+          mode = 2;
+        } else if (from.Equals("brotli")) {
+          mode = 3;
+        }
+        Telemetry::Accumulate(Telemetry::HTTP_CONTENT_ENCODING, mode);
+      }
       nextListener = converter;
     }
     else {
@@ -940,6 +951,14 @@ HttpBaseChannel::nsContentEncodings::GetNext(nsACString& aNextEncoding)
     }
   }
 
+  if (!haveType) {
+    encoding.BeginReading(start);
+    if (CaseInsensitiveFindInReadable(NS_LITERAL_CSTRING("brotli"), start, end)) {
+      aNextEncoding.AssignLiteral(APPLICATION_BROTLI);
+      haveType = true;
+    }
+  }
+
   // Prepare to fetch the next encoding
   mCurEnd = mCurStart;
   mReady = false;

netwerk/protocol/http/nsHttpHandler.cpp

@@ -493,22 +493,24 @@ nsHttpHandler::IsAcceptableEncoding(const char *enc, bool isSecure)
     if (!enc)
         return false;
 
-    // HTTP 1.1 allows servers to send x-gzip and x-compress instead
-    // of gzip and compress, for example.  So, we'll always strip off
-    // an "x-" prefix before matching the encoding to one we claim
-    // to accept.
-    if (!PL_strncasecmp(enc, "x-", 2))
-        enc += 2;
-
+    // we used to accept x-foo anytime foo was acceptable, but that's just
+    // continuing bad behavior.. so limit it to known x-* patterns
+    bool rv;
+    if (isSecure) {
+        rv = nsHttp::FindToken(mHttpsAcceptEncodings.get(), enc, HTTP_LWS ",") != nullptr;
+    } else {
+        rv = nsHttp::FindToken(mHttpAcceptEncodings.get(), enc, HTTP_LWS ",") != nullptr;
+    }
     // gzip and deflate are inherently acceptable in modern HTTP - always
     // process them if a stream converter can also be found.
-    if (!PL_strcasecmp(enc, "gzip") || !PL_strcasecmp(enc, "deflate"))
-        return true;
-
-    if (isSecure) {
-        return nsHttp::FindToken(mHttpsAcceptEncodings.get(), enc, HTTP_LWS ",") != nullptr;
+    if (!rv &&
+        (!PL_strcasecmp(enc, "gzip") || !PL_strcasecmp(enc, "deflate") ||
+         !PL_strcasecmp(enc, "x-gzip") || !PL_strcasecmp(enc, "x-deflate"))) {
+        rv = true;
     }
-    return nsHttp::FindToken(mHttpAcceptEncodings.get(), enc, HTTP_LWS ",") != nullptr;
+    LOG(("nsHttpHandler::IsAceptableEncoding %s https=%d %d\n",
+         enc, isSecure, rv));
+    return rv;
 }
 
 nsresult

netwerk/streamconv/converters/nsHTTPCompressConv.cpp

@@ -14,6 +14,14 @@
 #include "nsComponentManagerUtils.h"
 #include "nsThreadUtils.h"
 #include "mozilla/Preferences.h"
+#include "nsIForcePendingChannel.h"
+
+// brotli headers
+#include "state.h"
+#include "decode.h"
+
+extern PRLogModuleInfo *gHttpLog;
+#define LOG(args) MOZ_LOG(gHttpLog, mozilla::LogLevel::Debug, args)
 
 namespace mozilla {
 namespace net {
@@ -39,6 +47,7 @@ nsHTTPCompressConv::nsHTTPCompressConv()
   , mSkipCount(0)
   , mFlags(0)
 {
+  LOG(("nsHttpCompresssConv %p ctor\n", this));
   if (NS_IsMainThread()) {
     mFailUncleanStops =
       Preferences::GetBool("network.http.enforce-framing.http", false);
@@ -49,6 +58,7 @@ nsHTTPCompressConv::nsHTTPCompressConv()
 
 nsHTTPCompressConv::~nsHTTPCompressConv()
 {
+  LOG(("nsHttpCompresssConv %p dtor\n", this));
   if (mInpBuffer) {
     free(mInpBuffer);
   }
@@ -78,7 +88,11 @@ nsHTTPCompressConv::AsyncConvertData(const char *aFromType,
     mMode = HTTP_COMPRESS_GZIP;
   } else if (!PL_strncasecmp(aFromType, HTTP_DEFLATE_TYPE, sizeof(HTTP_DEFLATE_TYPE)-1)) {
     mMode = HTTP_COMPRESS_DEFLATE;
+  } else if (!PL_strncasecmp(aFromType, HTTP_BROTLI_TYPE, sizeof(HTTP_BROTLI_TYPE)-1)) {
+    mMode = HTTP_COMPRESS_BROTLI;
   }
+  LOG(("nsHttpCompresssConv %p AsyncConvertData %s %s mode %d\n",
+       this, aFromType, aToType, mMode));
 
   // hook ourself up with the receiving listener.
   mListener = aListener;
@@ -90,22 +104,115 @@ nsHTTPCompressConv::AsyncConvertData(const char *aFromType,
 NS_IMETHODIMP
 nsHTTPCompressConv::OnStartRequest(nsIRequest* request, nsISupports *aContext)
 {
+  LOG(("nsHttpCompresssConv %p onstart\n", this));
   return mListener->OnStartRequest(request, aContext);
 }
 
 NS_IMETHODIMP
 nsHTTPCompressConv::OnStopRequest(nsIRequest* request, nsISupports *aContext,
                                   nsresult aStatus)
 {
+  nsresult status = aStatus;
+  LOG(("nsHttpCompresssConv %p onstop %x\n", this, aStatus));
+
   // Framing integrity is enforced for content-encoding: gzip, but not for
   // content-encoding: deflate. Note that gzip vs deflate is NOT determined
   // by content sniffing but only via header.
-  if (!mStreamEnded && NS_SUCCEEDED(aStatus) &&
+  if (!mStreamEnded && NS_SUCCEEDED(status) &&
       (mFailUncleanStops && (mMode == HTTP_COMPRESS_GZIP)) ) {
     // This is not a clean end of gzip stream: the transfer is incomplete.
-    aStatus = NS_ERROR_NET_PARTIAL_TRANSFER;
+    status = NS_ERROR_NET_PARTIAL_TRANSFER;
+    LOG(("nsHttpCompresssConv %p onstop partial gzip\n", this));
+  }
+  if (NS_SUCCEEDED(status) && mMode == HTTP_COMPRESS_BROTLI) {
+    uint32_t waste;
+    nsCOMPtr<nsIForcePendingChannel> fpChannel = do_QueryInterface(request);
+    bool isPending = false;
+    if (request) {
+      request->IsPending(&isPending);
+    }
+    if (fpChannel && !isPending) {
+      fpChannel->ForcePending(true);
+    }
+    status = BrotliHandler(nullptr, this, nullptr, 0, 0, &waste);
+    LOG(("nsHttpCompresssConv %p onstop brotlihandler rv %x\n", this, status));
+    if (fpChannel && !isPending) {
+      fpChannel->ForcePending(false);
+    }
   }
-  return mListener->OnStopRequest(request, aContext, aStatus);
+  if (NS_FAILED(status) && status != aStatus) {
+    LOG(("nsHttpCompresssConv %p onstop calling cancel %x\n", this, status));
+    request->Cancel(status);
+  }
+  return mListener->OnStopRequest(request, aContext, status);
+}
+
+
+// static
+NS_METHOD
+nsHTTPCompressConv::BrotliHandler(nsIInputStream *stream, void *closure, const char *dataIn,
+                                  uint32_t, uint32_t aAvail, uint32_t *countRead)
+{
+  nsHTTPCompressConv *self = static_cast<nsHTTPCompressConv *>(closure);
+  *countRead = 0;
+
+  const uint32_t kOutSize = 128 * 1024; // just a chunk size, we call in a loop
+  unsigned char outBuffer[kOutSize];
+  unsigned char *outPtr;
+  size_t outSize;
+  size_t avail = aAvail;
+  BrotliResult res;
+
+  do {
+    outSize = kOutSize;
+    outPtr = outBuffer;
+
+    // brotli api is documented in brotli/dec/decode.h
+    LOG(("nsHttpCompresssConv %p brotlihandler decompress %d finish %d\n",
+         self, avail, !stream));
+    res = ::BrotliDecompressBufferStreaming(
+      &avail, reinterpret_cast<const unsigned char **>(&dataIn), stream ? 0 : 1,
+      &outSize, &outPtr, &self->mBrotli->mTotalOut, &self->mBrotli->mState);
+    outSize = kOutSize - outSize;
+    LOG(("nsHttpCompresssConv %p brotlihandler decompress rv=%x out=%d\n",
+         self, res, outSize));
+
+    if (res == BROTLI_RESULT_ERROR) {
+      LOG(("nsHttpCompressConv %p marking invalid encoding", self));
+      self->mBrotli->mStatus = NS_ERROR_INVALID_CONTENT_ENCODING;
+      return self->mBrotli->mStatus;
+    }
+
+    // in 'the current implementation' brotli consumes all input on success
+    MOZ_ASSERT(!avail);
+    if (avail) {
+      LOG(("nsHttpCompressConv %p did not consume all input", self));
+      self->mBrotli->mStatus = NS_ERROR_UNEXPECTED;
+      return self->mBrotli->mStatus;
+    }
+    if (outSize > 0) {
+      nsresult rv = self->do_OnDataAvailable(self->mBrotli->mRequest,
+                                             self->mBrotli->mContext,
+                                             self->mBrotli->mSourceOffset,
+                                             reinterpret_cast<const char *>(outBuffer),
+                                             outSize);
+      LOG(("nsHttpCompressConv %p BrotliHandler ODA rv=%x", self, rv));
+      if (NS_FAILED(rv)) {
+        self->mBrotli->mStatus = rv;
+        return self->mBrotli->mStatus;
+      }
+    }
+
+    if (res == BROTLI_RESULT_SUCCESS ||
+        res == BROTLI_RESULT_NEEDS_MORE_INPUT) {
+      *countRead = aAvail;
+      return NS_OK;
+    }
+    MOZ_ASSERT (res == BROTLI_RESULT_NEEDS_MORE_OUTPUT);
+  } while (res == BROTLI_RESULT_NEEDS_MORE_OUTPUT);
+
+  self->mBrotli->mStatus = NS_ERROR_UNEXPECTED;
+  return self->mBrotli->mStatus;
 }
 
 NS_IMETHODIMP
@@ -117,6 +224,7 @@ nsHTTPCompressConv::OnDataAvailable(nsIRequest* request,
 {
   nsresult rv = NS_ERROR_INVALID_CONTENT_ENCODING;
   uint32_t streamLen = aCount;
+  LOG(("nsHttpCompressConv %p OnDataAvailable %d", this, aCount));
 
   if (streamLen == 0) {
     NS_ERROR("count of zero passed to OnDataAvailable");
@@ -306,6 +414,27 @@ nsHTTPCompressConv::OnDataAvailable(nsIRequest* request,
     } /* gzip */
     break;
 
+  case HTTP_COMPRESS_BROTLI:
+  {
+    if (!mBrotli) {
+      mBrotli = new BrotliWrapper();
+    }
+
+    mBrotli->mRequest = request;
+    mBrotli->mContext = aContext;
+    mBrotli->mSourceOffset = aSourceOffset;
+
+    uint32_t countRead;
+    rv = iStr->ReadSegments(BrotliHandler, this, streamLen, &countRead);
+    if (NS_SUCCEEDED(rv)) {
+      rv = mBrotli->mStatus;
+    }
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
+  }
+    break;
+
   default:
     rv = mListener->OnDataAvailable(request, aContext, iStr, aSourceOffset, aCount);
     if (NS_FAILED (rv)) {
@@ -316,7 +445,6 @@ nsHTTPCompressConv::OnDataAvailable(nsIRequest* request,
   return NS_OK;
 } /* OnDataAvailable */
 
-
 // XXX/ruslan: need to implement this too
 
 NS_IMETHODIMP

netwerk/streamconv/converters/nsHTTPCompressConv.h

@@ -12,6 +12,11 @@
 
 #include "zlib.h"
 
+// brotli includes
+#undef assert
+#include "assert.h"
+#include "state.h"
+
 class nsIStringInputStream;
 
 #define NS_HTTPCOMPRESSCONVERTER_CID                    \
@@ -29,6 +34,7 @@ class nsIStringInputStream;
 #define	HTTP_X_GZIP_TYPE	    "x-gzip"
 #define	HTTP_COMPRESS_TYPE	    "compress"
 #define	HTTP_X_COMPRESS_TYPE	"x-compress"
+#define	HTTP_BROTLI_TYPE        "brotli"
 #define	HTTP_IDENTITY_TYPE	    "identity"
 #define	HTTP_UNCOMPRESSED_TYPE	"uncompressed"
 
@@ -39,9 +45,33 @@ typedef enum    {
   HTTP_COMPRESS_GZIP,
   HTTP_COMPRESS_DEFLATE,
   HTTP_COMPRESS_COMPRESS,
+  HTTP_COMPRESS_BROTLI,
   HTTP_COMPRESS_IDENTITY
 } CompressMode;
 
+class BrotliWrapper
+{
+public:
+  BrotliWrapper()
+    : mTotalOut(0)
+    , mStatus(NS_OK)
+  {
+    BrotliStateInit(&mState);
+  }
+  ~BrotliWrapper()
+  {
+    BrotliStateCleanup(&mState);
+  }
+
+  BrotliState mState;
+  size_t       mTotalOut;
+  nsresult     mStatus;
+
+  nsIRequest  *mRequest;
+  nsISupports *mContext;
+  uint64_t     mSourceOffset;
+};
+
 class nsHTTPCompressConv : public nsIStreamConverter	{
   public:
   // nsISupports methods
@@ -66,9 +96,15 @@ class nsHTTPCompressConv : public nsIStreamConverter	{
     uint32_t	mOutBufferLen;
     uint32_t	mInpBufferLen;
 
+    nsAutoPtr<BrotliWrapper> mBrotli;
+
     nsCOMPtr<nsISupports>   mAsyncConvContext;
     nsCOMPtr<nsIStringInputStream>  mStream;
 
+    static NS_METHOD
+    BrotliHandler(nsIInputStream *stream, void *closure, const char *dataIn,
+                  uint32_t, uint32_t avail, uint32_t *countRead);
+
     nsresult do_OnDataAvailable (nsIRequest *request, nsISupports *aContext,
                                  uint64_t aSourceOffset, const char *buffer,
                                  uint32_t aCount);