compression: add brotli compressor and decompressor (envoyproxy#12998)

Commit Message: compression: add brotli compressor and decompressor
Additional Description: Add new brotli compression extensions in addition to gzip.
Risk Level: Low, no existing functionality is touched
Testing: uni tests, manual tests with curl.
Docs Changes: updated docs for compression and decompression HTTP filters to refer the new available encoder/decoder.
Release Notes: updated current.rst
Fixes envoyproxy#4429

The PR adds a new dependency on https://github.com/google/brotli. Here's the current criteria answers:

| Criteria | Answer |
|---------|---------|
| Cloud Native Computing Foundation (CNCF) approved license | MIT |
| Dependencies must not substantially increase the binary size unless they are optional | brotli's binary size built with `-c opt` is 752K |
| No duplication of existing dependencies | no other dep provides Brotli |
| Hosted on a git repository and the archive fetch must directly reference this repository. | https://github.com/google/brotli |
| CVE history appears reasonable, no pathological CVE arcs | so far 4 CVEs related to brotli have been registered |
| Code review (ideally PRs) before merge | PRs are reviewed before merge |
| Security vulnerability process exists, with contact details and reporting/disclosure process | no policy exists, submitted google/brotli#878 |
| > 1 contributor responsible for a non-trivial number of commits | 75 contributors |
| Tests run in CI | CI set up with AppVeyor and Github actions |
| High test coverage (also static/dynamic analysis, fuzzing) | Fuzzers are run in CI |
| Envoy can obtain advanced notification of vulnerabilities or of security releases | brotli is registered in CPE |
| Do other significant projects have shared fate by using this dependency? | Google Chrome is using the library |
| Releases (with release notes) | https://github.com/google/brotli/releases |
| Commits/releases in last 90 days | last commit 9 days ago |

Signed-off-by: Dmitry Rozhkov <[email protected]>

CODEOWNERS

@@ -147,6 +147,7 @@ extensions/filters/common/original_src @snowp @klarose
 # Compression
 /*/extensions/compression/common @junr03 @rojkov
 /*/extensions/compression/gzip @junr03 @rojkov
+/*/extensions/compression/brotli @junr03 @rojkov
 /*/extensions/filters/http/decompressor @rojkov @dio
 # Watchdog Extensions
 /*/extensions/watchdog/profile_action @kbaichoo @antoniovicente

api/BUILD

@@ -165,6 +165,8 @@ proto_library(
         "//envoy/extensions/common/matching/v3:pkg",
         "//envoy/extensions/common/ratelimit/v3:pkg",
         "//envoy/extensions/common/tap/v3:pkg",
+        "//envoy/extensions/compression/brotli/compressor/v3:pkg",
+        "//envoy/extensions/compression/brotli/decompressor/v3:pkg",
         "//envoy/extensions/compression/gzip/compressor/v3:pkg",
         "//envoy/extensions/compression/gzip/decompressor/v3:pkg",
         "//envoy/extensions/filters/common/dependency/v3:pkg",

api/envoy/extensions/compression/brotli/compressor/v3/BUILD

@@ -0,0 +1,9 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"],
+)

api/envoy/extensions/compression/brotli/compressor/v3/brotli.proto

@@ -0,0 +1,54 @@
+syntax = "proto3";
+
+package envoy.extensions.compression.brotli.compressor.v3;
+
+import "google/protobuf/wrappers.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.compression.brotli.compressor.v3";
+option java_outer_classname = "BrotliProto";
+option java_multiple_files = true;
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Brotli Compressor]
+// [#extension: envoy.compression.brotli.compressor]
+
+// [#next-free-field: 7]
+message Brotli {
+  enum EncoderMode {
+    DEFAULT = 0;
+    GENERIC = 1;
+    TEXT = 2;
+    FONT = 3;
+  }
+
+  // Value from 0 to 11 that controls the main compression speed-density lever.
+  // The higher quality, the slower compression. The default value is 3.
+  google.protobuf.UInt32Value quality = 1 [(validate.rules).uint32 = {lte: 11}];
+
+  // A value used to tune encoder for specific input. For more information about modes,
+  // please refer to brotli manual: https://brotli.org/encode.html#aa6f
+  // This field will be set to "DEFAULT" if not specified.
+  EncoderMode encoder_mode = 2 [(validate.rules).enum = {defined_only: true}];
+
+  // Value from 10 to 24 that represents the base two logarithmic of the compressor's window size.
+  // Larger window results in better compression at the expense of memory usage. The default is 18.
+  // For more details about this parameter, please refer to brotli manual:
+  // https://brotli.org/encode.html#a9a8
+  google.protobuf.UInt32Value window_bits = 3 [(validate.rules).uint32 = {lte: 24 gte: 10}];
+
+  // Value from 16 to 24 that represents the base two logarithmic of the compressor's input block
+  // size. Larger input block results in better compression at the expense of memory usage. The
+  // default is 24. For more details about this parameter, please refer to brotli manual:
+  // https://brotli.org/encode.html#a9a8
+  google.protobuf.UInt32Value input_block_bits = 4 [(validate.rules).uint32 = {lte: 24 gte: 16}];
+
+  // Value for compressor's next output buffer. If not set, defaults to 4096.
+  google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}];
+
+  // If true, disables "literal context modeling" format feature.
+  // This flag is a "decoding-speed vs compression ratio" trade-off.
+  bool disable_literal_context_modeling = 6;
+}

api/envoy/extensions/compression/brotli/decompressor/v3/BUILD

@@ -0,0 +1,9 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"],
+)

api/envoy/extensions/compression/brotli/decompressor/v3/brotli.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+package envoy.extensions.compression.brotli.decompressor.v3;
+
+import "google/protobuf/wrappers.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.compression.brotli.decompressor.v3";
+option java_outer_classname = "BrotliProto";
+option java_multiple_files = true;
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Brotli Decompressor]
+// [#extension: envoy.compression.brotli.decompressor]
+
+message Brotli {
+  // If true, disables "canny" ring buffer allocation strategy.
+  // Ring buffer is allocated according to window size, despite the real size of the content.
+  bool disable_ring_buffer_reallocation = 1;
+
+  // Value for decompressor's next output buffer. If not set, defaults to 4096.
+  google.protobuf.UInt32Value chunk_size = 2 [(validate.rules).uint32 = {lte: 65536 gte: 4096}];
+}

api/versioning/BUILD

@@ -48,6 +48,8 @@ proto_library(
         "//envoy/extensions/common/matching/v3:pkg",
         "//envoy/extensions/common/ratelimit/v3:pkg",
         "//envoy/extensions/common/tap/v3:pkg",
+        "//envoy/extensions/compression/brotli/compressor/v3:pkg",
+        "//envoy/extensions/compression/brotli/decompressor/v3:pkg",
         "//envoy/extensions/compression/gzip/compressor/v3:pkg",
         "//envoy/extensions/compression/gzip/decompressor/v3:pkg",
         "//envoy/extensions/filters/common/dependency/v3:pkg",

bazel/repositories.bzl

@@ -158,6 +158,7 @@ def envoy_dependencies(skip_targets = []):
     _io_opentracing_cpp()
     _net_zlib()
     _com_github_zlib_ng_zlib_ng()
+    _org_brotli()
     _upb()
     _proxy_wasm_cpp_sdk()
     _proxy_wasm_cpp_host()
@@ -352,6 +353,19 @@ def _com_github_zlib_ng_zlib_ng():
         patches = ["@envoy//bazel/foreign_cc:zlib_ng.patch"],
     )
 
+def _org_brotli():
+    external_http_archive(
+        name = "org_brotli",
+    )
+    native.bind(
+        name = "brotlienc",
+        actual = "@org_brotli//:brotlienc",
+    )
+    native.bind(
+        name = "brotlidec",
+        actual = "@org_brotli//:brotlidec",
+    )
+
 def _com_google_cel_cpp():
     external_http_archive("com_google_cel_cpp")
     external_http_archive("rules_antlr")

bazel/repository_locations.bzl

@@ -406,6 +406,24 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         release_date = "2019-04-14",
         cpe = "cpe:2.3:a:gnu:zlib:*",
     ),
+    org_brotli = dict(
+        project_name = "brotli",
+        project_desc = "brotli compression library",
+        project_url = "https://brotli.org",
+        # Use the dev branch of brotli to resolve compilation issues.
+        # TODO(rojkov): Remove when brotli > 1.0.9 is released.
+        version = "0cd2e3926e95e7e2930f57ae3f4885508d462a25",
+        sha256 = "93810780e60304b51f2c9645fe313a6e4640711063ed0b860cfa60999dd256c5",
+        strip_prefix = "brotli-{version}",
+        urls = ["https://github.com/google/brotli/archive/{version}.tar.gz"],
+        use_category = ["dataplane_ext"],
+        extensions = [
+            "envoy.compression.brotli.compressor",
+            "envoy.compression.brotli.decompressor",
+        ],
+        release_date = "2020-09-08",
+        cpe = "cpe:2.3:a:google:brotli:*",
+    ),
     com_github_zlib_ng_zlib_ng = dict(
         project_name = "zlib-ng",
         project_desc = "zlib fork (higher performance)",

docs/root/api-v3/config/compression/compression.rst

@@ -6,3 +6,4 @@ Compression
   :maxdepth: 2
 
   ../../extensions/compression/gzip/*/v3/*
+  ../../extensions/compression/brotli/*/v3/*

docs/root/configuration/http/http_filters/compressor_filter.rst

@@ -23,8 +23,9 @@ determine whether or not the content should be compressed. The content is
 compressed and then sent to the client with the appropriate headers, if
 response and request allow.
 
-Currently the filter supports :ref:`gzip compression <envoy_v3_api_msg_extensions.compression.gzip.compressor.v3.Gzip>`
-only. Other compression libraries can be supported as extensions.
+Currently the filter supports :ref:`gzip <envoy_v3_api_msg_extensions.compression.gzip.compressor.v3.Gzip>`
+and :ref:`brotli <envoy_v3_api_msg_extensions.compression.brotli.compressor.v3.Brotli>`
+compression only. Other compression libraries can be supported as extensions.
 
 An example configuration of the filter may look like the following:
 

docs/root/configuration/http/http_filters/decompressor_filter.rst

@@ -16,8 +16,9 @@ determine whether or not the content should be decompressed. The content is
 decompressed and passed on to the rest of the filter chain. Note that decompression happens
 independently for request and responses based on the rules described below.
 
-Currently the filter supports :ref:`gzip compression <envoy_v3_api_msg_extensions.compression.gzip.decompressor.v3.Gzip>`
-only. Other compression libraries can be supported as extensions.
+Currently the filter supports :ref:`gzip <envoy_v3_api_msg_extensions.compression.gzip.decompressor.v3.Gzip>`
+and :ref:`brotli <envoy_v3_api_msg_extensions.compression.brotli.decompressor.v3.Brotli>`
+compression only. Other compression libraries can be supported as extensions.
 
 An example configuration of the filter may look like the following:
 

docs/root/intro/arch_overview/other_features/compression/libraries.rst

@@ -6,7 +6,8 @@ Compression Libraries
 Underlying implementation
 -------------------------
 
-Currently Envoy uses `zlib <http://zlib.net>`_ as a compression library.
+Currently Envoy uses `zlib <http://zlib.net>`_ and `brotli <https://brotli.org>`_ as compression
+libraries.
 
 .. note::
 

docs/root/version_history/current.rst

@@ -73,6 +73,7 @@ New Features
 ------------
 * access log: added the :ref:`formatters <envoy_v3_api_field_config.core.v3.SubstitutionFormatString.formatters>` extension point for custom formatters (command operators).
 * access log: support command operator: %REQUEST_HEADERS_BYTES%, %RESPONSE_HEADERS_BYTES%, and %RESPONSE_TRAILERS_BYTES%.
+* compression: add brotli :ref:`compressor <envoy_v3_api_msg_extensions.compression.brotli.compressor.v3.Brotli>` and :ref:`decompressor <envoy_v3_api_msg_extensions.compression.brotli.decompressor.v3.Brotli>`.
 * config: add `envoy.features.fail_on_any_deprecated_feature` runtime key, which matches the behaviour of compile-time flag `ENVOY_DISABLE_DEPRECATED_FEATURES`, i.e. use of deprecated fields will cause a crash.
 * dispatcher: supports a stack of `Envoy::ScopeTrackedObject` instead of a single tracked object. This will allow Envoy to dump more debug information on crash.
 * grpc_json_transcoder: added option :ref:`strict_http_request_validation <envoy_v3_api_field_extensions.filters.http.grpc_json_transcoder.v3.GrpcJsonTranscoder.strict_http_request_validation>` to reject invalid requests early.

source/common/http/headers.h

@@ -96,6 +96,7 @@ class CustomHeaderValues {
   } CacheControlValues;
 
   struct {
+    const std::string Brotli{"br"};
     const std::string Gzip{"gzip"};
   } ContentEncodingValues;
 

source/extensions/compression/brotli/common/BUILD

@@ -0,0 +1,18 @@
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_library",
+    "envoy_extension_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_extension_package()
+
+envoy_cc_library(
+    name = "brotli_base_lib",
+    srcs = ["base.cc"],
+    hdrs = ["base.h"],
+    deps = [
+        "//source/common/buffer:buffer_lib",
+    ],
+)

source/extensions/compression/brotli/common/base.cc

@@ -0,0 +1,36 @@
+#include "extensions/compression/brotli/common/base.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Compression {
+namespace Brotli {
+namespace Common {
+
+BrotliContext::BrotliContext(const uint32_t chunk_size)
+    : chunk_size_{chunk_size}, chunk_ptr_{std::make_unique<uint8_t[]>(chunk_size)}, next_in_{},
+      next_out_{chunk_ptr_.get()}, avail_in_{0}, avail_out_{chunk_size} {}
+
+void BrotliContext::updateOutput(Buffer::Instance& output_buffer) {
+  if (avail_out_ == 0) {
+    output_buffer.add(static_cast<void*>(chunk_ptr_.get()), chunk_size_);
+    resetOut();
+  }
+}
+
+void BrotliContext::finalizeOutput(Buffer::Instance& output_buffer) {
+  const size_t n_output = chunk_size_ - avail_out_;
+  if (n_output > 0) {
+    output_buffer.add(static_cast<void*>(chunk_ptr_.get()), n_output);
+  }
+}
+
+void BrotliContext::resetOut() {
+  avail_out_ = chunk_size_;
+  next_out_ = chunk_ptr_.get();
+}
+
+} // namespace Common
+} // namespace Brotli
+} // namespace Compression
+} // namespace Extensions
+} // namespace Envoy