http: add overload action to disable http keepalive (envoyproxy#4497)

Signed-off-by: Elisha Ziskind <[email protected]>

docs/root/configuration/http_conn_man/stats.rst

@@ -34,6 +34,7 @@ statistics:
    downstream_cx_tx_bytes_buffered, Gauge, Total sent bytes currently buffered
    downstream_cx_drain_close, Counter, Total connections closed due to draining
    downstream_cx_idle_timeout, Counter, Total connections closed due to idle timeout
+   downstream_cx_overload_disable_keepalive, Counter, Total connections for which HTTP 1.x keepalive has been disabled due to envoy overload
    downstream_flow_control_paused_reading_total, Counter, Total number of times reads were disabled due to flow control
    downstream_flow_control_resumed_reading_total, Counter, Total number of times reads were enabled on the connection due to flow control
    downstream_rq_total, Counter, Total requests

docs/root/configuration/overload_manager/overload_manager.rst

@@ -7,6 +7,31 @@ The :ref:`overload manager <arch_overview_overload_manager>` is configured in th
 :ref:`overload_manager <envoy_api_field_config.bootstrap.v2.Bootstrap.overload_manager>`
 field.
 
+An example configuration of the overload manager is shown below. It shows a configuration to
+disable HTTP/1.x keepalive when heap memory usage reaches 95% and to stop accepting
+requests when heap memory usage reaches 99%.
+
+.. code-block:: yaml
+
+   refresh_interval:
+     seconds: 0
+     nanos: 250000000
+   resource_monitors:
+     - name: "envoy.resource_monitors.fixed_heap"
+       config:
+         max_heap_size_bytes: 2147483648
+   actions:
+     - name: "envoy.overload_actions.disable_http_keepalive"
+       triggers:
+         - name: "envoy.resource_monitors.fixed_heap"
+           threshold:
+             value: 0.95
+     - name: "envoy.overload_actions.stop_accepting_requests"
+       triggers:
+         - name: "envoy.resource_monitors.fixed_heap"
+           threshold:
+             value: 0.99
+
 Resource monitors
 -----------------
 
@@ -24,6 +49,7 @@ The following overload actions are supported:
   :widths: 1, 2
 
   envoy.overload_actions.stop_accepting_requests, Envoy will immediately respond with a 503 response code to new requests
+  envoy.overload_actions.disable_http_keepalive, Envoy will disable keepalive on HTTP/1.x responses
 
 Statistics
 ----------

include/envoy/server/overload_manager.h

@@ -60,6 +60,9 @@ class OverloadActionNameValues {
 public:
   // Overload action to stop accepting new requests.
   const std::string StopAcceptingRequests = "envoy.overload_actions.stop_accepting_requests";
+
+  // Overload action to disable http keepalive (for HTTP1.x).
+  const std::string DisableHttpKeepAlive = "envoy.overload_actions.disable_http_keepalive";
 };
 
 typedef ConstSingleton<OverloadActionNameValues> OverloadActionNames;

source/common/http/conn_manager_config.h

@@ -38,6 +38,7 @@ namespace Http {
   GAUGE    (downstream_cx_tx_bytes_buffered)                                                       \
   COUNTER  (downstream_cx_drain_close)                                                             \
   COUNTER  (downstream_cx_idle_timeout)                                                            \
+  COUNTER  (downstream_cx_overload_disable_keepalive)                                              \
   COUNTER  (downstream_cx_delayed_close_timeout)                                                   \
   COUNTER  (downstream_flow_control_paused_reading_total)                                          \
   COUNTER  (downstream_flow_control_resumed_reading_total)                                         \

source/common/http/conn_manager_impl.cc

@@ -66,10 +66,14 @@ ConnectionManagerImpl::ConnectionManagerImpl(ConnectionManagerConfig& config,
       drain_close_(drain_close), random_generator_(random_generator), tracer_(tracer),
       runtime_(runtime), local_info_(local_info), cluster_manager_(cluster_manager),
       listener_stats_(config_.listenerStats()),
-      overload_stop_accepting_requests_(
+      overload_stop_accepting_requests_ref_(
           overload_manager ? overload_manager->getThreadLocalOverloadState().getState(
                                  Server::OverloadActionNames::get().StopAcceptingRequests)
                            : Server::OverloadManager::getInactiveState()),
+      overload_disable_keepalive_ref_(
+          overload_manager ? overload_manager->getThreadLocalOverloadState().getState(
+                                 Server::OverloadActionNames::get().DisableHttpKeepAlive)
+                           : Server::OverloadManager::getInactiveState()),
       time_system_(time_system) {}
 
 const HeaderMapImpl& ConnectionManagerImpl::continueHeader() {
@@ -501,7 +505,7 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(HeaderMapPtr&& headers,
   }
 
   // Drop new requests when overloaded as soon as we have decoded the headers.
-  if (connection_manager_.overload_stop_accepting_requests_ ==
+  if (connection_manager_.overload_stop_accepting_requests_ref_ ==
       Server::OverloadActionState::Active) {
     connection_manager_.stats_.named_.downstream_rq_overload_close_.inc();
     sendLocalReply(Grpc::Common::hasGrpcContentType(*request_headers_),
@@ -1058,6 +1062,13 @@ void ConnectionManagerImpl::ActiveStream::encodeHeaders(ActiveStreamEncoderFilte
     connection_manager_.drain_state_ = DrainState::Closing;
   }
 
+  if (connection_manager_.drain_state_ == DrainState::NotDraining &&
+      connection_manager_.overload_disable_keepalive_ref_ == Server::OverloadActionState::Active) {
+    ENVOY_STREAM_LOG(debug, "disabling keepalive due to envoy overload", *this);
+    connection_manager_.drain_state_ = DrainState::Closing;
+    connection_manager_.stats_.named_.downstream_cx_overload_disable_keepalive_.inc();
+  }
+
   // If we are destroying a stream before remote is complete and the connection does not support
   // multiplexing, we should disconnect since we don't want to wait around for the request to
   // finish.
@@ -1072,7 +1083,7 @@ void ConnectionManagerImpl::ActiveStream::encodeHeaders(ActiveStreamEncoderFilte
   if (connection_manager_.drain_state_ == DrainState::Closing &&
       connection_manager_.codec_->protocol() != Protocol::Http2) {
     // If the connection manager is draining send "Connection: Close" on HTTP/1.1 connections.
-    // Do not do this for H2 (which drains via GOAWA) or Upgrade (as the upgrade
+    // Do not do this for H2 (which drains via GOAWAY) or Upgrade (as the upgrade
     // payload is no longer HTTP/1.1)
     if (!Utility::isUpgrade(headers)) {
       headers.insertConnection().value().setReference(Headers::get().ConnectionValues.Close);

source/common/http/conn_manager_impl.h

@@ -460,7 +460,10 @@ class ConnectionManagerImpl : Logger::Loggable<Logger::Id::http>,
   WebSocketProxyPtr ws_connection_;
   Network::ReadFilterCallbacks* read_callbacks_{};
   ConnectionManagerListenerStats& listener_stats_;
-  const Server::OverloadActionState& overload_stop_accepting_requests_;
+  // References into the overload manager thread local state map. Using these lets us avoid a map
+  // lookup in the hot path of processing each request.
+  const Server::OverloadActionState& overload_stop_accepting_requests_ref_;
+  const Server::OverloadActionState& overload_disable_keepalive_ref_;
   Event::TimeSystem& time_system_;
 };
 

test/common/http/conn_manager_impl_test.cc

@@ -3311,5 +3311,39 @@ TEST_F(HttpConnectionManagerImplTest, NoNewStreamWhenOverloaded) {
   EXPECT_EQ(1U, stats_.named_.downstream_rq_overload_close_.value());
 }
 
+TEST_F(HttpConnectionManagerImplTest, DisableKeepAliveWhenOverloaded) {
+  setup(false, "");
+
+  overload_manager_.overload_state_.setState(
+      Server::OverloadActionNames::get().DisableHttpKeepAlive, Server::OverloadActionState::Active);
+
+  std::shared_ptr<MockStreamDecoderFilter> filter(new NiceMock<MockStreamDecoderFilter>());
+  EXPECT_CALL(filter_factory_, createFilterChain(_))
+      .WillOnce(Invoke([&](FilterChainFactoryCallbacks& callbacks) -> void {
+        callbacks.addStreamDecoderFilter(StreamDecoderFilterSharedPtr{filter});
+      }));
+
+  EXPECT_CALL(*codec_, dispatch(_)).WillRepeatedly(Invoke([&](Buffer::Instance& data) -> void {
+    StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_);
+    HeaderMapPtr headers{new TestHeaderMapImpl{
+        {":authority", "host"}, {":path", "/"}, {":method", "GET"}, {"connection", "keep-alive"}}};
+    decoder->decodeHeaders(std::move(headers), true);
+
+    HeaderMapPtr response_headers{new TestHeaderMapImpl{{":status", "200"}}};
+    filter->callbacks_->encodeHeaders(std::move(response_headers), true);
+
+    data.drain(4);
+  }));
+
+  EXPECT_CALL(response_encoder_, encodeHeaders(_, true))
+      .WillOnce(Invoke([](const HeaderMap& headers, bool) -> void {
+        EXPECT_STREQ("close", headers.Connection()->value().c_str());
+      }));
+
+  Buffer::OwnedImpl fake_input("1234");
+  conn_manager_->onData(fake_input, false);
+  EXPECT_EQ(1U, stats_.named_.downstream_cx_overload_disable_keepalive_.value());
+}
+
 } // namespace Http
 } // namespace Envoy

test/integration/overload_integration_test.cc

@@ -26,6 +26,11 @@ class OverloadIntegrationTest : public HttpProtocolIntegrationTest {
               - name: "envoy.resource_monitors.injected_resource"
                 threshold:
                   value: 0.9
+          - name: "envoy.overload_actions.disable_http_keepalive"
+            triggers:
+              - name: "envoy.resource_monitors.injected_resource"
+                threshold:
+                  value: 0.8
       )EOF",
                                                       injected_resource_filename_);
       *bootstrap.mutable_overload_manager() =
@@ -79,4 +84,38 @@ TEST_P(OverloadIntegrationTest, CloseStreamsWhenOverloaded) {
   EXPECT_EQ(0U, response->body().size());
 }
 
+TEST_P(OverloadIntegrationTest, DisableKeepaliveWhenOverloaded) {
+  if (downstreamProtocol() != Http::CodecClient::Type::HTTP1) {
+    return; // only relevant for downstream HTTP1.x connections
+  }
+
+  initialize();
+  fake_upstreams_[0]->set_allow_unexpected_disconnects(true);
+
+  // Put envoy in overloaded state and check that it disables keepalive
+  updateResource(0.8);
+  test_server_->waitForGaugeEq("overload.envoy.overload_actions.disable_http_keepalive.active", 1);
+
+  codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http"))));
+  Http::TestHeaderMapImpl request_headers{
+      {":method", "GET"}, {":path", "/test/long/url"}, {":scheme", "http"}, {":authority", "host"}};
+  auto response = sendRequestAndWaitForResponse(request_headers, 1, default_response_headers_, 1);
+  codec_client_->waitForDisconnect();
+
+  EXPECT_TRUE(response->complete());
+  EXPECT_STREQ("200", response->headers().Status()->value().c_str());
+  EXPECT_STREQ("close", response->headers().Connection()->value().c_str());
+
+  // Deactivate overload state and check that keepalive is not disabled
+  updateResource(0.7);
+  test_server_->waitForGaugeEq("overload.envoy.overload_actions.disable_http_keepalive.active", 0);
+
+  codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http"))));
+  response = sendRequestAndWaitForResponse(request_headers, 1, default_response_headers_, 1);
+
+  EXPECT_TRUE(response->complete());
+  EXPECT_STREQ("200", response->headers().Status()->value().c_str());
+  EXPECT_EQ(nullptr, response->headers().Connection());
+}
+
 } // namespace Envoy