Stream: speed up TCP peer recovery.

Previously, an unavailable peer was considered recovered after a successful
proxy session to this peer.  Until then, only a single client connection per
fail_timeout was allowed to be proxied to the peer.

Since stream sessions can be long, it may take indefinite time for a peer to
recover, limiting the ability of the peer to receive new connections.

Now, a peer is considered recovered after a successful TCP connection is
established to it.  Balancers are notified of this event via the notify()
callback.

src/stream/ngx_stream_proxy_module.c

@@ -808,6 +808,11 @@ ngx_stream_proxy_init_upstream(ngx_stream_session_t *s)
 
     u->state->connect_time = ngx_current_msec - u->state->response_time;
 
+    if (u->peer.notify) {
+        u->peer.notify(&u->peer, u->peer.data,
+                       NGX_STREAM_UPSTREAM_NOTIFY_CONNECT);
+    }
+
     c->log->action = "proxying connection";
 
     if (u->upstream_buf.start == NULL) {

src/stream/ngx_stream_upstream.h

@@ -24,6 +24,9 @@
 #define NGX_STREAM_UPSTREAM_MAX_CONNS     0x0100
 
 
+#define NGX_STREAM_UPSTREAM_NOTIFY_CONNECT     0x1
+
+
 typedef struct {
     ngx_array_t                        upstreams;
                                            /* ngx_stream_upstream_srv_conf_t */

src/stream/ngx_stream_upstream_round_robin.c

@@ -16,6 +16,8 @@
 
 static ngx_stream_upstream_rr_peer_t *ngx_stream_upstream_get_peer(
     ngx_stream_upstream_rr_peer_data_t *rrp);
+static void ngx_stream_upstream_notify_round_robin_peer(
+    ngx_peer_connection_t *pc, void *data, ngx_uint_t state);
 
 #if (NGX_STREAM_SSL)
 
@@ -288,6 +290,7 @@ ngx_stream_upstream_init_round_robin_peer(ngx_stream_session_t *s,
 
     s->upstream->peer.get = ngx_stream_upstream_get_round_robin_peer;
     s->upstream->peer.free = ngx_stream_upstream_free_round_robin_peer;
+    s->upstream->peer.notify = ngx_stream_upstream_notify_round_robin_peer;
     s->upstream->peer.tries = ngx_stream_upstream_tries(rrp->peers);
 #if (NGX_STREAM_SSL)
     s->upstream->peer.set_session =
@@ -659,6 +662,32 @@ ngx_stream_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
 }
 
 
+static void
+ngx_stream_upstream_notify_round_robin_peer(ngx_peer_connection_t *pc,
+    void *data, ngx_uint_t type)
+{
+    ngx_stream_upstream_rr_peer_data_t  *rrp = data;
+
+    ngx_stream_upstream_rr_peer_t  *peer;
+
+    peer = rrp->current;
+
+    if (type == NGX_STREAM_UPSTREAM_NOTIFY_CONNECT
+        && pc->connection->type == SOCK_STREAM)
+    {
+        ngx_stream_upstream_rr_peers_rlock(rrp->peers);
+        ngx_stream_upstream_rr_peer_lock(rrp->peers, peer);
+
+        if (peer->accessed < peer->checked) {
+            peer->fails = 0;
+        }
+
+        ngx_stream_upstream_rr_peer_unlock(rrp->peers, peer);
+        ngx_stream_upstream_rr_peers_unlock(rrp->peers);
+    }
+}
+
+
 #if (NGX_STREAM_SSL)
 
 static ngx_int_t