[HTTP/3] Support for HTTP/3 multiple connections (dotnet#101531)

* QuicConnection StreamsAvailable properties and event handler

* EnableHttp3MultipleConnections property

* H3 connection pool

* Fixed unused methods.

* Update src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http3Connection.cs

Co-authored-by: Radek Zikmund <[email protected]>

* Fix release reserved stream count once it's opened.

* Available streams count is thread safe

* Move StreamsAvailable callback into connection options, remove count properties

* HTTP adjustments to StreamsAvailable callback in options

* Comments about multiple connections, including mention of being against RFC

* Miha's Feedback: H/3 authority evaluation, user callback try-catch

* Fix releasing stream

* Added logging for stream counts

* Fix stream counting in case of open stream failure

* Feedback

* Renamed API according to review feedback

* Renamed API according to review feedback

* More renames; Stored Action callback feedback

* Fixed borked platform guard

* Use the cached delegate 🤦

* Easy feedback

* Generated ref source

* SupportedOS ordering, because I cannot stand it different

* Idle connection back where it was

* Assert

* Fix handling negative stream capacity and 0 reporting STREAMS_AVAILABLE

* Handling cummulative increments, canceled streams and more tests.

* Removed logging

---------

Co-authored-by: Radek Zikmund <[email protected]>

src/libraries/System.Net.Http/ref/System.Net.Http.cs

@@ -424,6 +424,7 @@ public SocketsHttpHandler() { }
         public System.Net.ICredentials? Credentials { get { throw null; } set { } }
         public System.Net.ICredentials? DefaultProxyCredentials { get { throw null; } set { } }
         public bool EnableMultipleHttp2Connections { get { throw null; } set { } }
+        public bool EnableMultipleHttp3Connections { get { throw null; } set { } }
         public System.TimeSpan Expect100ContinueTimeout { get { throw null; } set { } }
         public int InitialHttp2StreamWindowSize { get { throw null; } set { } }
         [System.Runtime.Versioning.UnsupportedOSPlatformGuardAttribute("browser")]

src/libraries/System.Net.Http/src/Resources/Strings.resx

@@ -471,6 +471,9 @@
   <data name="net_http_invalid_header_name" xml:space="preserve">
     <value>Received an invalid header name: '{0}'.</value>
   </data>
+  <data name="net_http_http3_connection_not_established" xml:space="preserve">
+    <value>An HTTP/3 connection could not be established because the server did not complete the HTTP/3 handshake.</value>
+  </data>
   <data name="net_http_http3_connection_error" xml:space="preserve">
     <value>The HTTP/3 server sent invalid data on the connection. HTTP/3 error code '{0}' (0x{1}).</value>
   </data>

src/libraries/System.Net.Http/src/System/Net/Http/BrowserHttpHandler/SocketsHttpHandler.cs

@@ -196,6 +196,12 @@ public bool EnableMultipleHttp2Connections
             set => throw new PlatformNotSupportedException();
         }
 
+        public bool EnableMultipleHttp3Connections
+        {
+            get => throw new PlatformNotSupportedException();
+            set => throw new PlatformNotSupportedException();
+        }
+
         public Func<SocketsHttpConnectionContext, CancellationToken, ValueTask<Stream>>? ConnectCallback
         {
             get => throw new PlatformNotSupportedException();

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/ConnectHelper.cs

@@ -109,10 +109,10 @@ public static async ValueTask<SslStream> EstablishSslConnectionAsync(SslClientAu
             return sslStream;
         }
 
-        [SupportedOSPlatform("windows")]
         [SupportedOSPlatform("linux")]
         [SupportedOSPlatform("macos")]
-        public static async ValueTask<QuicConnection> ConnectQuicAsync(HttpRequestMessage request, DnsEndPoint endPoint, TimeSpan idleTimeout, SslClientAuthenticationOptions clientAuthenticationOptions, CancellationToken cancellationToken)
+        [SupportedOSPlatform("windows")]
+        public static async ValueTask<QuicConnection> ConnectQuicAsync(HttpRequestMessage request, DnsEndPoint endPoint, TimeSpan idleTimeout, SslClientAuthenticationOptions clientAuthenticationOptions, Action<QuicConnection, QuicStreamCapacityChangedArgs> streamCapacityCallback, CancellationToken cancellationToken)
         {
             clientAuthenticationOptions = SetUpRemoteCertificateValidationCallback(clientAuthenticationOptions, request);
 
@@ -126,7 +126,8 @@ public static async ValueTask<QuicConnection> ConnectQuicAsync(HttpRequestMessag
                     DefaultStreamErrorCode = (long)Http3ErrorCode.RequestCancelled,
                     DefaultCloseErrorCode = (long)Http3ErrorCode.NoError,
                     RemoteEndPoint = endPoint,
-                    ClientAuthenticationOptions = clientAuthenticationOptions
+                    ClientAuthenticationOptions = clientAuthenticationOptions,
+                    StreamCapacityCallback = streamCapacityCallback,
                 }, cancellationToken).ConfigureAwait(false);
             }
             catch (Exception ex) when (ex is not OperationCanceledException)

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/ConnectionPool/HttpConnectionPool.cs

@@ -245,6 +245,10 @@ public HttpConnectionPool(HttpConnectionPoolManager poolManager, HttpConnectionK
             {
                 _http2RequestQueue = new RequestQueue<Http2Connection?>();
             }
+            if (IsHttp3Supported() && _http3Enabled)
+            {
+                _http3RequestQueue = new RequestQueue<Http3Connection?>();
+            }
 
             if (_proxyUri != null && HttpUtilities.IsSupportedSecureScheme(_proxyUri.Scheme))
             {
@@ -881,11 +885,12 @@ public void Dispose()
                     _availableHttp2Connections.Clear();
                 }
 
-                if (_http3Connection is not null)
+                if (IsHttp3Supported() && _availableHttp3Connections is not null)
                 {
                     toDispose ??= new();
-                    toDispose.Add(_http3Connection);
-                    _http3Connection = null;
+                    toDispose.AddRange(_availableHttp3Connections);
+                    _associatedHttp3ConnectionCount -= _availableHttp3Connections.Count;
+                    _availableHttp3Connections.Clear();
                 }
 
                 if (_authorityExpireTimer != null)
@@ -956,6 +961,14 @@ public bool CleanCacheAndDisposeIfUnused()
                     // Note: Http11 connections will decrement the _associatedHttp11ConnectionCount when disposed.
                     // Http2 connections will not, hence the difference in handing _associatedHttp2ConnectionCount.
                 }
+                if (IsHttp3Supported() && _availableHttp3Connections is not null)
+                {
+                    int removed = ScavengeHttp3ConnectionList(_availableHttp3Connections, ref toDispose, nowTicks, pooledConnectionLifetime, pooledConnectionIdleTimeout);
+                    _associatedHttp3ConnectionCount -= removed;
+
+                    // Note: Http11 connections will decrement the _associatedHttp11ConnectionCount when disposed.
+                    // Http3 connections will not, hence the difference in handing _associatedHttp3ConnectionCount.
+                }
             }
 
             // Dispose the stale connections outside the pool lock, to avoid holding the lock too long.

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs

@@ -25,7 +25,6 @@ internal sealed partial class Http2Connection : HttpConnectionBase
 
         private TaskCompletionSourceWithCancellation<bool>? _initialSettingsReceived;
 
-        private readonly HttpConnectionPool _pool;
         private readonly Stream _stream;
 
         // NOTE: These are mutable structs; do not make these readonly.
@@ -132,7 +131,6 @@ internal enum KeepAliveState
         public Http2Connection(HttpConnectionPool pool, Stream stream, IPEndPoint? remoteEndPoint)
             : base(pool, remoteEndPoint)
         {
-            _pool = pool;
             _stream = stream;
 
             _incomingBuffer = new ArrayBuffer(initialSize: 0, usePool: true);
@@ -1794,18 +1792,6 @@ private bool ForceSendConnectionWindowUpdate()
             return true;
         }
 
-        public override long GetIdleTicks(long nowTicks)
-        {
-            // The pool is holding the lock as part of its scavenging logic.
-            // We must not lock on Http2Connection.SyncObj here as that could lead to lock ordering problems.
-            Debug.Assert(_pool.HasSyncObjLock);
-
-            // There is a race condition here where the connection pool may see this connection as idle right before
-            // we start processing a new request and start its disposal. This is okay as we will either
-            // return false from TryReserveStream, or process pending requests before tearing down the transport.
-            return _streamsInUse == 0 ? base.GetIdleTicks(nowTicks) : 0;
-        }
-
         /// <summary>Abort all streams and cause further processing to fail.</summary>
         /// <param name="abortException">Exception causing Abort to be called.</param>
         private void Abort(Exception abortException)

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http3Connection.cs

@@ -14,12 +14,11 @@
 
 namespace System.Net.Http
 {
-    [SupportedOSPlatform("windows")]
     [SupportedOSPlatform("linux")]
     [SupportedOSPlatform("macos")]
+    [SupportedOSPlatform("windows")]
     internal sealed class Http3Connection : HttpConnectionBase
     {
-        private readonly HttpConnectionPool _pool;
         private readonly HttpAuthority _authority;
         private readonly byte[]? _altUsedEncodedHeader;
         private QuicConnection? _connection;
@@ -33,7 +32,7 @@ internal sealed class Http3Connection : HttpConnectionBase
 
         // Our control stream.
         private QuicStream? _clientControl;
-        private Task _sendSettingsTask;
+        private Task? _sendSettingsTask;
 
         // Server-advertised SETTINGS_MAX_FIELD_SECTION_SIZE
         // https://www.rfc-editor.org/rfc/rfc9114.html#section-7.2.4.1-2.2.1
@@ -54,6 +53,9 @@ internal sealed class Http3Connection : HttpConnectionBase
         public Exception? AbortException => Volatile.Read(ref _abortException);
         private object SyncObj => _activeRequests;
 
+        private int _availableRequestStreamsCount;
+        private TaskCompletionSource<bool>? _availableStreamsWaiter;
+
         /// <summary>
         /// If true, we've received GOAWAY, are aborting due to a connection-level error, or are disposing due to pool limits.
         /// </summary>
@@ -66,12 +68,10 @@ private bool ShuttingDown
             }
         }
 
-        public Http3Connection(HttpConnectionPool pool, HttpAuthority authority, QuicConnection connection, bool includeAltUsedHeader)
-            : base(pool, connection.RemoteEndPoint)
+        public Http3Connection(HttpConnectionPool pool, HttpAuthority authority, bool includeAltUsedHeader)
+            : base(pool)
         {
-            _pool = pool;
             _authority = authority;
-            _connection = connection;
 
             if (includeAltUsedHeader)
             {
@@ -87,6 +87,13 @@ public Http3Connection(HttpConnectionPool pool, HttpAuthority authority, QuicCon
                 // Use this as an initial value before we receive the SETTINGS frame.
                 _maxHeaderListSize = maxHeaderListSize;
             }
+        }
+
+        public void InitQuicConnection(QuicConnection connection)
+        {
+            MarkConnectionAsEstablished(connection.RemoteEndPoint);
+
+            _connection = connection;
 
             // Errors are observed via Abort().
             _sendSettingsTask = SendSettingsAsync();
@@ -128,6 +135,9 @@ private void CheckForShutdown()
             {
                 // Close the QuicConnection in the background.
 
+                _availableStreamsWaiter?.SetResult(false);
+                _availableStreamsWaiter = null;
+
                 _connectionClosedTask ??= _connection.CloseAsync((long)Http3ErrorCode.NoError).AsTask();
 
                 QuicConnection connection = _connection;
@@ -151,7 +161,7 @@ private void CheckForShutdown()
 
                     if (_clientControl != null)
                     {
-                        await _sendSettingsTask.ConfigureAwait(false);
+                        await _sendSettingsTask!.ConfigureAwait(false);
                         await _clientControl.DisposeAsync().ConfigureAwait(false);
                         _clientControl = null;
                     }
@@ -162,6 +172,75 @@ private void CheckForShutdown()
             }
         }
 
+        public bool TryReserveStream()
+        {
+            lock (SyncObj)
+            {
+                Debug.Assert(_availableRequestStreamsCount >= 0);
+
+                if (NetEventSource.Log.IsEnabled()) Trace($"_availableRequestStreamsCount = {_availableRequestStreamsCount}");
+
+                if (_availableRequestStreamsCount == 0)
+                {
+                    return false;
+                }
+
+                --_availableRequestStreamsCount;
+                return true;
+            }
+        }
+
+        public void ReleaseStream()
+        {
+            lock (SyncObj)
+            {
+                Debug.Assert(_availableRequestStreamsCount >= 0);
+
+                if (NetEventSource.Log.IsEnabled()) Trace($"_availableRequestStreamsCount = {_availableRequestStreamsCount}");
+                ++_availableRequestStreamsCount;
+
+                _availableStreamsWaiter?.SetResult(!ShuttingDown);
+                _availableStreamsWaiter = null;
+            }
+        }
+
+        public void StreamCapacityCallback(QuicConnection connection, QuicStreamCapacityChangedArgs args)
+        {
+            Debug.Assert(_connection is null || connection == _connection);
+
+            lock (SyncObj)
+            {
+                Debug.Assert(_availableRequestStreamsCount >= 0);
+
+                if (NetEventSource.Log.IsEnabled()) Trace($"_availableRequestStreamsCount = {_availableRequestStreamsCount} + bidirectionalStreamsCountIncrement = {args.BidirectionalIncrement}");
+
+                _availableRequestStreamsCount += args.BidirectionalIncrement;
+                _availableStreamsWaiter?.SetResult(!ShuttingDown);
+                _availableStreamsWaiter = null;
+            }
+        }
+
+        public Task<bool> WaitForAvailableStreamsAsync()
+        {
+            lock (SyncObj)
+            {
+                Debug.Assert(_availableRequestStreamsCount >= 0);
+
+                if (ShuttingDown)
+                {
+                    return Task.FromResult(false);
+                }
+                if (_availableRequestStreamsCount > 0)
+                {
+                    return Task.FromResult(true);
+                }
+
+                Debug.Assert(_availableStreamsWaiter is null);
+                _availableStreamsWaiter = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
+                return _availableStreamsWaiter.Task;
+            }
+        }
+
         public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, long queueStartingTimestamp, CancellationToken cancellationToken)
         {
             // Allocate an active request
@@ -184,7 +263,6 @@ public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, lon
                             {
                                 MarkConnectionAsNotIdle();
                             }
-
                             _activeRequests.Add(quicStream, requestStream);
                         }
                     }
@@ -363,10 +441,8 @@ public void RemoveStream(QuicStream stream)
             }
         }
 
-        public override long GetIdleTicks(long nowTicks) => throw new NotImplementedException("We aren't scavenging HTTP3 connections yet");
-
         public override void Trace(string message, [CallerMemberName] string? memberName = null) =>
-            Trace(0, message, memberName);
+            Trace(0, _connection is not null ? $"{_connection} {message}" : message, memberName);
 
         internal void Trace(long streamId, string message, [CallerMemberName] string? memberName = null) =>
             NetEventSource.Log.HandlerMessage(

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http3RequestStream.cs

@@ -17,9 +17,9 @@
 
 namespace System.Net.Http
 {
-    [SupportedOSPlatform("windows")]
     [SupportedOSPlatform("linux")]
     [SupportedOSPlatform("macos")]
+    [SupportedOSPlatform("windows")]
     internal sealed class Http3RequestStream : IHttpStreamHeadersHandler, IAsyncDisposable, IDisposable
     {
         private readonly HttpRequestMessage _request;

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnection.cs

@@ -42,7 +42,6 @@ internal sealed partial class HttpConnection : HttpConnectionBase
         private static readonly ulong s_http10Bytes = BitConverter.ToUInt64("HTTP/1.0"u8);
         private static readonly ulong s_http11Bytes = BitConverter.ToUInt64("HTTP/1.1"u8);
 
-        private readonly HttpConnectionPool _pool;
         internal readonly Stream _stream;
         private readonly TransportContext? _transportContext;
 
@@ -78,10 +77,8 @@ public HttpConnection(
             IPEndPoint? remoteEndPoint)
             : base(pool, remoteEndPoint)
         {
-            Debug.Assert(pool != null);
             Debug.Assert(stream != null);
 
-            _pool = pool;
             _stream = stream;
 
             _transportContext = transportContext;