Support PAC scripts that return multiple proxies (dotnet/corefx#40082)

* Support PAC scripts that return multiple proxies.

Resolves dotnet/corefx#39370 

Commit migrated from dotnet/corefx@5e97c2b

src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinInetProxyHelper.cs

@@ -14,6 +14,8 @@ namespace System.Net.Http
     internal class WinInetProxyHelper
     {
         private const int RecentAutoDetectionInterval = 120_000; // 2 minutes in milliseconds.
+        private readonly string _autoConfigUrl, _proxy, _proxyBypass;
+        private readonly bool _autoDetect;
         private readonly bool _useProxy = false;
         private bool _autoDetectionFailed;
         private int _lastTimeAutoDetectionFailed; // Environment.TickCount units (milliseconds).
@@ -26,10 +28,10 @@ public WinInetProxyHelper()
             {
                 if (Interop.WinHttp.WinHttpGetIEProxyConfigForCurrentUser(out proxyConfig))
                 {
-                    AutoConfigUrl = Marshal.PtrToStringUni(proxyConfig.AutoConfigUrl);
-                    AutoDetect = proxyConfig.AutoDetect;
-                    Proxy = Marshal.PtrToStringUni(proxyConfig.Proxy);
-                    ProxyBypass = Marshal.PtrToStringUni(proxyConfig.ProxyBypass);
+                    _autoConfigUrl = Marshal.PtrToStringUni(proxyConfig.AutoConfigUrl);
+                    _autoDetect = proxyConfig.AutoDetect;
+                    _proxy = Marshal.PtrToStringUni(proxyConfig.Proxy);
+                    _proxyBypass = Marshal.PtrToStringUni(proxyConfig.ProxyBypass);
 
                     if (NetEventSource.IsEnabled)
                     {
@@ -57,19 +59,19 @@ public WinInetProxyHelper()
             }
         }
 
-        public string AutoConfigUrl { get; }
+        public string AutoConfigUrl => _autoConfigUrl;
 
-        public bool AutoDetect { get; }
+        public bool AutoDetect => _autoDetect;
 
         public bool AutoSettingsUsed => AutoDetect || !string.IsNullOrEmpty(AutoConfigUrl);
 
         public bool ManualSettingsUsed => !string.IsNullOrEmpty(Proxy);
 
         public bool ManualSettingsOnly => !AutoSettingsUsed && ManualSettingsUsed;
 
-        public string Proxy { get; }
+        public string Proxy => _proxy;
 
-        public string ProxyBypass { get; }
+        public string ProxyBypass => _proxyBypass;
 
         public bool RecentAutoDetectionFailure =>
             _autoDetectionFailed &&

src/libraries/System.Net.Http.WinHttpHandler/tests/UnitTests/System.Net.Http.WinHttpHandler.Unit.Tests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <ProjectGuid>{A2ECDEDB-12B7-402C-9230-152B7601179F}</ProjectGuid>
     <NoWarn>$(NoWarn);0436</NoWarn>
@@ -130,6 +130,15 @@
     <Compile Include="..\..\..\System.Net.Http\src\System\Net\Http\SocketsHttpHandler\HttpWindowsProxy.cs">
       <Link>ProductionCode\HttpWindowsProxy.cs</Link>
     </Compile>
+    <Compile Include="..\..\..\System.Net.Http\src\System\Net\Http\SocketsHttpHandler\FailedProxyCache.cs">
+      <Link>ProductionCode\FailedProxyCache.cs</Link>
+    </Compile>
+    <Compile Include="..\..\..\System.Net.Http\src\System\Net\Http\SocketsHttpHandler\IMultiWebProxy.cs">
+      <Link>ProductionCode\IMultiWebProxy.cs</Link>
+    </Compile>
+    <Compile Include="..\..\..\System.Net.Http\src\System\Net\Http\SocketsHttpHandler\MultiProxy.cs">
+      <Link>ProductionCode\MultiProxy.cs</Link>
+    </Compile>
     <Compile Include="APICallHistory.cs" />
     <Compile Include="ClientCertificateHelper.cs" />
     <Compile Include="ClientCertificateScenarioTest.cs" />

src/libraries/System.Net.Http/src/System.Net.Http.csproj

@@ -44,6 +44,7 @@
     <Compile Include="System\Net\Http\MultipartFormDataContent.cs" />
     <Compile Include="System\Net\Http\NetEventSource.Http.cs" />
     <Compile Include="System\Net\Http\ReadOnlyMemoryContent.cs" />
+    <Compile Include="System\Net\Http\RequestRetryType.cs" />
     <Compile Include="System\Net\Http\StreamContent.cs" />
     <Compile Include="System\Net\Http\StreamToStreamCopy.cs" />
     <Compile Include="System\Net\Http\StringContent.cs" />
@@ -164,6 +165,9 @@
     <Compile Include="System\Net\Http\SocketsHttpHandler\RawConnectionStream.cs" />
     <Compile Include="System\Net\Http\SocketsHttpHandler\RedirectHandler.cs" />
     <Compile Include="System\Net\Http\SocketsHttpHandler\TaskCompletionSourceWithCancellation.cs" />
+    <Compile Include="System\Net\Http\SocketsHttpHandler\FailedProxyCache.cs" />
+    <Compile Include="System\Net\Http\SocketsHttpHandler\IMultiWebProxy.cs" />
+    <Compile Include="System\Net\Http\SocketsHttpHandler\MultiProxy.cs" />
     <Compile Include="$(CommonPath)\CoreLib\System\Collections\Concurrent\ConcurrentQueueSegment.cs">
       <Link>Common\CoreLib\System\Collections\Concurrent\ConcurrentQueueSegment.cs</Link>
     </Compile>

src/libraries/System.Net.Http/src/System/Net/Http/HttpRequestException.cs

@@ -10,7 +10,7 @@ namespace System.Net.Http
     [SuppressMessage("Microsoft.Serialization", "CA2229")]
     public class HttpRequestException : Exception
     {
-        internal bool AllowRetry { get; }
+        internal RequestRetryType AllowRetry { get; } = RequestRetryType.NoRetry;
 
         public HttpRequestException()
             : this(null, null)
@@ -31,7 +31,7 @@ public HttpRequestException(string message, Exception inner)
 
         // This constructor is used internally to indicate that a request was not successfully sent due to an IOException,
         // and the exception occurred early enough so that the request may be retried on another connection.
-        internal HttpRequestException(string message, Exception inner, bool allowRetry)
+        internal HttpRequestException(string message, Exception inner, RequestRetryType allowRetry)
             : this(message, inner)
         {
             AllowRetry = allowRetry;

src/libraries/System.Net.Http/src/System/Net/Http/RequestRetryType.cs

@@ -0,0 +1,16 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace System.Net.Http
+{
+    /// <summary>
+    /// Used with <see cref="HttpRequestException"/> to indicate if a request is safe to retry.
+    /// </summary>
+    internal enum RequestRetryType
+    {
+        NoRetry,
+        RetryOnSameOrNextProxy,
+        RetryOnNextProxy
+    }
+}

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/ConnectHelper.cs

@@ -72,7 +72,7 @@ public static async ValueTask<Stream> ConnectAsync(string host, int port, Cancel
             {
                 throw CancellationHelper.ShouldWrapInOperationCanceledException(error, cancellationToken) ?
                     CancellationHelper.CreateOperationCanceledException(error, cancellationToken) :
-                    new HttpRequestException(error.Message, error);
+                    new HttpRequestException(error.Message, error, RequestRetryType.RetryOnNextProxy);
             }
             finally
             {

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/FailedProxyCache.cs

@@ -0,0 +1,146 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Threading;
+
+namespace System.Net.Http
+{
+    /// <summary>
+    /// Holds a cache of failing proxies and manages when they should be retried.
+    /// </summary>
+    internal sealed class FailedProxyCache
+    {
+        /// <summary>
+        /// When returned by <see cref="GetProxyRenewTicks"/>, indicates a proxy is immediately usable.
+        /// </summary>
+        public const long Immediate = 0;
+
+        // If a proxy fails, time out 30 minutes. WinHTTP and Firefox both use this.
+        private const int FailureTimeoutInMilliseconds = 1000 * 60 * 30;
+
+        // Scan through the failures and flush any that have expired every 5 minutes.
+        private const int FlushFailuresTimerInMilliseconds = 1000 * 60 * 5;
+
+        // _failedProxies will only be flushed (rare but somewhat expensive) if we have more than this number of proxies in our dictionary. See Cleanup() for details.
+        private const int LargeProxyConfigBoundary = 8;
+
+        // Value is the Environment.TickCount64 to remove the proxy from the failure list.
+        private readonly ConcurrentDictionary<Uri, long> _failedProxies = new ConcurrentDictionary<Uri, long>();
+
+        // When Environment.TickCount64 >= _nextFlushTicks, cause a flush.
+        private long _nextFlushTicks = Environment.TickCount64 + FlushFailuresTimerInMilliseconds;
+
+        // This lock can be folded into _nextFlushTicks for space optimization, but
+        // this class should only have a single instance so would rather have clarity.
+        private SpinLock _flushLock = new SpinLock();
+
+        /// <summary>
+        /// Checks when a proxy will become usable.
+        /// </summary>
+        /// <param name="uri">The <see cref="Uri"/> of the proxy to check.</param>
+        /// <returns>If the proxy can be used, <see cref="Immediate"/>. Otherwise, the next <see cref="Environment.TickCount64"/> that it should be used.</returns>
+        public long GetProxyRenewTicks(Uri uri)
+        {
+            Cleanup();
+
+            // If not failed, ready immediately.
+            if (!_failedProxies.TryGetValue(uri, out long renewTicks))
+            {
+                return Immediate;
+            }
+
+            // If we haven't reached out renew time, the proxy can't be used.
+            if (Environment.TickCount64 < renewTicks)
+            {
+                return renewTicks;
+            }
+
+            // Renew time reached, we can remove the proxy from the cache.
+            if (TryRenewProxy(uri, renewTicks))
+            {
+                return Immediate;
+            }
+
+            // Another thread updated the cache before we could remove it.
+            // We can't know if this is a removal or an update, so check again.
+            return _failedProxies.TryGetValue(uri, out renewTicks) ? renewTicks : Immediate;
+        }
+
+        /// <summary>
+        /// Sets a proxy as failed, to avoid trying it again for some time.
+        /// </summary>
+        /// <param name="uri">The URI of the proxy.</param>
+        public void SetProxyFailed(Uri uri)
+        {
+            _failedProxies[uri] = Environment.TickCount64 + FailureTimeoutInMilliseconds;
+            Cleanup();
+        }
+
+        /// <summary>
+        /// Renews a proxy prior to its period expiring. Used when all proxies are failed to renew the proxy closest to being renewed.
+        /// </summary>
+        /// <param name="uri">The <paramref name="uri"/> of the proxy to renew.</param>
+        /// <param name="renewTicks">The current renewal time for the proxy. If the value has changed from this, the proxy will not be renewed.</param>
+        public bool TryRenewProxy(Uri uri, long renewTicks)
+        {
+            var collection = (ICollection<KeyValuePair<Uri, long>>)_failedProxies;
+            return collection.Remove(new KeyValuePair<Uri, long>(uri, renewTicks));
+        }
+
+        /// <summary>
+        /// Cleans up any old proxies that should no longer be marked as failing.
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private void Cleanup()
+        {
+            if (_failedProxies.Count > LargeProxyConfigBoundary && Environment.TickCount64 >= Interlocked.Read(ref _nextFlushTicks))
+            {
+                CleanupHelper();
+            }
+        }
+
+        /// <summary>
+        /// Cleans up any old proxies that should no longer be marked as failing.
+        /// </summary>
+        /// <remarks>
+        /// I expect this to never be called by <see cref="Cleanup"/> in a production system. It is only needed in the case
+        /// that a system has a very large number of proxies that the PAC script cycles through. It is moderately expensive,
+        /// so it's only run periodically and is disabled until we exceed <see cref="LargeProxyConfigBoundary"/> failed proxies.
+        /// </remarks>
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private void CleanupHelper()
+        {
+            bool lockTaken = false;
+            try
+            {
+                _flushLock.TryEnter(ref lockTaken);
+                if (!lockTaken)
+                {
+                    return;
+                }
+
+                long curTicks = Environment.TickCount64;
+
+                foreach (KeyValuePair<Uri, long> kvp in _failedProxies)
+                {
+                    if (curTicks >= kvp.Value)
+                    {
+                        ((ICollection<KeyValuePair<Uri, long>>)_failedProxies).Remove(kvp);
+                    }
+                }
+            }
+            finally
+            {
+                if (lockTaken)
+                {
+                    Interlocked.Exchange(ref _nextFlushTicks, Environment.TickCount64 + FlushFailuresTimerInMilliseconds);
+                    _flushLock.Exit(false);
+                }
+            }
+        }
+    }
+}

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs

@@ -1112,7 +1112,7 @@ private HttpRequestException GetShutdownException()
                 innerException = new ObjectDisposedException(nameof(Http2Connection));
             }
 
-            return new HttpRequestException(SR.net_http_client_execution_error, innerException, allowRetry: true);
+            return new HttpRequestException(SR.net_http_client_execution_error, innerException, allowRetry: RequestRetryType.RetryOnSameOrNextProxy);
         }
 
         private async ValueTask<Http2Stream> SendHeadersAsync(HttpRequestMessage request, CancellationToken cancellationToken, bool mustFlush)

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Stream.cs

@@ -758,7 +758,7 @@ private void CheckResponseBodyState()
                 {
                     if (_canRetry)
                     {
-                        throw new HttpRequestException(SR.net_http_request_aborted, _resetException, allowRetry: true);
+                        throw new HttpRequestException(SR.net_http_request_aborted, _resetException, allowRetry: RequestRetryType.RetryOnSameOrNextProxy);
                     }
 
                     throw new IOException(SR.net_http_request_aborted, _resetException);

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnection.cs

@@ -720,7 +720,7 @@ public async Task<HttpResponseMessage> SendAsyncCore(HttpRequestMessage request,
                 {
                     // For consistency with other handlers we wrap the exception in an HttpRequestException.
                     // If the request is retryable, indicate that on the exception.
-                    throw new HttpRequestException(SR.net_http_client_execution_error, ioe, _canRetry);
+                    throw new HttpRequestException(SR.net_http_client_execution_error, ioe, _canRetry ? RequestRetryType.RetryOnSameOrNextProxy : RequestRetryType.NoRetry);
                 }
                 else
                 {

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionBase.cs

@@ -53,7 +53,7 @@ internal static HttpRequestException CreateRetryException()
             // processed the request in any way, and thus the request can be retried.
             // This will be caught in HttpConnectionPool.SendWithRetryAsync and the retry logic will kick in.
             // The user should never see this exception.
-            throw new HttpRequestException(null, null, allowRetry: true);
+            throw new HttpRequestException(null, null, allowRetry: RequestRetryType.RetryOnSameOrNextProxy);
         }
 
         internal static bool IsDigit(byte c) => (uint)(c - '0') <= '9' - '0';

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs

@@ -540,7 +540,7 @@ public async Task<HttpResponseMessage> SendWithRetryAsync(HttpRequestMessage req
                         return await connection.SendAsync(request, cancellationToken).ConfigureAwait(false);
                     }
                 }
-                catch (HttpRequestException e) when (!isNewConnection && e.AllowRetry)
+                catch (HttpRequestException e) when (!isNewConnection && e.AllowRetry == RequestRetryType.RetryOnSameOrNextProxy)
                 {
                     if (NetEventSource.IsEnabled)
                     {

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPoolManager.cs

@@ -5,6 +5,7 @@
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Runtime.ExceptionServices;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -281,14 +282,26 @@ public Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, bool doRe
             {
                 if (!_proxy.IsBypassed(request.RequestUri))
                 {
-                    proxyUri = _proxy.GetProxy(request.RequestUri);
+                    if (_proxy is IMultiWebProxy multiWebProxy)
+                    {
+                        MultiProxy multiProxy = multiWebProxy.GetMultiProxy(request.RequestUri);
+
+                        if (multiProxy.ReadNext(out proxyUri, out bool isFinalProxy) && !isFinalProxy)
+                        {
+                            return SendAsyncMultiProxy(request, doRequestAuth, multiProxy, proxyUri, cancellationToken);
+                        }
+                    }
+                    else
+                    {
+                        proxyUri = _proxy.GetProxy(request.RequestUri);
+                    }
                 }
             }
             catch (Exception ex)
             {
                 // Eat any exception from the IWebProxy and just treat it as no proxy.
                 // This matches the behavior of other handlers.
-                if (NetEventSource.IsEnabled) NetEventSource.Error(this, $"Exception from IWebProxy.GetProxy({request.RequestUri}): {ex}");
+                if (NetEventSource.IsEnabled) NetEventSource.Error(this, $"Exception from {_proxy.GetType().Name}.GetProxy({request.RequestUri}): {ex}");
             }
 
             if (proxyUri != null && proxyUri.Scheme != UriScheme.Http)
@@ -299,6 +312,32 @@ public Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, bool doRe
             return SendAsyncCore(request, proxyUri, doRequestAuth, isProxyConnect: false, cancellationToken);
         }
 
+        /// <summary>
+        /// Iterates a request over a set of proxies until one works, or all proxies have failed.
+        /// </summary>
+        /// <param name="multiProxy">The set of proxies to use.</param>
+        /// <param name="firstProxy">The first proxy try.</param>
+        private async Task<HttpResponseMessage> SendAsyncMultiProxy(HttpRequestMessage request, bool doRequestAuth, MultiProxy multiProxy, Uri firstProxy, CancellationToken cancellationToken)
+        {
+            HttpRequestException rethrowException = null;
+
+            do
+            {
+                try
+                {
+                    return await SendAsyncCore(request, firstProxy, doRequestAuth, isProxyConnect: false, cancellationToken).ConfigureAwait(false);
+                }
+                catch (HttpRequestException ex) when (ex.AllowRetry != RequestRetryType.NoRetry)
+                {
+                    rethrowException = ex;
+                }
+            }
+            while (multiProxy.ReadNext(out firstProxy, out _));
+
+            ExceptionDispatchInfo.Throw(rethrowException);
+            return null; // should never be reached: VS doesn't realize Throw() never returns.
+        }
+
         /// <summary>Disposes of the pools, disposing of each individual pool.</summary>
         public void Dispose()
         {