Fix for ApiDataProvider (QuantConnect#6461)

- Avoid race condition while downloading data.
- Reused http client for downloads

Api/Api.cs

@@ -38,6 +38,7 @@ namespace QuantConnect.Api
     /// </summary>
     public class Api : IApi, IDownloadProvider
     {
+        private readonly HttpClient _client = new HttpClient();
         private string _dataFolder;
 
         /// <summary>
@@ -877,9 +878,7 @@ public bool DownloadData(string filePath, string organizationId)
             {
                 // Download the file
                 var uri = new Uri(dataLink.Url);
-
-                using var client = new HttpClient();
-                using var dataStream = client.GetStreamAsync(uri);
+                using var dataStream = _client.GetStreamAsync(uri);
 
                 using var fileStream = new FileStream(filePath, FileMode.Create);
                 dataStream.Result.CopyTo(fileStream);
@@ -997,7 +996,7 @@ public virtual string Download(string address, IEnumerable<KeyValuePair<string,
         /// <filterpriority>2</filterpriority>
         public virtual void Dispose()
         {
-            // NOP
+            _client.DisposeSafely();
         }
 
         /// <summary>

Engine/DataFeeds/BaseDownloaderDataProvider.cs

@@ -51,17 +51,16 @@ protected Stream DownloadOnce(string key, Action<string> download)
                         _currentDownloads.TryRemove(key, out _);
                         return base.Fetch(key);
                     }
-
-                    // this is rare
-                    _currentDownloads.TryGetValue(key, out var existingKey);
-                    lock (existingKey ?? new object())
-                    {
-                        return base.Fetch(key);
-                    }
                 }
             }
 
-            return base.Fetch(key);
+            // even though we should not download in this path, we need to wait for any download in progress to be finished
+            // in this case it would be present in the '_currentDownloads' collection with it's lock taken
+            _currentDownloads.TryGetValue(key, out var existingKey);
+            lock (existingKey ?? new object())
+            {
+                return base.Fetch(key);
+            }
         }
 
         /// <summary>