DownloaderDataProvider improvements (QuantConnect#5730)

- Ignore margin files requests. Only provided by ApiDataProvider - Reduce noise by LeanData.TryParse() call on failure
Coulbe · Jul 2, 2021 · e5a5010 · e5a5010
1 parent c8d1308
commit e5a5010
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 14 deletions.
diff --git a/Common/Util/LeanData.cs b/Common/Util/LeanData.cs
@@ -992,22 +992,34 @@ public static bool TryParseSecurityType(string fileName, out SecurityType securi
         public static bool TryParsePath(string filePath, out Symbol symbol, out DateTime date,
             out Resolution resolution, out TickType tickType, out Type dataType)
         {
-            if (!TryParsePath(filePath, out symbol, out date, out resolution))
+            symbol = default;
+            tickType = default;
+            dataType = default;
+            date = default;
+            resolution = default;
+
+            try
             {
-                tickType = TickType.Trade;
-                dataType = null;
-                return false;
-            }
+                if (!TryParsePath(filePath, out symbol, out date, out resolution))
+                {
+                    return false;
+                }
 
-            tickType = GetCommonTickType(symbol.SecurityType);
-            var fileName = Path.GetFileNameWithoutExtension(filePath);
-            if (fileName.Contains("_"))
+                tickType = GetCommonTickType(symbol.SecurityType);
+                var fileName = Path.GetFileNameWithoutExtension(filePath);
+                if (fileName.Contains("_"))
+                {
+                    tickType = (TickType)Enum.Parse(typeof(TickType), fileName.Split('_')[1], true);
+                }
+
+                dataType = GetDataType(resolution, tickType);
+                return true;
+            }
+            catch (Exception ex)
             {
-                tickType = (TickType)Enum.Parse(typeof(TickType), fileName.Split('_')[1], true);
+                Log.Debug($"LeanData.TryParsePath(): Error encountered while parsing the path {filePath}. Error: {ex.GetBaseException()}");
             }
-
-            dataType = GetDataType(resolution, tickType);
-            return true;
+            return false;
         }
 
         /// <summary>
@@ -1083,7 +1095,7 @@ public static bool TryParsePath(string fileName, out Symbol symbol, out DateTime
             }
             catch (Exception ex)
             {
-                Log.Error($"LeanData.TryParsePath(): Error encountered while parsing the path {fileName}. Error: {ex.GetBaseException()}");
+                Log.Debug($"LeanData.TryParsePath(): Error encountered while parsing the path {fileName}. Error: {ex.GetBaseException()}");
                 return false;
             }
 

diff --git a/Engine/DataFeeds/DownloaderDataProvider.cs b/Engine/DataFeeds/DownloaderDataProvider.cs
@@ -116,7 +116,8 @@ protected override bool NeedToDownload(string filePath)
             if (filePath == null
                 || filePath.Contains("fine", StringComparison.InvariantCultureIgnoreCase) && filePath.Contains("fundamental", StringComparison.InvariantCultureIgnoreCase)
                 || filePath.Contains("map_files", StringComparison.InvariantCultureIgnoreCase)
-                || filePath.Contains("factor_files", StringComparison.InvariantCultureIgnoreCase))
+                || filePath.Contains("factor_files", StringComparison.InvariantCultureIgnoreCase)
+                || filePath.Contains("margins", StringComparison.InvariantCultureIgnoreCase) && filePath.Contains("future", StringComparison.InvariantCultureIgnoreCase))
             {
                 return false;
             }