Feature: find different Ticker data download parameters (QuantConnect…

…#7845)

* refactor: map files in lean/data folder

* feat: Indexers in MapFile

* feat: extension for mapFile and downloader param

* refactor: getAllTickers from MapFiles

* feat: addition out variable in TryParsePath
fix: tests with new variable

* refactor: DownloadDataProvider with getting all ticker with different DateTime Ranges
feat: test

* revert: "refactor: map files in lean/data folder"

This reverts commit 41fa26e.

* revert: "feat: addition out variable in TryParsePath"
refactor: some actual changing

This reverts commit 9de482b.

* revert: "feat: Indexers in MapFile"
fix: actual changes

This reverts commit 393181c.

* remove: high performance Any validation
feat: validation on ticker start with specific sign
fix: option test

* feat: validation on mapping
refactor: when resolution less hour
test: new test cases + validation of SecurityType

* refactor: add additional day when return date range from MapFile
fix: FirstTicker instead of Permtick when generate new Equity
feat: validation of request Date in DownloaderDataProvider
refactor: by style proj DownloaderDataProvider
test:fix: starting Date in TestCases

* remove: duplicate code

* fix: check of income data.Time with requested Start/End-Date

* test:fix: download test cuz we can not get/write future data

* remove:  extra Symbol Create mthd

* test:feat: validate of TryParse returns correct symbol

* remove: excess if condition

* test:remove: extra chaning
connected # 3a62adf

* refactor: split big method to small ones (add readability)

* fix: typo in xml description

* refactor: get the correct symbol value

* refactor: use local time instead of Utc in download Param extension
refactor: use struct instead of class

* test:fix: convert time to Utc from local time

Common/Data/Auxiliary/MappingExtensions.cs

@@ -134,5 +134,68 @@ public static IEnumerable<TickerDateRange> RetrieveSymbolHistoricalDefinitionsIn
                 }
             }
         }
+
+        /// <summary>
+        /// Retrieves all Symbol from map files based on specific Symbol.
+        /// </summary>
+        /// <param name="mapFileProvider">The provider for map files containing ticker data.</param>
+        /// <param name="symbol">The symbol to get <see cref="MapFileResolver"/> and generate new Symbol.</param>
+        /// <returns>An enumerable collection of <see cref="SymbolDateRange"/></returns>
+        /// <exception cref="ArgumentException">Throw if <paramref name="mapFileProvider"/> is null.</exception>
+        public static IEnumerable<SymbolDateRange> RetrieveAllMappedSymbolInDateRange(this IMapFileProvider mapFileProvider, Symbol symbol)
+        {
+            if (mapFileProvider == null || symbol == null)
+            {
+                throw new ArgumentException($"The map file provider and symbol cannot be null. {(mapFileProvider == null ? nameof(mapFileProvider) : nameof(symbol))}");
+            }
+
+            var mapFileResolver = mapFileProvider.Get(AuxiliaryDataKey.Create(symbol));
+
+            var tickerUpperCase = symbol.HasUnderlying ? symbol.Underlying.Value.ToUpperInvariant() : symbol.Value.ToUpperInvariant();
+
+            var isOptionSymbol = symbol.SecurityType == SecurityType.Option;
+            foreach (var mapFile in mapFileResolver)
+            {
+                // Check if 'mapFile' contains the desired ticker symbol.
+                if (!mapFile.Any(mapFileRow => mapFileRow.MappedSymbol == tickerUpperCase))
+                {
+                    continue;
+                }
+
+                foreach (var tickerDateRange in mapFile.GetTickerDateRanges(tickerUpperCase))
+                {
+                    var sid = SecurityIdentifier.GenerateEquity(mapFile.FirstDate, mapFile.FirstTicker, symbol?.ID.Market);
+
+                    var newSymbol = new Symbol(sid, tickerUpperCase);
+
+                    if (isOptionSymbol)
+                    {
+                        newSymbol = Symbol.CreateCanonicalOption(newSymbol);
+                    }
+
+                    yield return new(newSymbol, tickerDateRange.StartDate, tickerDateRange.EndDate);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Retrieves the date ranges associated with a specific ticker symbol from the provided map file.
+        /// </summary>
+        /// <param name="mapFile">The map file containing the data ranges for various ticker.</param>
+        /// <param name="ticker">The ticker for which to retrieve the date ranges.</param>
+        /// <returns>An enumerable collection of tuples representing the start and end dates for each date range associated with the specified ticker symbol.</returns>
+        private static IEnumerable<(DateTime StartDate, DateTime EndDate)> GetTickerDateRanges(this MapFile mapFile, string ticker)
+        {
+            var previousRowDate = mapFile.FirstOrDefault().Date;
+            foreach (var currentRow in mapFile.Skip(1))
+            {
+                if (ticker == currentRow.MappedSymbol)
+                {
+                    yield return (previousRowDate, currentRow.Date.AddDays(1));
+                }
+                // MapFile maintains the latest date associated with each ticker name, except first Row
+                previousRowDate = currentRow.Date.AddDays(1);
+            }
+        }
     }
 }

Common/Data/Auxiliary/SymbolDateRange.cs

@@ -0,0 +1,55 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+
+namespace QuantConnect.Data.Auxiliary
+{
+    /// <summary>
+    /// Represents security identifier within a date range.
+    /// </summary>
+#pragma warning disable CA1815 // Override equals and operator equals on value types
+    public readonly struct SymbolDateRange
+    {
+        /// <summary>
+        /// Represents a unique security identifier.
+        /// </summary>
+        public Symbol Symbol { get; }
+
+        /// <summary>
+        /// Ticker Start Date Time in Local
+        /// </summary>
+        public DateTime StartDateTimeLocal { get; }
+
+        /// <summary>
+        /// Ticker End Date Time in Local
+        /// </summary>
+        public DateTime EndDateTimeLocal { get; }
+
+        /// <summary>
+        /// Create the instance of <see cref="SymbolDateRange"/> struct.
+        /// </summary>
+        /// <param name="symbol">The unique security identifier</param>
+        /// <param name="startDateTimeLocal">Start Date Time Local</param>
+        /// <param name="endDateTimeLocal">End Date Time Local</param>
+        public SymbolDateRange(Symbol symbol, DateTime startDateTimeLocal, DateTime endDateTimeLocal)
+        {
+            Symbol = symbol;
+            StartDateTimeLocal = startDateTimeLocal;
+            EndDateTimeLocal = endDateTimeLocal;
+        }
+    }
+#pragma warning restore CA1815
+}

Common/Data/Auxiliary/TickerDateRange.cs

@@ -30,12 +30,12 @@ public readonly struct TickerDateRange
         public string Ticker { get; }
 
         /// <summary>
-        /// Ticker Start Date Time in UTC
+        /// Ticker Start Date Time in Local
         /// </summary>
         public DateTime StartDateTimeLocal { get; }
 
         /// <summary>
-        /// Ticker End Date Time in UTC
+        /// Ticker End Date Time in Local
         /// </summary>
         public DateTime EndDateTimeLocal { get; }
 

Common/Data/DownloaderExtensions.cs

@@ -0,0 +1,81 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+using NodaTime;
+
+namespace QuantConnect.Data
+{
+    /// <summary>
+    /// Contains extension methods for the Downloader functionality.
+    /// </summary>
+    public static class DownloaderExtensions
+    {
+        /// <summary>
+        /// Get <see cref="DataDownloaderGetParameters"/> for all mapped <seealso cref="Symbol"/> with appropriate ticker name in specific date time range.
+        /// </summary>
+        /// <param name="dataDownloaderParameter">Generated class in "Lean.Engine.DataFeeds.DownloaderDataProvider"</param>
+        /// <param name="mapFileProvider">Provides instances of <see cref="MapFileResolver"/> at run time</param>
+        /// <param name="exchangeTimeZone">Provides the time zone this exchange</param>
+        /// <returns>
+        /// Return DataDownloaderGetParameters with different 
+        /// <see cref="DataDownloaderGetParameters.StartUtc"/> - <seealso cref="DataDownloaderGetParameters.EndUtc"/> range
+        /// and <seealso cref="Symbol"/>
+        /// </returns>
+        /// <exception cref="ArgumentNullException">Thrown when <paramref name="dataDownloaderParameter"/> is null.</exception>
+        public static IEnumerable<DataDownloaderGetParameters> GetDataDownloaderParameterForAllMappedSymbols(
+            this DataDownloaderGetParameters dataDownloaderParameter,
+            IMapFileProvider mapFileProvider,
+            DateTimeZone exchangeTimeZone)
+        {
+            if (dataDownloaderParameter == null)
+            {
+                throw new ArgumentNullException(nameof(dataDownloaderParameter));
+            }
+
+            if (dataDownloaderParameter.Symbol.SecurityType != SecurityType.Future
+                && dataDownloaderParameter.Symbol.RequiresMapping()
+                && dataDownloaderParameter.Resolution >= Resolution.Hour)
+            {
+                var yieldMappedSymbol = default(bool);
+                foreach (var symbolDateRange in mapFileProvider.RetrieveAllMappedSymbolInDateRange(dataDownloaderParameter.Symbol))
+                {
+                    var endDateTimeUtc = symbolDateRange.EndDateTimeLocal.ConvertToUtc(exchangeTimeZone);
+
+                    // The first start date returns from mapFile like IPO (DateTime) and can not be greater then request StartTime
+                    // The Downloader doesn't know start DateTime exactly, it always download all data
+                    var startDateTime = symbolDateRange.StartDateTimeLocal.ConvertToUtc(exchangeTimeZone);
+                    var endDateTime = endDateTimeUtc > dataDownloaderParameter.EndUtc ? dataDownloaderParameter.EndUtc : endDateTimeUtc;
+
+                    yield return new DataDownloaderGetParameters(
+                        symbolDateRange.Symbol, dataDownloaderParameter.Resolution, startDateTime, endDateTime, dataDownloaderParameter.TickType);
+                    yieldMappedSymbol = true;
+                }
+
+                if (!yieldMappedSymbol)
+                {
+                    yield return dataDownloaderParameter;
+                }
+            }
+            else
+            {
+                yield return dataDownloaderParameter;
+            }
+        }
+    }
+}

Engine/DataFeeds/DownloaderDataProvider.cs

@@ -15,14 +15,17 @@
 */
 
 using System;
+using NodaTime;
 using System.IO;
 using System.Linq;
 using QuantConnect.Util;
 using QuantConnect.Data;
 using QuantConnect.Logging;
 using QuantConnect.Securities;
 using QuantConnect.Interfaces;
+using System.Collections.Generic;
 using QuantConnect.Configuration;
+using QuantConnect.Data.Auxiliary;
 using System.Collections.Concurrent;
 
 namespace QuantConnect.Lean.Engine.DataFeeds
@@ -42,6 +45,7 @@ public class DownloaderDataProvider : BaseDownloaderDataProvider
         private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
         private readonly IDataDownloader _dataDownloader;
         private readonly IDataCacheProvider _dataCacheProvider = new DiskDataCacheProvider(DiskSynchronizer);
+        private readonly IMapFileProvider _mapFileProvider = Composer.Instance.GetPart<IMapFileProvider>();
 
         /// <summary>
         /// Creates a new instance
@@ -118,7 +122,7 @@ public override Stream Fetch(string key)
                         startTimeUtc = date.ConvertToUtc(dataTimeZone);
                         // let's get the whole day
                         endTimeUtc = date.AddDays(1).ConvertToUtc(dataTimeZone);
-                        if(endTimeUtc > endTimeUtcLimit)
+                        if (endTimeUtc > endTimeUtcLimit)
                         {
                             // we are at the limit, avoid getting partial data
                             return;
@@ -163,36 +167,16 @@ public override Stream Fetch(string key)
                         LeanDataWriter writer = null;
                         var getParams = new DataDownloaderGetParameters(symbol, resolution, startTimeUtc, endTimeUtc, tickType);
 
-                        var downloadData = _dataDownloader.Get(getParams);
+                        var downloaderDataParameters = getParams.GetDataDownloaderParameterForAllMappedSymbols(_mapFileProvider, exchangeTimeZone);
 
-                        if (downloadData == null)
-                        {
-                            // doesn't support this download request, that's okay
-                            return;
-                        }
-
-                        var data = downloadData
-                            .Where(baseData =>
-                            {
-                                if(symbol.SecurityType == SecurityType.Base || baseData.GetType() == dataType)
-                                {
-                                    // we need to store the data in data time zone
-                                    baseData.Time = baseData.Time.ConvertTo(exchangeTimeZone, dataTimeZone);
-                                    baseData.EndTime = baseData.EndTime.ConvertTo(exchangeTimeZone, dataTimeZone);
-                                    return true;
-                                }
-                                return false;
-                            })
-                            // for canonical symbols, downloader will return data for all of the chain
-                            .GroupBy(baseData => baseData.Symbol);
+                        var downloadedData = GetDownloadedData(downloaderDataParameters, symbol, exchangeTimeZone, dataTimeZone, dataType);
 
-                        foreach (var dataPerSymbol in data)
+                        foreach (var dataPerSymbol in downloadedData)
                         {
                             if (writer == null)
                             {
                                 writer = new LeanDataWriter(resolution, symbol, Globals.DataFolder, tickType, mapSymbol: true, dataCacheProvider: _dataCacheProvider);
                             }
-
                             // Save the data
                             writer.Write(dataPerSymbol);
                         }
@@ -205,12 +189,74 @@ public override Stream Fetch(string key)
             });
         }
 
+        /// <summary>
+        /// Retrieves downloaded data grouped by symbol based on <see cref="IDownloadProvider"/>.
+        /// </summary>
+        /// <param name="downloaderDataParameters">Parameters specifying the data to be retrieved.</param>
+        /// <param name="symbol">Represents a unique security identifier, generate by ticker name.</param>
+        /// <param name="exchangeTimeZone">The time zone of the exchange where the symbol is traded.</param>
+        /// <param name="dataTimeZone">The time zone in which the data is represented.</param>
+        /// <param name="dataType">The type of data to be retrieved. (e.g. <see cref="Data.Market.TradeBar"/>)</param>
+        /// <returns>An IEnumerable containing groups of data grouped by symbol. Each group contains data related to a specific symbol.</returns>
+        /// <exception cref="ArgumentException"> Thrown when the downloaderDataParameters collection is null or empty.</exception>
+        public IEnumerable<IGrouping<Symbol, BaseData>> GetDownloadedData(
+            IEnumerable<DataDownloaderGetParameters> downloaderDataParameters,
+            Symbol symbol,
+            DateTimeZone exchangeTimeZone,
+            DateTimeZone dataTimeZone,
+            Type dataType)
+        {
+            if (downloaderDataParameters.IsNullOrEmpty())
+            {
+                throw new ArgumentException($"{nameof(DownloaderDataProvider)}.{nameof(GetDownloadedData)}: DataDownloaderGetParameters are empty or equal to null.");
+            }
+
+            foreach (var downloaderDataParameter in downloaderDataParameters)
+            {
+                var downloadedData = _dataDownloader.Get(downloaderDataParameter);
+
+                if (downloadedData == null)
+                {
+                    // doesn't support this download request, that's okay
+                    continue;
+                }
+                var startDateTimeInExchangeTimeZone = downloaderDataParameter.StartUtc.ConvertFromUtc(exchangeTimeZone);
+                var endDateTimeInExchangeTimeZone = downloaderDataParameter.EndUtc.ConvertFromUtc(exchangeTimeZone);
+
+                var groupedData = downloadedData
+                    .Where(baseData =>
+                    {
+                        // Sometimes, external Downloader provider returns excess data
+                        if (baseData.Time < startDateTimeInExchangeTimeZone || baseData.Time > endDateTimeInExchangeTimeZone)
+                        {
+                            return false;
+                        }
+
+                        if (symbol.SecurityType == SecurityType.Base || baseData.GetType() == dataType)
+                        {
+                            // we need to store the data in data time zone
+                            baseData.Time = baseData.Time.ConvertTo(exchangeTimeZone, dataTimeZone);
+                            baseData.EndTime = baseData.EndTime.ConvertTo(exchangeTimeZone, dataTimeZone);
+                            return true;
+                        }
+                        return false;
+                    })
+                    // for canonical symbols, downloader will return data for all of the chain
+                    .GroupBy(baseData => baseData.Symbol);
+
+                foreach (var data in groupedData)
+                {
+                    yield return data;
+                }
+            }
+        }
+
         /// <summary>
         /// Get's the stream for a given file path
         /// </summary>
         protected override Stream GetStream(string key)
         {
-            if(LeanData.TryParsePath(key, out var symbol, out var date, out var resolution) && resolution > Resolution.Minute && symbol.RequiresMapping())
+            if (LeanData.TryParsePath(key, out var symbol, out var date, out var resolution) && resolution > Resolution.Minute && symbol.RequiresMapping())
             {
                 // because the file could be updated even after it's created because of symbol mapping we can't stream from disk
                 return DiskSynchronizer.Execute(key, () =>