Add BaseData.AdjustResolution

- Adding `BaseData.AdjustResolution()` that should return a valid resolution for the given data and security type. This allows us to set a limitation which is useful to avoid invalid data requests or unnecessary fill forward situations. The user will be notified through a console message. - Adding unit and regression test - Updating example algorithms custom data resolution - Some performance improvements. Wont change console color if `SelectedOptimization` is defined
Coulbe · Nov 4, 2019 · 1d43dcd · 1d43dcd
1 parent 7e33be1
commit 1d43dcd
Show file tree

Hide file tree

Showing 40 changed files with 477 additions and 60 deletions.
diff --git a/Algorithm.CSharp/AdjustResolutionRegressionAlgorithm.cs b/Algorithm.CSharp/AdjustResolutionRegressionAlgorithm.cs
@@ -0,0 +1,118 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Data.Custom;
+using QuantConnect.Data.Custom.CBOE;
+using QuantConnect.Data.Custom.Fred;
+using QuantConnect.Data.Custom.SEC;
+using QuantConnect.Data.Custom.Tiingo;
+using QuantConnect.Data.Custom.USEnergy;
+using QuantConnect.Data.Custom.USTreasury;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// This regression algorithm tests the performance related GH issue 3772, AssertResolution
+    /// </summary>
+    public class AdjustResolutionRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        public override void Initialize()
+        {
+            SetStartDate(2013, 10, 11);
+            SetEndDate(2013, 10, 12);
+            var spy = AddEquity("SPY").Symbol;
+
+            var types = new[]
+            {
+                typeof(SECReport8K),
+                typeof(SECReport10K),
+                typeof(SECReport10Q),
+                typeof(USTreasuryYieldCurveRate),
+                typeof(USEnergy),
+                typeof(CBOE),
+                typeof(TiingoPrice),
+                typeof(Fred)
+            };
+
+            foreach (var type in types)
+            {
+                var custom = AddData(type, spy, Resolution.Hour, null);
+                if (SubscriptionManager.SubscriptionDataConfigService
+                    .GetSubscriptionDataConfigs(custom.Symbol)
+                    .Any(config => config.Resolution != Resolution.Daily))
+                {
+                    throw new Exception("Was expecting resolution to be adjusted to Daily");
+                }
+            }
+
+            var security = AddData<USEnergyAPI>(spy, Resolution.Tick, null);
+            if (SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs(security.Symbol)
+                .Any(config => config.Resolution != Resolution.Hour))
+            {
+                throw new Exception("Was expecting resolution to be adjusted to Hour");
+            }
+
+            var option = AddOption("AAPL", Resolution.Daily);
+            if (SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs(option.Symbol)
+                .Any(config => config.Resolution != Resolution.Minute))
+            {
+                throw new Exception("Was expecting resolution to be adjusted to Minute");
+            }
+
+            Quit();
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Trades", "0"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "0"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$0.00"},
+        };
+    }
+}
diff --git a/Algorithm.CSharp/AltData/CachedAlternativeDataAlgorithm.cs b/Algorithm.CSharp/AltData/CachedAlternativeDataAlgorithm.cs
@@ -35,11 +35,11 @@ public override void Initialize()
             // QuantConnect caches a small subset of alternative data for easy consumption for the community.
             // You can use this in your algorithm as demonstrated below:
 
-            _cboeVix = AddData<CBOE>("VIX").Symbol;
+            _cboeVix = AddData<CBOE>("VIX", Resolution.Daily).Symbol;
             // United States EIA data: https://eia.gov/
-            _usEnergy = AddData<USEnergy>(USEnergy.Petroleum.UnitedStates.WeeklyGrossInputsIntoRefineries).Symbol;
+            _usEnergy = AddData<USEnergy>(USEnergy.Petroleum.UnitedStates.WeeklyGrossInputsIntoRefineries, Resolution.Daily).Symbol;
             // FRED data
-            _fredPeakToTrough = AddData<Fred>(Fred.OECDRecessionIndicators.UnitedStatesFromPeakThroughTheTrough).Symbol;
+            _fredPeakToTrough = AddData<Fred>(Fred.OECDRecessionIndicators.UnitedStatesFromPeakThroughTheTrough, Resolution.Daily).Symbol;
         }
 
         public override void OnData(Slice data)

diff --git a/Algorithm.CSharp/AltData/SECReport8KAlgorithm.cs b/Algorithm.CSharp/AltData/SECReport8KAlgorithm.cs
@@ -39,7 +39,7 @@ public override void Initialize()
             // Request underlying equity data.
             var ibm = AddEquity("IBM", Resolution.Minute).Symbol;
             // Add SEC report 10-Q data for the underlying IBM asset
-            var earningsFiling = AddData<SECReport10Q>(ibm).Symbol;
+            var earningsFiling = AddData<SECReport10Q>(ibm, Resolution.Daily).Symbol;
             // Request 120 days of history with the SECReport10Q IBM custom data Symbol.
             var history = History<SECReport10Q>(earningsFiling, 120, Resolution.Daily);
 

diff --git a/Algorithm.CSharp/AltData/USTreasuryYieldCurveRateAlgorithm.cs b/Algorithm.CSharp/AltData/USTreasuryYieldCurveRateAlgorithm.cs
@@ -33,7 +33,7 @@ public override void Initialize()
             SetCash(100000);
 
             _spy = AddEquity("SPY", Resolution.Hour).Symbol;
-            _yieldCurve = AddData<USTreasuryYieldCurveRate>("USTYCR").Symbol;
+            _yieldCurve = AddData<USTreasuryYieldCurveRate>("USTYCR", Resolution.Daily).Symbol;
 
             // Request 60 days of history with the USTreasuryYieldCurveRate custom data Symbol.
             var history = History<USTreasuryYieldCurveRate>(_yieldCurve, 60, Resolution.Daily);

diff --git a/Algorithm.CSharp/CustomDataUsingMapFileRegressionAlgorithm.cs b/Algorithm.CSharp/CustomDataUsingMapFileRegressionAlgorithm.cs
@@ -47,7 +47,15 @@ public override void Initialize()
             SetEndDate(2013, 07, 02);
 
             var foxa = QuantConnect.Symbol.Create("FOXA", SecurityType.Equity, Market.USA);
-            _symbol = AddData<CustomDataUsingMapping>(foxa).Symbol;
+            _symbol = AddData<CustomDataUsingMapping>(foxa, Resolution.Tick).Symbol;
+
+            foreach (var config in SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs(_symbol))
+            {
+                if (config.Resolution != Resolution.Minute)
+                {
+                    throw new Exception("Expected resolution to be adjust to Minute");
+                }
+            }
         }
 
         /// <summary>
@@ -164,6 +172,11 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date
             {
                 return ParseEquity(config, line, date);
             }
+
+            public override Resolution AdjustResolution(Resolution resolution)
+            {
+                return Resolution.Minute;
+            }
         }
     }
 }
diff --git a/Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj b/Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj
@@ -158,6 +158,7 @@
     <Compile Include="AltData\PsychSignalSentimentAlgorithm.cs" />
     <Compile Include="AltData\TradingEconomicsAlgorithm.cs" />
     <Compile Include="AltData\TiingoNewsAlgorithm.cs" />
+    <Compile Include="AdjustResolutionRegressionAlgorithm.cs" />
     <Compile Include="BasicPythonIntegrationTemplateAlgorithm.cs" />
     <Compile Include="BasicSetAccountCurrencyAlgorithm.cs" />
     <Compile Include="Benchmarks\SECReportBenchmarkAlgorithm.cs" />

diff --git a/Algorithm.CSharp/SECReportDataAlgorithm.cs b/Algorithm.CSharp/SECReportDataAlgorithm.cs
@@ -40,8 +40,8 @@ public override void Initialize()
             SetEndDate(2019, 1, 31);
             SetCash(100000);
 
-            _symbol = AddData<SECReport10Q>(Ticker).Symbol;
-            AddData<SECReport8K>(Ticker);
+            _symbol = AddData<SECReport10Q>(Ticker, Resolution.Daily).Symbol;
+            AddData<SECReport8K>(Ticker, Resolution.Daily);
         }
 
         public override void OnData(Slice slice)

diff --git a/Algorithm.CSharp/USEnergyInformationAdministrationAlgorithm.cs b/Algorithm.CSharp/USEnergyInformationAdministrationAlgorithm.cs
@@ -52,7 +52,7 @@ public override void Initialize()
             USEnergyAPI.SetAuthCode("my-us-energy-information-api-token");
 
             _tiingoSymbol = AddData<TiingoPrice>(tiingoTicker, Resolution.Daily).Symbol;
-            _energySymbol = AddData<USEnergyAPI>(energyTicker).Symbol;
+            _energySymbol = AddData<USEnergyAPI>(energyTicker, Resolution.Hour).Symbol;
 
             _emaFast = EMA(_tiingoSymbol, 5);
             _emaSlow = EMA(_tiingoSymbol, 10);

diff --git a/Algorithm.CSharp/USTreasuryYieldCurveDataAlgorithm.cs b/Algorithm.CSharp/USTreasuryYieldCurveDataAlgorithm.cs
@@ -36,7 +36,7 @@ public override void Initialize()
             SetCash(100000);
 
             // Since yield data isn't associated with any ticker, we must put a placeholder ticker
-            AddData<USTreasuryYieldCurveRate>("USTYC");
+            AddData<USTreasuryYieldCurveRate>("USTYC", Resolution.Daily);
         }
 
         public override void OnData(Slice slice)

diff --git a/Algorithm.Python/AltData/CachedAlternativeDataAlgorithm.py b/Algorithm.Python/AltData/CachedAlternativeDataAlgorithm.py
@@ -36,11 +36,11 @@ def Initialize(self):
         # QuantConnect caches a small subset of alternative data for easy consumption for the community.
         # You can use this in your algorithm as demonstrated below:
 
-        self.cboeVix = self.AddData(CBOE, "VIX").Symbol
+        self.cboeVix = self.AddData(CBOE, "VIX", Resolution.Daily).Symbol
         # United States EIA data: https://eia.gov/
-        self.usEnergy = self.AddData(USEnergy, USEnergy.Petroleum.UnitedStates.WeeklyGrossInputsIntoRefineries).Symbol
+        self.usEnergy = self.AddData(USEnergy, USEnergy.Petroleum.UnitedStates.WeeklyGrossInputsIntoRefineries, Resolution.Daily).Symbol
         # FRED data
-        self.fredPeakToTrough = self.AddData(Fred, Fred.OECDRecessionIndicators.UnitedStatesFromPeakThroughTheTrough).Symbol
+        self.fredPeakToTrough = self.AddData(Fred, Fred.OECDRecessionIndicators.UnitedStatesFromPeakThroughTheTrough, Resolution.Daily).Symbol
 
     def OnData(self, data):
         if data.ContainsKey(self.cboeVix):

diff --git a/Algorithm.Python/AltData/SECReport8KAlgorithm.py b/Algorithm.Python/AltData/SECReport8KAlgorithm.py
@@ -37,7 +37,7 @@ def Initialize(self):
         # Request underlying equity data.
         ibm = self.AddEquity("IBM", Resolution.Minute).Symbol
         # Add news data for the underlying IBM asset
-        earningsFiling = self.AddData(SECReport10Q, ibm).Symbol
+        earningsFiling = self.AddData(SECReport10Q, ibm, Resolution.Daily).Symbol
         # Request 120 days of history with the SECReport10Q IBM custom data Symbol
         history = self.History(SECReport10Q, earningsFiling, 120, Resolution.Daily)
 

diff --git a/Algorithm.Python/AltData/USTreasuryYieldCurveRateAlgorithm.py b/Algorithm.Python/AltData/USTreasuryYieldCurveRateAlgorithm.py
@@ -33,7 +33,7 @@ def Initialize(self):
         self.SetCash(100000)
 
         self.spy = self.AddEquity("SPY", Resolution.Hour).Symbol
-        self.yieldCurve = self.AddData(USTreasuryYieldCurveRate, "USTYCR").Symbol
+        self.yieldCurve = self.AddData(USTreasuryYieldCurveRate, "USTYCR", Resolution.Daily).Symbol
         self.lastInversion = datetime(1, 1, 1)
 
         # Request 60 days of history with the USTreasuryYieldCurveRate custom data Symbol.

diff --git a/Algorithm.Python/CustomDataUsingMapFileRegressionAlgorithm.py b/Algorithm.Python/CustomDataUsingMapFileRegressionAlgorithm.py
@@ -44,7 +44,11 @@ def Initialize(self):
         self.initialMapping = False
         self.executionMapping = False
         self.foxa = Symbol.Create("FOXA", SecurityType.Equity, Market.USA)
-        self.symbol = self.AddData(CustomDataUsingMapping, self.foxa).Symbol
+        self.symbol = self.AddData(CustomDataUsingMapping, self.foxa, Resolution.Tick).Symbol
+
+        for config in self.SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs(self.symbol):
+            if config.Resolution != Resolution.Minute:
+                raise ValueError("Expected resolution to be adjust to Minute")
 
     def OnData(self, slice):
         date = self.Time.date()
@@ -86,4 +90,8 @@ def Reader(self, config, line, date, isLiveMode):
 
     def RequiresMapping(self):
         '''True indicates mapping should be done'''
-        return True
+        return True
+
+    def AdjustResolution(self, resolution):
+        '''Adjust resolution'''
+        return Resolution.Minute
diff --git a/Algorithm.Python/SECReportDataAlgorithm.py b/Algorithm.Python/SECReportDataAlgorithm.py
@@ -37,8 +37,8 @@ def Initialize(self):
         self.SetCash(100000)
 
         self.ticker = "AAPL"
-        self.symbol = self.AddData(SECReport10Q, self.ticker).Symbol
-        self.AddData(SECReport8K, self.ticker)
+        self.symbol = self.AddData(SECReport10Q, self.ticker, Resolution.Daily).Symbol
+        self.AddData(SECReport8K, self.ticker, Resolution.Daily)
 
     def OnData(self, slice):
         # OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.

diff --git a/Algorithm.Python/USEnergyInformationAdministrationAlgorithm.py b/Algorithm.Python/USEnergyInformationAdministrationAlgorithm.py
@@ -46,7 +46,7 @@ def Initialize(self):
         self.tiingoTicker = "AAPL"
         self.energyTicker = "NUC_STATUS.OUT.US.D"
         self.tiingoSymbol = self.AddData(TiingoDailyData, self.tiingoTicker, Resolution.Daily).Symbol
-        self.energySymbol = self.AddData(USEnergyAPI, self.energyTicker).Symbol
+        self.energySymbol = self.AddData(USEnergyAPI, self.energyTicker, Resolution.Hour).Symbol
 
 
         self.emaFast = self.EMA(self.tiingoSymbol, 5)

diff --git a/Algorithm.Python/USTreasuryYieldCurveDataAlgorithm.py b/Algorithm.Python/USTreasuryYieldCurveDataAlgorithm.py
@@ -35,7 +35,7 @@ def Initialize(self):
         self.SetCash(100000)
 
         # Define the symbol and "type" of our generic data:
-        self.symbol = self.AddData(USTreasuryYieldCurveRate, "USTYC").Symbol
+        self.symbol = self.AddData(USTreasuryYieldCurveRate, "USTYC", Resolution.Daily).Symbol
 
     def OnData(self, slice):
         if not slice.ContainsKey(self.symbol):

diff --git a/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs b/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs
@@ -44,9 +44,8 @@ public class AlgorithmPythonWrapper : IAlgorithm
         private readonly dynamic _algorithm;
         private readonly dynamic _onData;
         private readonly dynamic _onOrderEvent;
+        private readonly dynamic _onMarginCall;
         private readonly IAlgorithm _baseAlgorithm;
-        private readonly bool _isOnDataDefined;
-        private readonly bool _isOnMaginCallDefined;
 
         /// <summary>
         /// <see cref = "AlgorithmPythonWrapper"/> constructor.
@@ -85,14 +84,12 @@ public AlgorithmPythonWrapper(string moduleName)
 
                             // determines whether OnData method was defined or inherits from QCAlgorithm
                             // If it is not, OnData from the base class will not be called
-                            _onData = (_algorithm as PyObject).GetAttr("OnData");
-                            var onDataPythonType = _onData.GetPythonType();
-                            _isOnDataDefined = onDataPythonType.Repr().Equals("<class \'method\'>");
+                            var pyAlgorithm = _algorithm as PyObject;
+                            _onData = pyAlgorithm.GetPythonMethod("OnData");
 
-                            var onMarginCallPythonType = _algorithm.OnMarginCall.GetPythonType();
-                            _isOnMaginCallDefined = onMarginCallPythonType.Repr().Equals("<class \'method\'>");
+                            _onMarginCall = pyAlgorithm.GetPythonMethod("OnMarginCall");
 
-                            _onOrderEvent = (_algorithm as PyObject).GetAttr("OnOrderEvent");
+                            _onOrderEvent = pyAlgorithm.GetAttr("OnOrderEvent");
                         }
                         attr.Dispose();
                     }
@@ -545,7 +542,7 @@ public void OnBrokerageReconnect()
         /// <param name="slice">The current slice of data</param>
         public void OnData(Slice slice)
         {
-            if (_isOnDataDefined)
+            if (_onData != null)
             {
                 using (Py.GIL())
                 {
@@ -642,7 +639,7 @@ public void OnMarginCall(List<SubmitOrderRequest> requests)
             {
                 var result = _algorithm.OnMarginCall(requests);
 
-                if (_isOnMaginCallDefined)
+                if (_onMarginCall != null)
                 {
                     var pyRequests = result as PyObject;
                     // If the method does not return or returns a non-iterable PyObject, throw an exception

diff --git a/Common/Data/BaseData.cs b/Common/Data/BaseData.cs
@@ -144,6 +144,7 @@ public virtual SubscriptionDataSource GetSource(SubscriptionDataConfig config, D
         /// <summary>
         /// Indicates if there is support for mapping
         /// </summary>
+        /// <remarks>Relies on the <see cref="Symbol"/> property value</remarks>
         /// <returns>True indicates mapping should be used</returns>
         public virtual bool RequiresMapping()
         {
@@ -153,13 +154,25 @@ public virtual bool RequiresMapping()
         /// <summary>
         /// Indicates that the data set is expected to be sparse
         /// </summary>
+        /// <remarks>Relies on the <see cref="Symbol"/> property value</remarks>
         /// <returns>True if the data set represented by this type is expected to be sparse</returns>
         public virtual bool IsSparseData()
         {
             // by default, we'll assume all custom data is sparse data
             return Symbol.SecurityType == SecurityType.Base;
         }
 
+        /// <summary>
+        /// Will adjust the requested resolution to match a supported one
+        /// for the current data and security type
+        /// </summary>
+        /// <remarks>Relies on the <see cref="Symbol"/> property value</remarks>
+        /// <param name="resolution">The resolution to check support</param>
+        public virtual Resolution AdjustResolution(Resolution resolution)
+        {
+            return Symbol.SecurityType == SecurityType.Option ? Resolution.Minute : resolution;
+        }
+
         /// <summary>
         /// Specifies the data time zone for this data type. This is useful for custom data types
         /// </summary>