Refactor Api (QuantConnect#5251)

* Use lean data key as param for request * key -> filePath rename and some cleanup * Refactor * Add Organizations Endpoints * Add some organization api wrapper objects * Address namespace issue * Reorganize Api Test into seperate files using one ApiTestBase * Add Organization tests * Use capitalized "API" test namespace to reduce amount of file changes * Add License to test base * Update /data endpoint functions and response objects * Update ApiDataProvider Logic * Handle deserialization of organization products * Simplify converter * Only throw for equity requests when not subscribed to map/factor files * Add missing header * Make arguement exception * Api adjustments * Add Zip factor and map file providers - Common project will now reference Compression project and not the other way round. - Adding Zip FactorFile and MapFile providers * Refactor FactorFileProvider to use DataProvider to fetch files * Use resulting MinimumDate in construction of FactorFile * Nit FactorFile comments and arrangement * Refactor MapFileProviders to use DataProvider for fetching files * Refactor ZipFileProvider * Clean up * Refactor Backtesting Future/Option chain providers to use dataprovider * Fixes for data/ endpoints and test adjustments * Response objects adjustments/cleanups * ApiDateProvider fixes and testing * Add LocalZipFactorFileTests * Update ApiDataProvider download test to verify stream is not null * Implement posting of agreement summary and signed time * Mark all Api related tests as explicit and document details on running * Clarify default token on ApiTestBase * Adjust summary * Update Api responses for QCC, except org products which are sold in USD * Implement cache expiration for zip MapFile and FactorFiles. Adding unit tests * Fix multiple markets for ZipFactorFile provider * Use Symbol as cache key * Api.cs review * Dispose of factorFileStream after reading * Use zip.EntryFileNames * Address a few reviews * Few more fixes * Address Api Review * Add Job Org id to config * Minor tweaks * Compare with invariant culture * Fixes Option Universe selection * ZipEntryNameSubscriptionDataSourceReader will use IDataProvider * Fix research * Fix null reference exception * Make duplicate log debug Co-authored-by: Martin-Molinero <[email protected]>
Coulbe · Jun 7, 2021 · ab60277 · ab60277
1 parent 37473c4
commit ab60277
Show file tree

Hide file tree

Showing 102 changed files with 2,650 additions and 1,019 deletions.
diff --git a/Algorithm.CSharp/AdjustedVolumeRegressionAlgorithm.cs b/Algorithm.CSharp/AdjustedVolumeRegressionAlgorithm.cs
@@ -15,9 +15,11 @@
 
 using System;
 using System.Collections.Generic;
+using QuantConnect.Configuration;
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Interfaces;
+using QuantConnect.Util;
 
 namespace QuantConnect.Algorithm.CSharp
 {
@@ -28,7 +30,7 @@ public class AdjustedVolumeRegressionAlgorithm : QCAlgorithm, IRegressionAlgorit
     {
         private Symbol _aapl;
         private const string Ticker = "AAPL";
-        private readonly FactorFile _factorFile = FactorFile.Read(Ticker, Market.USA);
+        private FactorFile _factorFile;
         private readonly IEnumerator<decimal> _expectedAdjustedVolume = new List<decimal> { 1541213, 761013, 920088, 867077, 542487, 663132, 
             374927, 379554, 413805, 377622 }.GetEnumerator();
         private readonly IEnumerator<decimal> _expectedAdjustedAskSize = new List<decimal> { 53900, 1400, 6300, 2100, 1400, 1400, 700, 
@@ -43,6 +45,18 @@ public override void Initialize()
 
             UniverseSettings.DataNormalizationMode = DataNormalizationMode.SplitAdjusted;
             _aapl = AddEquity(Ticker, Resolution.Minute).Symbol;
+
+            var dataProvider =
+                Composer.Instance.GetExportedValueByTypeName<IDataProvider>(Config.Get("data-provider",
+                    "DefaultDataProvider"));
+
+            var mapFileProvider = new LocalDiskMapFileProvider();
+            mapFileProvider.Initialize(dataProvider);
+            var factorFileProvider = new LocalDiskFactorFileProvider();
+            factorFileProvider.Initialize(mapFileProvider, dataProvider);
+
+
+            _factorFile = factorFileProvider.Get(_aapl);
         }
 
         /// <summary>

diff --git a/Algorithm.CSharp/RawDataRegressionAlgorithm.cs b/Algorithm.CSharp/RawDataRegressionAlgorithm.cs
@@ -15,9 +15,11 @@
 
 using System;
 using System.Collections.Generic;
+using QuantConnect.Configuration;
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Interfaces;
+using QuantConnect.Util;
 
 namespace QuantConnect.Algorithm.CSharp
 {
@@ -30,7 +32,7 @@ namespace QuantConnect.Algorithm.CSharp
     public class RawDataRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         private const string Ticker = "GOOGL";
-        private readonly FactorFile _factorFile = FactorFile.Read(Ticker, Market.USA);
+        private FactorFile _factorFile;
         private readonly IEnumerator<decimal> _expectedRawPrices = new List<decimal> { 1158.1100m, 1158.7200m,
             1131.7800m, 1114.2800m, 1119.6100m, 1114.5500m, 1135.3200m, 567.59000m, 571.4900m, 545.3000m, 540.6400m }.GetEnumerator();
         private Symbol _googl;
@@ -45,6 +47,17 @@ public override void Initialize()
             UniverseSettings.DataNormalizationMode = DataNormalizationMode.Raw;
             _googl = AddEquity(Ticker, Resolution.Daily).Symbol;
 
+            // Get our factor file for this regression
+            var dataProvider =
+                Composer.Instance.GetExportedValueByTypeName<IDataProvider>(Config.Get("data-provider",
+                    "DefaultDataProvider"));
+
+            var mapFileProvider = new LocalDiskMapFileProvider();
+            mapFileProvider.Initialize(dataProvider);
+            var factorFileProvider = new LocalDiskFactorFileProvider();
+            factorFileProvider.Initialize(mapFileProvider, dataProvider);
+            _factorFile = factorFileProvider.Get(_googl);
+
             // Prime our expected values
             _expectedRawPrices.MoveNext();
         }

diff --git a/Algorithm.Python/RawDataRegressionAlgorithm.py b/Algorithm.Python/RawDataRegressionAlgorithm.py
@@ -15,17 +15,22 @@
 AddReference("System.Core")
 AddReference("QuantConnect.Common")
 AddReference("QuantConnect.Algorithm")
+AddReference("QuantConnect.Configuration")
+AddReference("QuantConnect.Lean.Engine")
 
 from System import *
 from QuantConnect import *
 from QuantConnect.Algorithm import QCAlgorithm
-from QuantConnect.Data.Auxiliary import FactorFile
+from QuantConnect.Data.Auxiliary import *
 from QuantConnect.Data.UniverseSelection import *
 from QuantConnect.Orders import OrderStatus
 from QuantConnect.Orders.Fees import ConstantFeeModel
+from QuantConnect.Configuration import Config
+from QuantConnect.Util import Composer
+from QuantConnect.Interfaces import IDataProvider
+from QuantConnect.Lean.Engine.DataFeeds import DefaultDataProvider
 
 _ticker = "GOOGL";
-_factorFile = FactorFile.Read(_ticker, Market.USA);
 _expectedRawPrices = [ 1158.1100, 1158.7200,
 1131.7800, 1114.2800, 1119.6100, 1114.5500, 1135.3200, 567.59000, 571.4900, 545.3000, 540.6400 ]
 
@@ -45,6 +50,16 @@ def Initialize(self):
             # Set our DataNormalizationMode to raw
             self.UniverseSettings.DataNormalizationMode = DataNormalizationMode.Raw;
             self._googl = self.AddEquity(_ticker, Resolution.Daily).Symbol;
+
+            # Get our factor file for this regression
+            dataProvider = DefaultDataProvider();
+            mapFileProvider = LocalDiskMapFileProvider();
+            mapFileProvider.Initialize(dataProvider);
+            factorFileProvider = LocalDiskFactorFileProvider();
+            factorFileProvider.Initialize(mapFileProvider, dataProvider);
+
+            # Get our factor file for this regression
+            self._factorFile = factorFileProvider.Get(self._googl);
 
 
         def OnData(self, data):
@@ -57,7 +72,7 @@ def OnData(self, data):
                 # Assert our volume matches what we expected
                 if _expectedRawPrices.pop(0) != googlData.Close:
                     # Our values don't match lets try and give a reason why
-                    dayFactor = _factorFile.GetPriceScaleFactor(googlData.Time);
+                    dayFactor = self._factorFile.GetPriceScaleFactor(googlData.Time);
                     probableRawPrice = googlData.Close / dayFactor; # Undo adjustment
 
                     if _expectedRawPrices.Current == probableRawPrice: