diff --git a/Algorithm.CSharp/AdjustedVolumeRegressionAlgorithm.cs b/Algorithm.CSharp/AdjustedVolumeRegressionAlgorithm.cs index 83af59e0ebdf..409772fb0828 100644 --- a/Algorithm.CSharp/AdjustedVolumeRegressionAlgorithm.cs +++ b/Algorithm.CSharp/AdjustedVolumeRegressionAlgorithm.cs @@ -15,9 +15,11 @@ using System; using System.Collections.Generic; +using QuantConnect.Configuration; using QuantConnect.Data; using QuantConnect.Data.Auxiliary; using QuantConnect.Interfaces; +using QuantConnect.Util; namespace QuantConnect.Algorithm.CSharp { @@ -28,7 +30,7 @@ public class AdjustedVolumeRegressionAlgorithm : QCAlgorithm, IRegressionAlgorit { private Symbol _aapl; private const string Ticker = "AAPL"; - private readonly FactorFile _factorFile = FactorFile.Read(Ticker, Market.USA); + private FactorFile _factorFile; private readonly IEnumerator _expectedAdjustedVolume = new List { 1541213, 761013, 920088, 867077, 542487, 663132, 374927, 379554, 413805, 377622 }.GetEnumerator(); private readonly IEnumerator _expectedAdjustedAskSize = new List { 53900, 1400, 6300, 2100, 1400, 1400, 700, @@ -43,6 +45,18 @@ public override void Initialize() UniverseSettings.DataNormalizationMode = DataNormalizationMode.SplitAdjusted; _aapl = AddEquity(Ticker, Resolution.Minute).Symbol; + + var dataProvider = + Composer.Instance.GetExportedValueByTypeName(Config.Get("data-provider", + "DefaultDataProvider")); + + var mapFileProvider = new LocalDiskMapFileProvider(); + mapFileProvider.Initialize(dataProvider); + var factorFileProvider = new LocalDiskFactorFileProvider(); + factorFileProvider.Initialize(mapFileProvider, dataProvider); + + + _factorFile = factorFileProvider.Get(_aapl); } /// diff --git a/Algorithm.CSharp/RawDataRegressionAlgorithm.cs b/Algorithm.CSharp/RawDataRegressionAlgorithm.cs index 4990336ff134..4fe51ae208c6 100644 --- a/Algorithm.CSharp/RawDataRegressionAlgorithm.cs +++ b/Algorithm.CSharp/RawDataRegressionAlgorithm.cs @@ -15,9 +15,11 @@ using System; using System.Collections.Generic; +using QuantConnect.Configuration; using QuantConnect.Data; using QuantConnect.Data.Auxiliary; using QuantConnect.Interfaces; +using QuantConnect.Util; namespace QuantConnect.Algorithm.CSharp { @@ -30,7 +32,7 @@ namespace QuantConnect.Algorithm.CSharp public class RawDataRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition { private const string Ticker = "GOOGL"; - private readonly FactorFile _factorFile = FactorFile.Read(Ticker, Market.USA); + private FactorFile _factorFile; private readonly IEnumerator _expectedRawPrices = new List { 1158.1100m, 1158.7200m, 1131.7800m, 1114.2800m, 1119.6100m, 1114.5500m, 1135.3200m, 567.59000m, 571.4900m, 545.3000m, 540.6400m }.GetEnumerator(); private Symbol _googl; @@ -45,6 +47,17 @@ public override void Initialize() UniverseSettings.DataNormalizationMode = DataNormalizationMode.Raw; _googl = AddEquity(Ticker, Resolution.Daily).Symbol; + // Get our factor file for this regression + var dataProvider = + Composer.Instance.GetExportedValueByTypeName(Config.Get("data-provider", + "DefaultDataProvider")); + + var mapFileProvider = new LocalDiskMapFileProvider(); + mapFileProvider.Initialize(dataProvider); + var factorFileProvider = new LocalDiskFactorFileProvider(); + factorFileProvider.Initialize(mapFileProvider, dataProvider); + _factorFile = factorFileProvider.Get(_googl); + // Prime our expected values _expectedRawPrices.MoveNext(); } diff --git a/Algorithm.Python/RawDataRegressionAlgorithm.py b/Algorithm.Python/RawDataRegressionAlgorithm.py index f77b6f2e8dd7..1535901b7026 100644 --- a/Algorithm.Python/RawDataRegressionAlgorithm.py +++ b/Algorithm.Python/RawDataRegressionAlgorithm.py @@ -15,17 +15,22 @@ AddReference("System.Core") AddReference("QuantConnect.Common") AddReference("QuantConnect.Algorithm") +AddReference("QuantConnect.Configuration") +AddReference("QuantConnect.Lean.Engine") from System import * from QuantConnect import * from QuantConnect.Algorithm import QCAlgorithm -from QuantConnect.Data.Auxiliary import FactorFile +from QuantConnect.Data.Auxiliary import * from QuantConnect.Data.UniverseSelection import * from QuantConnect.Orders import OrderStatus from QuantConnect.Orders.Fees import ConstantFeeModel +from QuantConnect.Configuration import Config +from QuantConnect.Util import Composer +from QuantConnect.Interfaces import IDataProvider +from QuantConnect.Lean.Engine.DataFeeds import DefaultDataProvider _ticker = "GOOGL"; -_factorFile = FactorFile.Read(_ticker, Market.USA); _expectedRawPrices = [ 1158.1100, 1158.7200, 1131.7800, 1114.2800, 1119.6100, 1114.5500, 1135.3200, 567.59000, 571.4900, 545.3000, 540.6400 ] @@ -45,6 +50,16 @@ def Initialize(self): # Set our DataNormalizationMode to raw self.UniverseSettings.DataNormalizationMode = DataNormalizationMode.Raw; self._googl = self.AddEquity(_ticker, Resolution.Daily).Symbol; + + # Get our factor file for this regression + dataProvider = DefaultDataProvider(); + mapFileProvider = LocalDiskMapFileProvider(); + mapFileProvider.Initialize(dataProvider); + factorFileProvider = LocalDiskFactorFileProvider(); + factorFileProvider.Initialize(mapFileProvider, dataProvider); + + # Get our factor file for this regression + self._factorFile = factorFileProvider.Get(self._googl); def OnData(self, data): @@ -57,7 +72,7 @@ def OnData(self, data): # Assert our volume matches what we expected if _expectedRawPrices.pop(0) != googlData.Close: # Our values don't match lets try and give a reason why - dayFactor = _factorFile.GetPriceScaleFactor(googlData.Time); + dayFactor = self._factorFile.GetPriceScaleFactor(googlData.Time); probableRawPrice = googlData.Close / dayFactor; # Undo adjustment if _expectedRawPrices.Current == probableRawPrice: diff --git a/Api/Api.cs b/Api/Api.cs index 3a1576e60680..53153ba0cbb0 100644 --- a/Api/Api.cs +++ b/Api/Api.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -16,14 +16,15 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; +using System.Net.Http; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using QuantConnect.Interfaces; using QuantConnect.Logging; using QuantConnect.Orders; using RestSharp; -using RestSharp.Extensions; using QuantConnect.Util; namespace QuantConnect.Api @@ -41,7 +42,7 @@ public class Api : IApi, IDownloadProvider protected ApiConnection ApiConnection { get; private set; } /// - /// Initialize the API using the config.json file. + /// Initialize the API with the given variables /// public virtual void Initialize(int userId, string token, string dataFolder) { @@ -58,36 +59,47 @@ public virtual void Initialize(int userId, string token, string dataFolder) /// /// Check if Api is successfully connected with correct credentials /// - public bool Connected - { - get - { - return ApiConnection.Connected; - } - } + public bool Connected => ApiConnection.Connected; /// /// Create a project with the specified name and language via QuantConnect.com API /// /// Project name /// Programming language to use + /// Optional param for specifying organization to create project under. + /// If none provided web defaults to preferred. /// Project object from the API. - public ProjectResponse CreateProject(string name, Language language) + public ProjectResponse CreateProject(string name, Language language, string organizationId = null) { var request = new RestRequest("projects/create", Method.POST) { RequestFormat = DataFormat.Json }; - request.AddParameter("application/json", JsonConvert.SerializeObject(new + // Only include organization Id if its not null or empty + string jsonParams; + if (string.IsNullOrEmpty(organizationId)) { - name, - language - }), ParameterType.RequestBody); + jsonParams = JsonConvert.SerializeObject(new + { + name, + language + }); + } + else + { + jsonParams = JsonConvert.SerializeObject(new + { + name, + language, + organizationId + }); + } - ProjectResponse result; - ApiConnection.TryRequest(request, out result); + request.AddParameter("application/json", jsonParams, ParameterType.RequestBody); + + ApiConnection.TryRequest(request, out ProjectResponse result); return result; } @@ -109,8 +121,7 @@ public ProjectResponse ReadProject(int projectId) projectId }), ParameterType.RequestBody); - ProjectResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out ProjectResponse result); return result; } @@ -126,8 +137,7 @@ public ProjectResponse ListProjects() RequestFormat = DataFormat.Json }; - ProjectResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out ProjectResponse result); return result; } @@ -154,8 +164,7 @@ public ProjectFilesResponse AddProjectFile(int projectId, string name, string co content }), ParameterType.RequestBody); - ProjectFilesResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out ProjectFilesResponse result); return result; } @@ -182,8 +191,7 @@ public RestResponse UpdateProjectFileName(int projectId, string oldFileName, str newName = newFileName }), ParameterType.RequestBody); - RestResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out RestResponse result); return result; } @@ -210,8 +218,7 @@ public RestResponse UpdateProjectFileContent(int projectId, string fileName, str content = newFileContents }), ParameterType.RequestBody); - RestResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out RestResponse result); return result; } @@ -234,8 +241,7 @@ public ProjectFilesResponse ReadProjectFiles(int projectId) projectId }), ParameterType.RequestBody); - ProjectFilesResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out ProjectFilesResponse result); return result; } @@ -260,8 +266,7 @@ public ProjectFilesResponse ReadProjectFile(int projectId, string fileName) name = fileName }), ParameterType.RequestBody); - ProjectFilesResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out ProjectFilesResponse result); return result; } @@ -285,8 +290,7 @@ public RestResponse DeleteProjectFile(int projectId, string name) name, }), ParameterType.RequestBody); - RestResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out RestResponse result); return result; } @@ -308,8 +312,7 @@ public RestResponse DeleteProject(int projectId) projectId }), ParameterType.RequestBody); - RestResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out RestResponse result); return result; } @@ -331,8 +334,7 @@ public Compile CreateCompile(int projectId) projectId }), ParameterType.RequestBody); - Compile result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out Compile result); return result; } @@ -356,8 +358,7 @@ public Compile ReadCompile(int projectId, string compileId) compileId }), ParameterType.RequestBody); - Compile result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out Compile result); return result; } @@ -384,8 +385,7 @@ public Backtest CreateBacktest(int projectId, string compileId, string backtestN backtestName }), ParameterType.RequestBody); - BacktestResponseWrapper result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out BacktestResponseWrapper result); // Use API Response values for Backtest Values result.Backtest.Success = result.Success; @@ -416,8 +416,7 @@ public Backtest ReadBacktest(int projectId, string backtestId, bool getCharts = backtestId }), ParameterType.RequestBody); - BacktestResponseWrapper result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out BacktestResponseWrapper result); if (!result.Success) { @@ -450,8 +449,7 @@ public Backtest ReadBacktest(int projectId, string backtestId, bool getCharts = chart = chart.Key.Replace(' ', '+') }), ParameterType.RequestBody); - BacktestResponseWrapper chartResponse; - ApiConnection.TryRequest(chartRequest, out chartResponse); + ApiConnection.TryRequest(chartRequest, out BacktestResponseWrapper chartResponse); // Add this chart to our updated collection if (chartResponse.Success) @@ -499,8 +497,7 @@ public RestResponse UpdateBacktest(int projectId, string backtestId, string name note }), ParameterType.RequestBody); - Backtest result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out Backtest result); return result; } @@ -522,8 +519,7 @@ public BacktestList ListBacktests(int projectId) projectId, }), ParameterType.RequestBody); - BacktestList result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out BacktestList result); return result; } @@ -547,8 +543,7 @@ public RestResponse DeleteBacktest(int projectId, string backtestId) backtestId }), ParameterType.RequestBody); - RestResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out RestResponse result); return result; } @@ -633,8 +628,7 @@ public LiveList ListLiveAlgorithms(AlgorithmStatus? status = null, request.AddParameter("application/json", JsonConvert.SerializeObject(obj), ParameterType.RequestBody); - LiveList result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out LiveList result); return result; } @@ -658,8 +652,7 @@ public LiveAlgorithmResults ReadLiveAlgorithm(int projectId, string deployId) deployId }), ParameterType.RequestBody); - LiveAlgorithmResults result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out LiveAlgorithmResults result); return result; } @@ -681,8 +674,7 @@ public RestResponse LiquidateLiveAlgorithm(int projectId) projectId }), ParameterType.RequestBody); - RestResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out RestResponse result); return result; } @@ -704,8 +696,7 @@ public RestResponse StopLiveAlgorithm(int projectId) projectId }), ParameterType.RequestBody); - RestResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out RestResponse result); return result; } @@ -737,21 +728,26 @@ public LiveLog ReadLiveLogs(int projectId, string algorithmId, DateTime? startTi end = epochEndTime }), ParameterType.RequestBody); - LiveLog result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out LiveLog result); return result; } /// /// Gets the link to the downloadable data. /// - /// Symbol of security of which data will be requested. - /// Resolution of data requested. - /// Date of the data requested. + /// File path representing the data requested + /// Organization to download from /// to the downloadable data. - - public Link ReadDataLink(Symbol symbol, Resolution resolution, DateTime date) + public DataLink ReadDataLink(string filePath, string organizationId) { + if (filePath == null) + { + throw new ArgumentException("Api.ReadDataLink(): Filepath must not be null"); + } + + // Prepare filePath for request + filePath = FormatPathForDataRequest(filePath); + var request = new RestRequest("data/read", Method.POST) { RequestFormat = DataFormat.Json @@ -760,15 +756,66 @@ public Link ReadDataLink(Symbol symbol, Resolution resolution, DateTime date) request.AddParameter("application/json", JsonConvert.SerializeObject(new { format = "link", - ticker = symbol.Value.ToLowerInvariant(), - type = symbol.ID.SecurityType.ToLower(), - market = symbol.ID.Market, - resolution = resolution.ToString(), - date = date.ToStringInvariant("yyyyMMdd") + filePath, + organizationId }), ParameterType.RequestBody); - Link result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out DataLink result); + return result; + } + + /// + /// Get valid data entries for a given filepath from data/list + /// + /// + public DataList ReadDataDirectory(string filePath) + { + if (filePath == null) + { + throw new ArgumentException("Api.ReadDataDirectory(): Filepath must not be null"); + } + + // Prepare filePath for request + filePath = FormatPathForDataRequest(filePath); + + // Verify the filePath for this request is at least three directory deep + // (requirement of endpoint) + if (filePath.Count(x => x == '/') < 3) + { + throw new ArgumentException($"Api.ReadDataDirectory(): Data directory requested must be at least" + + $" three directories deep. FilePath: {filePath}"); + } + + var request = new RestRequest("data/list", Method.POST) + { + RequestFormat = DataFormat.Json + }; + + request.AddParameter("application/json", JsonConvert.SerializeObject(new + { + filePath + }), ParameterType.RequestBody); + + ApiConnection.TryRequest(request, out DataList result); + return result; + } + + /// + /// Gets data prices from data/prices + /// + public DataPricesList ReadDataPrices(string organizationId) + { + var request = new RestRequest("data/prices", Method.POST) + { + RequestFormat = DataFormat.Json + }; + + request.AddParameter("application/json", JsonConvert.SerializeObject(new + { + organizationId + }), ParameterType.RequestBody); + + ApiConnection.TryRequest(request, out DataPricesList result); return result; } @@ -791,61 +838,50 @@ public BacktestReport ReadBacktestReport(int projectId, string backtestId) projectId }), ParameterType.RequestBody); - BacktestReport report; - ApiConnection.TryRequest(request, out report); + ApiConnection.TryRequest(request, out BacktestReport report); return report; } /// - /// Method to download and save the data purchased through QuantConnect + /// Method to purchase and download data from QuantConnect /// - /// Symbol of security of which data will be requested. - /// Resolution of data requested. - /// Date of the data requested. + /// File path representing the data requested + /// Organization to buy the data with /// A indicating whether the data was successfully downloaded or not. - public bool DownloadData(Symbol symbol, Resolution resolution, DateTime date) + public bool DownloadData(string filePath, string organizationId) { // Get a link to the data - var link = ReadDataLink(symbol, resolution, date); + var dataLink = ReadDataLink(filePath, organizationId); // Make sure the link was successfully retrieved - if (!link.Success) + if (!dataLink.Success) return false; - // Save csv in same folder heirarchy as Lean - var path = Path.Combine(_dataFolder, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, date, resolution)); - // Make sure the directory exist before writing - (new FileInfo(path)).Directory.Create(); + var directory = Path.GetDirectoryName(filePath); + if (!Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } - // Download and save the data - var uri = new Uri(link.DataLink); - var client = new RestClient(uri.Scheme + "://" + uri.Host); - var request = new RestRequest(uri.PathAndQuery, Method.GET); + try + { + // Download the file + var uri = new Uri(dataLink.Url); - // MAke a request for the data at the link - var response = client.Execute(request); + using var client = new HttpClient(); + using var dataStream = client.GetStreamAsync(uri); - // If the response is JSON it doesn't contain any data, try and extract the message and write it - if (response.ContentType.ToLowerInvariant() == "application/json") + using var fileStream = new FileStream(filePath, FileMode.Create); + dataStream.Result.CopyTo(fileStream); + } + catch { - try - { - var contentObj = JObject.Parse(response.Content); - var message = contentObj["message"].Value(); - Log.Error($"Api.DownloadData(): Failed to download zip for {symbol} {resolution} data for date {date}, Api response: {message}"); - } - catch - { - Log.Error($"Api.DownloadData(): Failed to download zip for {symbol} {resolution} data for date {date}. Api response could not be parsed."); - } - + Log.Error($"Api.DownloadData(): Failed to download zip for path ({filePath})"); return false; } - - // Any other case save the content to given path - response.RawBytes.SaveAs(path); + return true; } @@ -991,8 +1027,7 @@ public CreatedNode CreateNode(string name, string organizationId, SKU sku) sku = sku.ToString() }), ParameterType.RequestBody); - CreatedNode result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out CreatedNode result); return result; } @@ -1014,8 +1049,7 @@ public NodeList ReadNodes(string organizationId) organizationId, }), ParameterType.RequestBody); - NodeList result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out NodeList result); return result; } @@ -1040,8 +1074,7 @@ public RestResponse UpdateNode(string nodeId, string newName, string organizatio organizationId }), ParameterType.RequestBody); - RestResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out RestResponse result); return result; } @@ -1064,8 +1097,7 @@ public RestResponse DeleteNode(string nodeId, string organizationId) organizationId }), ParameterType.RequestBody); - RestResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out RestResponse result); return result; } @@ -1088,8 +1120,7 @@ public RestResponse StopNode(string nodeId, string organizationId) organizationId }), ParameterType.RequestBody); - RestResponse result; - ApiConnection.TryRequest(request, out result); + ApiConnection.TryRequest(request, out RestResponse result); return result; } @@ -1099,7 +1130,7 @@ public RestResponse StopNode(string nodeId, string organizationId) /// The target organization id, if null will return default organization public Account ReadAccount(string organizationId = null) { - var request = new RestRequest("account/read/", Method.POST) + var request = new RestRequest("account/read", Method.POST) { RequestFormat = DataFormat.Json }; @@ -1109,9 +1140,63 @@ public Account ReadAccount(string organizationId = null) request.AddParameter("application/json", JsonConvert.SerializeObject(new { organizationId }), ParameterType.RequestBody); } - Account account; - ApiConnection.TryRequest(request, out account); + ApiConnection.TryRequest(request, out Account account); return account; } + + /// + /// Get a list of organizations tied to this account + /// + /// + public List ListOrganizations() + { + var request = new RestRequest("organizations/list", Method.POST) + { + RequestFormat = DataFormat.Json + }; + + ApiConnection.TryRequest(request, out OrganizationResponseList response); + return response.List; + } + + /// + /// Fetch organization data from web API + /// + /// + /// + public Organization ReadOrganization(string organizationId = null) + { + var request = new RestRequest("organizations/read", Method.POST) + { + RequestFormat = DataFormat.Json + }; + + if (organizationId != null) + { + request.AddParameter("application/json", JsonConvert.SerializeObject(new { organizationId }), ParameterType.RequestBody); + } + + ApiConnection.TryRequest(request, out OrganizationResponse response); + return response.Organization; + } + + /// + /// Helper method to normalize path for api data requests + /// + /// Filepath to format + /// Normalized path + public string FormatPathForDataRequest(string filePath) + { + // Normalize windows paths to linux format + filePath = filePath.Replace("\\", "/", StringComparison.InvariantCulture); + + // Remove data root directory from path for request if included + if (filePath.StartsWith(_dataFolder, StringComparison.InvariantCulture)) + { + filePath = filePath.Substring(_dataFolder.Length); + } + + return filePath; + } } -} \ No newline at end of file +} diff --git a/Common/Api/Data.cs b/Common/Api/Data.cs new file mode 100644 index 000000000000..5e867dedea31 --- /dev/null +++ b/Common/Api/Data.cs @@ -0,0 +1,129 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using Newtonsoft.Json; + +// Collection of response objects for Quantconnect Data/ endpoints +namespace QuantConnect.Api +{ + /// + /// Data/Read response wrapper, contains link to requested data + /// + public class DataLink : RestResponse + { + /// + /// Url to the data requested + /// + [JsonProperty(PropertyName = "link")] + public string Url { get; set; } + + /// + /// Remaining QCC balance on account after this transaction + /// + [JsonProperty(PropertyName = "balance")] + public double Balance { get; set; } + + /// + /// QCC Cost for this data link + /// + [JsonProperty(PropertyName = "cost")] + public double Cost { get; set; } + } + + /// + /// Data/List response wrapper for available data + /// + public class DataList : RestResponse + { + /// + /// List of all available data from this request + /// + [JsonProperty(PropertyName = "objects")] + public List AvailableData { get; set; } + } + + /// + /// Data/Prices response wrapper for prices by vendor + /// + public class DataPricesList : RestResponse + { + /// + /// Collection of prices objects + /// + [JsonProperty(PropertyName = "prices")] + public List Prices { get; set; } + + /// + /// The Agreement URL for this Organization + /// + [JsonProperty(PropertyName = "agreement")] + public string AgreementUrl { get; set; } + + /// + /// Get the price in QCC for a given data file + /// + /// Lean data path of the file + /// QCC price for data, -1 if no entry found + public int GetPrice(string path) + { + if (path == null) + { + return -1; + } + + var entry = Prices.FirstOrDefault(x => Regex.IsMatch(path, x.RegEx)); + return entry == null ? -1 : entry.Price; + } + } + + /// + /// Prices entry for Data/Prices response + /// + public class PriceEntry + { + /// + /// Vendor for this price + /// + [JsonProperty(PropertyName = "vendorName")] + public string Vendor { get; set; } + + /// + /// Regex for this data price entry + /// Trims regex open, close, and multiline flag + /// because it won't match otherwise + /// + public string RegEx + { + get => RawRegEx.TrimStart('/').TrimEnd('m').TrimEnd('/'); + set => RawRegEx = value; + } + + /// + /// RegEx directly from response + /// + [JsonProperty(PropertyName = "regex")] + public string RawRegEx { get; set; } + + /// + /// The price for this entry in QCC + /// + [JsonProperty(PropertyName = "price")] + public int Price { get; set; } + } +} diff --git a/Common/Api/Link.cs b/Common/Api/Link.cs deleted file mode 100644 index 13817fc06880..000000000000 --- a/Common/Api/Link.cs +++ /dev/null @@ -1,31 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using Newtonsoft.Json; - -namespace QuantConnect.Api -{ - /// - /// Response from reading purchased data - /// - public class Link : RestResponse - { - /// - /// Link to the data - /// - [JsonProperty(PropertyName = "link")] - public string DataLink { get; set; } - } -} diff --git a/Common/Api/Organization.cs b/Common/Api/Organization.cs new file mode 100644 index 000000000000..4800be852201 --- /dev/null +++ b/Common/Api/Organization.cs @@ -0,0 +1,242 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; +using QuantConnect.Api.Serialization; + +// Collection of response objects for QuantConnect Organization/ endpoints +namespace QuantConnect.Api +{ + /// + /// Response wrapper for Organizations/List + /// TODO: The response objects in the array do not contain all Organization Properties; do we need another wrapper object? + /// + public class OrganizationResponseList : RestResponse + { + /// + /// List of organizations in the response + /// + [JsonProperty(PropertyName = "organizations")] + public List List { get; set; } + } + + /// + /// Response wrapper for Organizations/Read + /// + public class OrganizationResponse : RestResponse + { + /// + /// Organization read from the response + /// + [JsonProperty(PropertyName = "organization")] + public Organization Organization { get; set; } + } + + /// + /// Object representation of Organization from QuantConnect Api + /// + public class Organization + { + /// + /// Organization ID; Used for API Calls + /// + [JsonProperty(PropertyName = "id")] + public string Id { get; set; } + + /// + /// Seats in Organization + /// + [JsonProperty(PropertyName = "seats")] + public int Seats { get; set; } + + /// + /// Data Agreement information + /// + [JsonProperty(PropertyName = "data")] + public DataAgreement DataAgreement { get; set; } + + /// + /// Organization Product Subscriptions + /// + [JsonProperty(PropertyName = "products")] + public List Products { get; set; } + + /// + /// Organization Credit Balance and Transactions + /// + [JsonProperty(PropertyName = "credit")] + public Credit Credit { get; set; } + } + + /// + /// Organization Data Agreement + /// + public class DataAgreement + { + /// + /// Epoch time the Data Agreement was Signed + /// + [JsonProperty(PropertyName = "signedTime")] + public long? EpochSignedTime { get; set; } + + /// + /// DateTime the agreement was signed. + /// Uses EpochSignedTime converted to a standard datetime. + /// + public DateTime? SignedTime => EpochSignedTime.HasValue ? DateTimeOffset.FromUnixTimeSeconds(EpochSignedTime.Value).DateTime : null; + + /// + /// True/False if it is currently signed + /// + [JsonProperty(PropertyName = "current")] + public bool Signed { get; set; } + } + + /// + /// Organization Credit Object + /// + public class Credit + { + /// + /// Represents a change in organization credit + /// + public class Movement + { + /// + /// Date of the change in credit + /// + [JsonProperty(PropertyName = "date")] + public DateTime Date { get; set; } + + /// + /// Credit description + /// + [JsonProperty(PropertyName = "description")] + public string Description { get; set; } + + /// + /// Amount of change + /// + [JsonProperty(PropertyName = "amount")] + public decimal Amount { get; set; } + + /// + /// Ending Balance in QCC after Movement + /// + [JsonProperty(PropertyName = "balance")] + public decimal Balance { get; set; } + } + + /// + /// QCC Current Balance + /// + [JsonProperty(PropertyName = "balance")] + public decimal Balance { get; set; } + + /// + /// List of changes to Credit + /// + [JsonProperty(PropertyName = "movements")] + public List Movements { get; set; } + } + + /// + /// QuantConnect Products + /// + [JsonConverter(typeof(ProductJsonConverter))] + public class Product + { + /// + /// Product Type + /// + public ProductType Type { get; set; } + + /// + /// Collection of item subscriptions + /// Nodes/Data/Seats/etc + /// + public List Items { get; set; } + } + + /// + /// QuantConnect ProductItem + /// + public class ProductItem + { + /// + /// Product Type + /// + [JsonProperty(PropertyName = "name")] + public string Name { get; set; } + + /// + /// Collection of item subscriptions + /// Nodes/Data/Seats/etc + /// + [JsonProperty(PropertyName = "quantity")] + public int Quantity { get; set; } + + /// + /// USD Unit price for this item + /// + [JsonProperty(PropertyName = "unitPrice")] + public int UnitPrice { get; set; } + + /// + /// USD Total price for this product + /// + [JsonProperty(PropertyName = "total")] + public int TotalPrice { get; set; } + } + + /// + /// Product types offered by QuantConnect + /// Used by Product class + /// + public enum ProductType + { + /// + /// Professional Seats Subscriptions + /// + ProfessionalSeats, + + /// + /// Backtest Nodes Subscriptions + /// + BacktestNode, + + /// + /// Research Nodes Subscriptions + /// + ResearchNode, + + /// + /// Live Trading Nodes Subscriptions + /// + LiveNode, + + /// + /// Support Subscriptions + /// + Support, + + /// + /// Data Subscriptions + /// + Data + } +} diff --git a/Common/Api/Prices.cs b/Common/Api/Prices.cs deleted file mode 100644 index 2707fad191a5..000000000000 --- a/Common/Api/Prices.cs +++ /dev/null @@ -1,63 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using System.Collections.Generic; -using Newtonsoft.Json; -using QuantConnect.Util; - -namespace QuantConnect.Api -{ - /// - /// Prices rest response wrapper - /// - public class Prices - { - /// - /// The requested Symbol - /// - public Symbol Symbol { get; set; } - - /// - /// The requested symbol ID - /// - [JsonProperty(PropertyName = "symbol")] - public string SymbolID { get; set; } - - /// - /// The requested price - /// - [JsonProperty(PropertyName = "price")] - public decimal Price { get; set; } - - /// - /// UTC time the price was updated - /// - [JsonProperty(PropertyName = "updated"), JsonConverter(typeof(DoubleUnixSecondsDateTimeJsonConverter))] - public DateTime Updated; - } - - /// - /// Collection container for a list of prices objects - /// - public class PricesList : RestResponse - { - /// - /// Collection of prices objects - /// - [JsonProperty(PropertyName = "prices")] - public List Prices; - } -} diff --git a/Common/Api/Serialization/ProductJsonConverter.cs b/Common/Api/Serialization/ProductJsonConverter.cs new file mode 100644 index 000000000000..0166a8b24a2b --- /dev/null +++ b/Common/Api/Serialization/ProductJsonConverter.cs @@ -0,0 +1,102 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace QuantConnect.Api.Serialization +{ + /// + /// Provides an implementation of that can deserialize + /// + public class ProductJsonConverter : JsonConverter + { + private Dictionary _productTypeMap = new Dictionary() + { + {"Professional Seats", ProductType.ProfessionalSeats}, + {"Backtest Node", ProductType.BacktestNode}, + {"Research Node", ProductType.ResearchNode}, + {"Live Trading Node", ProductType.LiveNode}, + {"Support", ProductType.Support}, + {"Data", ProductType.Data}, + }; + + /// + /// Gets a value indicating whether this can write JSON. + /// + /// + /// true if this can write JSON; otherwise, false. + /// + public override bool CanWrite => false; + + /// + /// Determines whether this instance can convert the specified object type. + /// + /// Type of the object. + /// + /// true if this instance can convert the specified object type; otherwise, false. + /// + public override bool CanConvert(Type objectType) + { + return objectType == typeof(Product); + } + + /// + /// Writes the JSON representation of the object. + /// + /// The to write to.The value.The calling serializer. + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + throw new NotImplementedException("The OrderJsonConverter does not implement a WriteJson method;."); + } + + /// + /// Reads the JSON representation of the object. + /// + /// The to read from.Type of the object.The existing value of object being read.The calling serializer. + /// + /// The object value. + /// + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + var jObject = JObject.Load(reader); + + var result = CreateProductFromJObject(jObject); + + return result; + } + + /// + /// Create an order from a simple JObject + /// + /// + /// Order Object + public Product CreateProductFromJObject(JObject jObject) + { + if (jObject == null) + { + return null; + } + + return new Product + { + Type = _productTypeMap[jObject["name"].Value()], + Items = jObject["items"].ToObject>() + }; + } + } +} diff --git a/Common/Data/Auxiliary/FactorFile.cs b/Common/Data/Auxiliary/FactorFile.cs index 3bcc0d38c8cc..895c55d3bc78 100644 --- a/Common/Data/Auxiliary/FactorFile.cs +++ b/Common/Data/Auxiliary/FactorFile.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -97,10 +97,9 @@ public FactorFile(string permtick, IEnumerable data, DateTime? fa /// /// Reads a FactorFile in from the . /// - public static FactorFile Read(string permtick, string market) + public static FactorFile Read(string permtick, Stream file) { - DateTime? factorFileMinimumDate; - return new FactorFile(permtick, FactorFileRow.Read(permtick, market, out factorFileMinimumDate), factorFileMinimumDate); + return new FactorFile(permtick, FactorFileRow.Read(file, out DateTime? factorFileMinimumDate), factorFileMinimumDate); } /// @@ -389,4 +388,4 @@ IEnumerator IEnumerable.GetEnumerator() return GetEnumerator(); } } -} \ No newline at end of file +} diff --git a/Common/Data/Auxiliary/FactorFileRow.cs b/Common/Data/Auxiliary/FactorFileRow.cs index 4e054729bbb5..eed6d5840e9d 100644 --- a/Common/Data/Auxiliary/FactorFileRow.cs +++ b/Common/Data/Auxiliary/FactorFileRow.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -19,6 +19,7 @@ using System.Globalization; using System.IO; using System.Linq; +using System.Text; using QuantConnect.Data.Market; using QuantConnect.Securities; using static QuantConnect.StringExtensions; @@ -95,12 +96,24 @@ public FactorFileRow(DateTime date, decimal priceFactor, decimal splitFactor, de /// /// Reads in the factor file for the specified equity symbol /// - public static IEnumerable Read(string permtick, string market, out DateTime? factorFileMinimumDate) + public static IEnumerable Read(Stream file, out DateTime? factorFileMinimumDate) { factorFileMinimumDate = null; - var path = Path.Combine(Globals.CacheDataFolder, "equity", market, "factor_files", permtick.ToLowerInvariant() + ".csv"); - var lines = File.ReadAllLines(path).Where(l => !string.IsNullOrWhiteSpace(l)); + var streamReader = new StreamReader(file, Encoding.UTF8); + + string line; + var lines = new List(); + while (!streamReader.EndOfStream) + { + line = streamReader.ReadLine(); + if (!string.IsNullOrWhiteSpace(line)) + { + lines.Add(line); + } + } + + streamReader.Dispose(); return Parse(lines, out factorFileMinimumDate); } @@ -329,4 +342,4 @@ private void UpdatePriceScaleFactor() PriceScaleFactor = _priceFactor * _splitFactor; } } -} \ No newline at end of file +} diff --git a/Common/Data/Auxiliary/FactorFileZipHelper.cs b/Common/Data/Auxiliary/FactorFileZipHelper.cs new file mode 100644 index 000000000000..64e5f36baae7 --- /dev/null +++ b/Common/Data/Auxiliary/FactorFileZipHelper.cs @@ -0,0 +1,72 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.IO; +using System.Linq; +using System.Collections.Generic; + +namespace QuantConnect.Data.Auxiliary +{ + /// + /// Provides methods for reading factor file zips + /// + public static class FactorFileZipHelper + { + /// + /// Reads the zip bytes as text and parses as FactorFileRows to create FactorFiles + /// + public static IEnumerable> ReadFactorFileZip(Stream file, MapFileResolver mapFileResolver, string market) + { + if (file == null || file.Length == 0) + { + return new Dictionary(); + } + + var keyValuePairs = ( + from kvp in Compression.Unzip(file) + let filename = kvp.Key + let lines = kvp.Value + let factorFile = SafeRead(filename, lines) + let mapFile = mapFileResolver.GetByPermtick(factorFile.Permtick) + where mapFile != null + let sid = SecurityIdentifier.GenerateEquity(mapFile.FirstDate, mapFile.FirstTicker, market) + let symbol = new Symbol(sid, mapFile.Permtick) + select new KeyValuePair(symbol, factorFile) + ); + + return keyValuePairs; + } + + /// + /// Parses the contents as a FactorFile, if error returns a new empty factor file + /// + public static FactorFile SafeRead(string filename, IEnumerable contents) + { + var permtick = Path.GetFileNameWithoutExtension(filename); + try + { + DateTime? minimumDate; + // FactorFileRow.Parse handles entries with 'inf' and exponential notation and provides the associated minimum tradeable date for these cases + // previously these cases were not handled causing an exception and returning an empty factor file + return new FactorFile(permtick, FactorFileRow.Parse(contents, out minimumDate), minimumDate); + } + catch + { + return new FactorFile(permtick, Enumerable.Empty()); + } + } + } +} diff --git a/Common/Data/Auxiliary/LocalDiskFactorFileProvider.cs b/Common/Data/Auxiliary/LocalDiskFactorFileProvider.cs index 991c6ee96a62..c82b9a3566a1 100644 --- a/Common/Data/Auxiliary/LocalDiskFactorFileProvider.cs +++ b/Common/Data/Auxiliary/LocalDiskFactorFileProvider.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -15,7 +15,7 @@ */ using System.Collections.Concurrent; -using QuantConnect.Configuration; +using System.IO; using QuantConnect.Interfaces; using QuantConnect.Util; @@ -26,27 +26,28 @@ namespace QuantConnect.Data.Auxiliary /// public class LocalDiskFactorFileProvider : IFactorFileProvider { - private readonly IMapFileProvider _mapFileProvider; + private IMapFileProvider _mapFileProvider; + private IDataProvider _dataProvider; private readonly ConcurrentDictionary _cache; /// - /// Initializes a new instance of that uses configuration - /// to resolve an instance of from the + /// Creates a new instance of the /// public LocalDiskFactorFileProvider() - : this(Composer.Instance.GetExportedValueByTypeName(Config.Get("map-file-provider", "LocalDiskMapFileProvider"))) { + _cache = new ConcurrentDictionary(); } /// - /// Initializes a new instance of the using the specified - /// map file provider + /// Initializes our FactorFileProvider by supplying our mapFileProvider + /// and dataProvider /// - /// The map file provider used to resolve permticks of securities - public LocalDiskFactorFileProvider(IMapFileProvider mapFileProvider) + /// MapFileProvider to use + /// DataProvider to use + public void Initialize(IMapFileProvider mapFileProvider, IDataProvider dataProvider) { _mapFileProvider = mapFileProvider; - _cache = new ConcurrentDictionary(); + _dataProvider = dataProvider; } /// @@ -86,9 +87,14 @@ public FactorFile Get(Symbol symbol) private FactorFile GetFactorFile(Symbol symbol, string permtick, string market) { FactorFile factorFile = null; - if (FactorFile.HasScalingFactors(permtick, market)) + + var path = Path.Combine(Globals.DataFolder, "equity", market, "factor_files", permtick.ToLowerInvariant() + ".csv"); + + var factorFileStream = _dataProvider.Fetch(path); + if (factorFileStream != null) { - factorFile = FactorFile.Read(permtick, market); + factorFile = FactorFile.Read(permtick, factorFileStream); + factorFileStream.DisposeSafely(); _cache.AddOrUpdate(symbol, factorFile, (s, c) => factorFile); } else diff --git a/Common/Data/Auxiliary/LocalDiskMapFileProvider.cs b/Common/Data/Auxiliary/LocalDiskMapFileProvider.cs index 074f964bf6d0..1424d868e403 100644 --- a/Common/Data/Auxiliary/LocalDiskMapFileProvider.cs +++ b/Common/Data/Auxiliary/LocalDiskMapFileProvider.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -29,7 +29,25 @@ namespace QuantConnect.Data.Auxiliary public class LocalDiskMapFileProvider : IMapFileProvider { private static int _wroteTraceStatement; - private readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); + private readonly ConcurrentDictionary _cache; + private IDataProvider _dataProvider; + + /// + /// Creates a new instance of the + /// + public LocalDiskMapFileProvider() + { + _cache = new ConcurrentDictionary(); + } + + /// + /// Initializes our MapFileProvider by supplying our dataProvider + /// + /// DataProvider to use + public void Initialize(IDataProvider dataProvider) + { + _dataProvider = dataProvider; + } /// /// Gets a representing all the map @@ -39,6 +57,9 @@ public class LocalDiskMapFileProvider : IMapFileProvider /// A containing all map files for the specified market public MapFileResolver Get(string market) { + // TODO: Consider using DataProvider to load in the files from disk to unify data fetching behavior + // Reference LocalDiskFactorFile, LocalZipFactorFile, and LocalZipMapFile providers for examples. + market = market.ToLowerInvariant(); return _cache.GetOrAdd(market, GetMapFileResolver); } diff --git a/Common/Data/Auxiliary/LocalZipFactorFileProvider.cs b/Common/Data/Auxiliary/LocalZipFactorFileProvider.cs new file mode 100644 index 000000000000..0478c9031e81 --- /dev/null +++ b/Common/Data/Auxiliary/LocalZipFactorFileProvider.cs @@ -0,0 +1,158 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.IO; +using QuantConnect.Util; +using QuantConnect.Logging; +using System.Threading.Tasks; +using QuantConnect.Interfaces; +using System.Collections.Generic; + +namespace QuantConnect.Data.Auxiliary +{ + /// + /// Provides an implementation of that searches the local disk for a zip file containing all factor files + /// + public class LocalZipFactorFileProvider : IFactorFileProvider + { + private readonly object _lock; + private IDataProvider _dataProvider; + private IMapFileProvider _mapFileProvider; + private Dictionary _seededMarket; + private readonly Dictionary _factorFiles; + + /// + /// The cached refresh period for the factor files + /// + /// Exposed for testing + protected virtual TimeSpan CacheRefreshPeriod => TimeSpan.FromDays(1); + + /// + /// Creates a new instance of the class. + /// + public LocalZipFactorFileProvider() + { + _factorFiles = new Dictionary(); + _seededMarket = new Dictionary(); + _lock = new object(); + } + + /// + /// Initializes our FactorFileProvider by supplying our mapFileProvider + /// and dataProvider + /// + /// MapFileProvider to use + /// DataProvider to use + public void Initialize(IMapFileProvider mapFileProvider, IDataProvider dataProvider) + { + _mapFileProvider = mapFileProvider; + _dataProvider = dataProvider; + StartExpirationTask(); + } + + /// + /// Gets a instance for the specified symbol, or null if not found + /// + /// The security's symbol whose factor file we seek + /// The resolved factor file, or null if not found + public FactorFile Get(Symbol symbol) + { + var market = symbol.ID.Market.ToLowerInvariant(); + lock (_lock) + { + if (!_seededMarket.ContainsKey(market)) + { + HydrateFactorFileFromLatestZip(market); + _seededMarket[market] = true; + } + + FactorFile factorFile; + if (_factorFiles.TryGetValue(symbol, out factorFile)) + { + return factorFile; + } + } + + // Could not find factor file for symbol + Log.Error($"LocalZipFactorFileProvider.Get({symbol}): No factor file found."); + return null; + } + + /// + /// Helper method that will clear any cached factor files in a daily basis, this is useful for live trading + /// + protected virtual void StartExpirationTask() + { + lock (_lock) + { + // we clear the seeded markets so they are reloaded + _seededMarket = new Dictionary(); + } + _ = Task.Delay(CacheRefreshPeriod).ContinueWith(_ => StartExpirationTask()); + } + + /// Hydrate the from the latest zipped factor file on disk + private void HydrateFactorFileFromLatestZip(string market) + { + if (market != QuantConnect.Market.USA.ToLowerInvariant()) + { + // don't explode for other markets which request factor files and we don't have + return; + } + // start the search with yesterday, today's file will be available tomorrow + var todayNewYork = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date; + var date = todayNewYork.AddDays(-1); + + var count = 0; + + do + { + var zipFileName = $"equity/{market}/factor_files/factor_files_{date:yyyyMMdd}.zip"; + var factorFilePath = Path.Combine(Globals.DataFolder, zipFileName); + + // Fetch a stream for our zip from our data provider + var stream = _dataProvider.Fetch(factorFilePath); + + // If the file was found we can read the file + if (stream != null) + { + var mapFileResolver = _mapFileProvider.Get(market); + foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market)) + { + // we merge with existing, this will allow to hold multiple markets + _factorFiles[keyValuePair.Key] = keyValuePair.Value; + } + stream.DisposeSafely(); + Log.Trace($"LocalZipFactorFileProvider.Get({market}): Fetched factor files for: {date.ToShortDateString()} NY"); + + return; + } + + // Otherwise we will search back another day + Log.Debug($"LocalZipFactorFileProvider.Get(): No factor file found for date {date.ToShortDateString()}"); + + // prevent infinite recursion if something is wrong + if (count++ > 7) + { + throw new InvalidOperationException($"LocalZipFactorFileProvider.Get(): Could not find any factor files going all the way back to {date}"); + } + + date = date.AddDays(-1); + } + while (true); + } + } +} diff --git a/Common/Data/Auxiliary/LocalZipMapFileProvider.cs b/Common/Data/Auxiliary/LocalZipMapFileProvider.cs new file mode 100644 index 000000000000..fee6c6f362b2 --- /dev/null +++ b/Common/Data/Auxiliary/LocalZipMapFileProvider.cs @@ -0,0 +1,135 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.IO; +using QuantConnect.Util; +using QuantConnect.Logging; +using System.Threading.Tasks; +using QuantConnect.Interfaces; +using System.Collections.Generic; + +namespace QuantConnect.Data.Auxiliary +{ + /// + /// Provides an implementation of that reads from a local zip file + /// + public class LocalZipMapFileProvider : IMapFileProvider + { + private Dictionary _cache; + private IDataProvider _dataProvider; + private object _lock; + + /// + /// The cached refresh period for the map files + /// + /// Exposed for testing + protected virtual TimeSpan CacheRefreshPeriod => TimeSpan.FromDays(1); + + /// + /// Creates a new instance of the + /// + public LocalZipMapFileProvider() + { + _lock = new object(); + _cache = new Dictionary(); + } + + /// + /// Initializes our MapFileProvider by supplying our dataProvider + /// + /// DataProvider to use + public void Initialize(IDataProvider dataProvider) + { + _dataProvider = dataProvider; + StartExpirationTask(); + } + + /// + /// Gets a representing all the map files for the specified market + /// + /// The equity market, for example, 'usa' + /// A containing all map files for the specified market + public MapFileResolver Get(string market) + { + market = market.ToLowerInvariant(); + MapFileResolver result; + // we use a lock so that only 1 thread loads the map file resolver while the rest wait + // else we could have multiple threads loading the map file resolver at the same time! + lock (_lock) + { + if (!_cache.TryGetValue(market, out result)) + { + _cache[market] = result = GetMapFileResolver(market); + } + } + return result; + } + + /// + /// Helper method that will clear any cached factor files in a daily basis, this is useful for live trading + /// + protected virtual void StartExpirationTask() + { + lock (_lock) + { + // we clear the seeded markets so they are reloaded + _cache = new Dictionary(); + } + _ = Task.Delay(CacheRefreshPeriod).ContinueWith(_ => StartExpirationTask()); + } + + private MapFileResolver GetMapFileResolver(string market) + { + if (market != QuantConnect.Market.USA.ToLowerInvariant()) + { + // don't explode for other markets which request map files and we don't have + return MapFileResolver.Empty; + } + var timestamp = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork); + var todayNewYork = timestamp.Date; + var yesterdayNewYork = todayNewYork.AddDays(-1); + + // start the search with yesterday, today's file will be available tomorrow + var count = 0; + var date = yesterdayNewYork; + do + { + var zipFileName = Path.Combine(Globals.DataFolder, MapFileZipHelper.GetMapFileZipFileName(market, date)); + + // Fetch a stream for our zip from our data provider + var stream = _dataProvider.Fetch(zipFileName); + + // If we found a file we can read it + if (stream != null) + { + Log.Trace("LocalZipMapFileProvider.Get({0}): Fetched map files for: {1} NY", market, date.ToShortDateString()); + var result = new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream)); + stream.DisposeSafely(); + return result; + } + + // prevent infinite recursion if something is wrong + if (count++ > 30) + { + throw new InvalidOperationException($"LocalZipMapFileProvider couldn't find any map files going all the way back to {date}"); + } + + date = date.AddDays(-1); + } + while (true); + } + } +} diff --git a/Common/Data/Auxiliary/MapFileZipHelper.cs b/Common/Data/Auxiliary/MapFileZipHelper.cs new file mode 100644 index 000000000000..fe2599c0f344 --- /dev/null +++ b/Common/Data/Auxiliary/MapFileZipHelper.cs @@ -0,0 +1,71 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.IO; +using System.Linq; +using System.Collections.Generic; + +namespace QuantConnect.Data.Auxiliary +{ + /// + /// Helper class for handling mapfile zip files + /// + public static class MapFileZipHelper + { + /// + /// Gets the mapfile zip filename for the specified date + /// + public static string GetMapFileZipFileName(string market, DateTime date) + { + return $"equity/{market}/map_files/map_files_{date:yyyyMMdd}.zip"; + } + + /// + /// Reads the zip bytes as text and parses as MapFileRows to create MapFiles + /// + public static IEnumerable ReadMapFileZip(Stream file) + { + if (file == null || file.Length == 0) + { + return Enumerable.Empty(); + } + + var result = from kvp in Compression.Unzip(file) + let filename = kvp.Key + where filename.EndsWith(".csv", StringComparison.InvariantCultureIgnoreCase) + let lines = kvp.Value.Where(line => !string.IsNullOrEmpty(line)) + let mapFile = SafeRead(filename, lines) + select mapFile; + return result; + } + + /// + /// Parses the contents as a MapFile, if error returns a new empty map file + /// + private static MapFile SafeRead(string filename, IEnumerable contents) + { + var permtick = Path.GetFileNameWithoutExtension(filename); + try + { + return new MapFile(permtick, contents.Select(MapFileRow.Parse)); + } + catch + { + return new MapFile(permtick, Enumerable.Empty()); + } + } + } +} diff --git a/Common/Interfaces/IApi.cs b/Common/Interfaces/IApi.cs index 39052901aae0..1bb71281857e 100644 --- a/Common/Interfaces/IApi.cs +++ b/Common/Interfaces/IApi.cs @@ -37,8 +37,9 @@ public interface IApi : IDisposable /// /// Project name /// Programming language to use + /// Organization to create this project under /// that includes information about the newly created project - ProjectResponse CreateProject(string name, Language language); + ProjectResponse CreateProject(string name, Language language, string organizationId = null); /// /// Read in a project from the QuantConnect.com API. @@ -181,20 +182,17 @@ public interface IApi : IDisposable /// /// Gets the link to the downloadable data. /// - /// Symbol of security of which data will be requested. - /// Resolution of data requested. - /// Date of the data requested. + /// File path representing the data requested + /// Organization to purchase this data with /// Link to the downloadable data. - Link ReadDataLink(Symbol symbol, Resolution resolution, DateTime date); + DataLink ReadDataLink(string filePath, string organizationId); /// /// Method to download and save the data purchased through QuantConnect /// - /// Symbol of security of which data will be requested. - /// Resolution of data requested. - /// Date of the data requested. + /// File path representing the data requested /// A bool indicating whether the data was successfully downloaded or not. - bool DownloadData(Symbol symbol, Resolution resolution, DateTime date); + bool DownloadData(string filePath, string organizationId); /// /// Create a new live algorithm for a logged in user. @@ -238,8 +236,6 @@ public interface IApi : IDisposable /// RestResponse StopLiveAlgorithm(int projectId); - - //Status StatusRead(int projectId, string algorithmId); //RestResponse StatusUpdate(int projectId, string algorithmId, AlgorithmStatus status, string message = ""); //LogControl LogAllowanceRead(); diff --git a/Common/Interfaces/IFactorFileProvider.cs b/Common/Interfaces/IFactorFileProvider.cs index 8ad884efda0e..b10b542949ee 100644 --- a/Common/Interfaces/IFactorFileProvider.cs +++ b/Common/Interfaces/IFactorFileProvider.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -25,6 +25,14 @@ namespace QuantConnect.Interfaces [InheritedExport(typeof(IFactorFileProvider))] public interface IFactorFileProvider { + /// + /// Initializes our FactorFileProvider by supplying our mapFileProvider + /// and dataProvider + /// + /// MapFileProvider to use + /// DataProvider to use + void Initialize(IMapFileProvider mapFileProvider, IDataProvider dataProvider); + /// /// Gets a instance for the specified symbol, or null if not found /// @@ -32,4 +40,4 @@ public interface IFactorFileProvider /// The resolved factor file, or null if not found FactorFile Get(Symbol symbol); } -} \ No newline at end of file +} diff --git a/Common/Interfaces/IMapFileProvider.cs b/Common/Interfaces/IMapFileProvider.cs index fc506b8e17d3..d1b7a4a072f1 100644 --- a/Common/Interfaces/IMapFileProvider.cs +++ b/Common/Interfaces/IMapFileProvider.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -25,6 +25,12 @@ namespace QuantConnect.Interfaces [InheritedExport(typeof(IMapFileProvider))] public interface IMapFileProvider { + /// + /// Initializes our MapFileProvider by supplying our dataProvider + /// + /// DataProvider to use + void Initialize(IDataProvider dataProvider); + /// /// Gets a representing all the map /// files for the specified market diff --git a/Common/QuantConnect.csproj b/Common/QuantConnect.csproj index 5e660b26b870..39cf3690d52d 100644 --- a/Common/QuantConnect.csproj +++ b/Common/QuantConnect.csproj @@ -71,6 +71,7 @@ + diff --git a/Compression/Compression.cs b/Compression/Compression.cs index 1b6cc89072b8..31ae3f64223f 100644 --- a/Compression/Compression.cs +++ b/Compression/Compression.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -37,6 +37,18 @@ namespace QuantConnect /// QuantConnect's data library is stored in zip format locally on the hard drive. public static class Compression { + /// + /// Global Flag :: Operating System + /// + private static bool IsLinux + { + get + { + var p = (int)Environment.OSVersion.Platform; + return (p == 4) || (p == 6) || (p == 128); + } + } + /// /// Create a zip file of the supplied file names and string data source /// @@ -214,7 +226,8 @@ public static Dictionary UnzipData(byte[] zipData, Encoding enco zipStream.Read(buffer, 0, (int)entry.Size); //Save into array: - data.Add(entry.Name, buffer.GetString(encoding)); + var str = (encoding ?? Encoding.ASCII).GetString(buffer); + data.Add(entry.Name, str); } else { @@ -426,7 +439,7 @@ public static bool Unzip(string zip, string directory, bool overwrite = false) // skip directories if (file.Name == "") continue; var filepath = Path.Combine(directory, file.FullName); - if (OS.IsLinux) filepath = filepath.Replace(@"\", "/"); + if (IsLinux) filepath = filepath.Replace(@"\", "/"); var outputFile = new FileInfo(filepath); if (!outputFile.Directory.Exists) { @@ -875,10 +888,20 @@ public static IEnumerable GetZipEntryFileNames(string zipFileName) { using (var zip = ZipFile.Read(zipFileName)) { - foreach (var entry in zip) - { - yield return entry.FileName; - } + return zip.EntryFileNames; + } + } + + /// + /// Return the entry file names contained in a zip file + /// + /// Stream to the file + /// IEnumerable of entry file names + public static IEnumerable GetZipEntryFileNames(Stream zipFileStream) + { + using (var zip = ZipFile.Read(zipFileStream)) + { + return zip.EntryFileNames; } } @@ -891,7 +914,7 @@ public static IEnumerable GetZipEntryFileNames(string zipFileName) /// The extraction failed because of a timeout or the exit code was not 0 public static void Extract7ZipArchive(string inputFile, string outputDirectory, int execTimeout = 60000) { - var zipper = OS.IsWindows ? "C:/Program Files/7-Zip/7z.exe" : "7z"; + var zipper = IsLinux ? "7z" : "C:/Program Files/7-Zip/7z.exe"; var psi = new ProcessStartInfo(zipper, " e " + inputFile + " -o" + outputDirectory) { CreateNoWindow = true, diff --git a/Compression/QuantConnect.Compression.csproj b/Compression/QuantConnect.Compression.csproj index baf268cb9360..111891d635f2 100644 --- a/Compression/QuantConnect.Compression.csproj +++ b/Compression/QuantConnect.Compression.csproj @@ -41,7 +41,6 @@ - diff --git a/Engine/DataFeeds/ApiDataProvider.cs b/Engine/DataFeeds/ApiDataProvider.cs index 1c5fa7e08e72..e2061c1e8869 100644 --- a/Engine/DataFeeds/ApiDataProvider.cs +++ b/Engine/DataFeeds/ApiDataProvider.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -16,6 +16,9 @@ using System; using System.IO; +using System.Linq; +using System.Threading; +using QuantConnect.Api; using QuantConnect.Configuration; using QuantConnect.Interfaces; using QuantConnect.Logging; @@ -24,14 +27,26 @@ namespace QuantConnect.Lean.Engine.DataFeeds { /// - /// An instance of the that will attempt to retrieve files not present on the filesystem from the API + /// An instance of the that will download and update data files as needed via QC's Api. /// - public class ApiDataProvider : IDataProvider + public class ApiDataProvider : DefaultDataProvider { private readonly int _uid = Config.GetInt("job-user-id", 0); private readonly string _token = Config.Get("api-access-token", "1"); + private readonly string _organizationId = Config.Get("job-organization-id"); private readonly string _dataPath = Config.Get("data-folder", "../../../Data/"); - private static readonly int DownloadPeriod = Config.GetInt("api-data-update-period", 5); + private readonly bool _subscribedToEquityMapAndFactorFiles; + private readonly DataPricesList _dataPrices; + + /// + /// Data Purchase limit measured in QCC (QuantConnect Credits) + /// + private decimal _purchaseLimit = Config.GetValue("data-purchase-limit", decimal.MaxValue); + + /// + /// Period to re-download + /// + private static readonly int DownloadPeriod = Config.GetInt("api-data-update-period", 1); private readonly Api.Api _api; /// @@ -40,36 +55,92 @@ public class ApiDataProvider : IDataProvider public ApiDataProvider() { _api = new Api.Api(); - _api.Initialize(_uid, _token, _dataPath); + + // If we have no value for organization get account preferred + if (string.IsNullOrEmpty(_organizationId)) + { + var account = _api.ReadAccount(); + _organizationId = account.OrganizationId; + Log.Trace($"ApiDataProvider(): Will use organization Id '{_organizationId}'."); + } + + // Read in data prices and organization details + _dataPrices = _api.ReadDataPrices(_organizationId); + var organization = _api.ReadOrganization(_organizationId); + + // Determine if the user is subscribed to map and factor files + if (organization.Products.Where(x => x.Type == ProductType.Data).Any(x => x.Items.Any(x => x.Name.Contains("Factor", StringComparison.InvariantCultureIgnoreCase)))) + { + _subscribedToEquityMapAndFactorFiles = true; + } + + // Verify user has agreed to data provider agreements + if (organization.DataAgreement.Signed) + { + //Log Agreement Highlights + Log.Trace("ApiDataProvider(): Data Terms of Use has been signed. \r\n" + + $" Find full agreement at: {_dataPrices.AgreementUrl} \r\n" + + "==========================================================================\r\n" + + $"CLI API Access Agreement: On {organization.DataAgreement.SignedTime:d} You Agreed:\r\n" + + " - Display or distribution of data obtained through CLI API Access is not permitted. \r\n" + + " - Data and Third Party Data obtained via CLI API Access can only be used for individual or internal employee's use.\r\n" + + " - Data is provided in LEAN format can not be manipulated for transmission or use in other applications. \r\n" + + " - QuantConnect is not liable for the quality of data received and is not responsible for trading losses. \r\n" + + "=========================================================================="); + Thread.Sleep(TimeSpan.FromSeconds(3)); + } + else + { + // Log URL to go accept terms + throw new InvalidOperationException($"ApiDataProvider(): Must agree to terms at {_dataPrices.AgreementUrl}, before using the ApiDataProvider"); + } + + // Verify we have the balance to maintain our purchase limit, if not adjust it to meet our balance + var balance = organization.Credit.Balance; + if (balance < _purchaseLimit) + { + if (_purchaseLimit != decimal.MaxValue) + { + Log.Error("ApiDataProvider(): Purchase limit is greater than balance." + + $" Setting purchase limit to balance : {balance}"); + } + _purchaseLimit = balance; + } } /// /// Retrieves data to be used in an algorithm. /// If file does not exist, an attempt is made to download them from the api /// - /// A string representing where the data is stored + /// File path representing where the data requested /// A of the data requested - public Stream Fetch(string key) + public override Stream Fetch(string filePath) { - Symbol symbol; - DateTime date; - Resolution resolution; - - // Fetch the details of this data request - if (LeanData.TryParsePath(key, out symbol, out date, out resolution)) + // If we don't already have this file or its out of date, download it + if (NeedToDownload(filePath)) { - if (!File.Exists(key) || IsOutOfDate(resolution, key)) + // Verify we have enough credit to handle this + var pricePath = _api.FormatPathForDataRequest(filePath); + var price = _dataPrices.GetPrice(pricePath); + + // No price found + if (price == -1) + { + throw new ArgumentException($"ApiDataProvider.Fetch(): No price found for {pricePath}"); + } + + if (_purchaseLimit < price) { - return DownloadData(key, symbol, date, resolution); + throw new ArgumentException($"ApiDataProvider.Fetch(): Cost {price} for {pricePath} data exceeds remaining purchase limit: {_purchaseLimit}"); } - // Use the file already on the disk - return new FileStream(key, FileMode.Open, FileAccess.Read, FileShare.Read); + // Update our purchase limit and balance. + _purchaseLimit -= price; + return DownloadData(filePath); } - Log.Error("ApiDataProvider.Fetch(): failed to parse key {0}", key); - return null; + return base.Fetch(filePath); } /// @@ -88,35 +159,80 @@ public static bool IsOutOfDate(Resolution resolution, string filepath) /// /// Attempt to download data using the Api for and return a FileStream of that data. /// - /// Filepath to save too - /// - /// - /// + /// The path to store the file /// A FileStream of the data - private FileStream DownloadData(string filepath, Symbol symbol, DateTime date, Resolution resolution) + private FileStream DownloadData(string filePath) { - Log.Trace("ApiDataProvider.Fetch(): Attempting to get data from QuantConnect.com's data library for symbol({0}), resolution({1}) and date({2}).", - symbol.Value, - resolution, - date.Date.ToShortDateString()); + Log.Debug($"ApiDataProvider.Fetch(): Attempting to get data from QuantConnect.com's data library for {filePath}."); - var downloadSuccessful = _api.DownloadData(symbol, resolution, date); + var downloadSuccessful = _api.DownloadData(filePath, _organizationId); if (downloadSuccessful) { - Log.Trace("ApiDataProvider.Fetch(): Successfully retrieved data for symbol({0}), resolution({1}) and date({2}).", - symbol.Value, - resolution, - date.Date.ToShortDateString()); + Log.Trace($"ApiDataProvider.Fetch(): Successfully retrieved data for {filePath}."); - return new FileStream(filepath, FileMode.Open, FileAccess.Read, FileShare.Read); + return new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); } // Failed to download - Log.Error("ApiDataProvider.Fetch(): Unable to remotely retrieve data for path {0}. " + - "Please make sure you have the necessary data in your online QuantConnect data library.", - filepath); + Log.Error($"ApiDataProvider.Fetch(): Unable to remotely retrieve data for path {filePath}."); return null; } + + /// + /// Helper method to determine if this file needs to be downloaded + /// + /// File we are looking at + /// True if should download + private bool NeedToDownload(string filePath) + { + var fileExists = File.Exists(filePath); + + if (CanParsePath(filePath) && LeanData.TryParsePath(filePath, out Symbol symbol, out DateTime date, out Resolution resolution)) + { + var shouldDownload = !fileExists || IsOutOfDate(resolution, filePath); + + // If we need to download this, symbol is an equity, and user is not subscribed to map and factor files throw an error + if (shouldDownload && symbol.ID.SecurityType == SecurityType.Equity) + { + CheckMapFactorFileSubscription(); + } + + return shouldDownload; + } + + // For files that can't be parsed or fail to be + return !fileExists || (DateTime.Now - TimeSpan.FromDays(DownloadPeriod)) > File.GetLastWriteTime(filePath); + } + + /// + /// Helper method to determine if we can parse this path with TryParsePath() + /// Used to limit error throwing by calling TryParsePath() + /// + /// Filepath to check + /// True if can be parsed + private bool CanParsePath(string filepath) + { + // Only not true for these cases + var equitiesAuxData = filepath.Contains("map_files", StringComparison.InvariantCulture) + || filepath.Contains("factor_files", StringComparison.InvariantCulture) + || filepath.Contains("fundamental", StringComparison.InvariantCulture); + + if (equitiesAuxData) + { + CheckMapFactorFileSubscription(); + } + + return !equitiesAuxData; + } + + private void CheckMapFactorFileSubscription() + { + if(!_subscribedToEquityMapAndFactorFiles) + { + throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider" + + "to download Equity data from QuantConnect."); + } + } } } diff --git a/Engine/DataFeeds/BacktestingFutureChainProvider.cs b/Engine/DataFeeds/BacktestingFutureChainProvider.cs index 466a9d3030e8..4bf1eff86d93 100644 --- a/Engine/DataFeeds/BacktestingFutureChainProvider.cs +++ b/Engine/DataFeeds/BacktestingFutureChainProvider.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -15,7 +15,6 @@ using System; using System.Collections.Generic; -using System.IO; using QuantConnect.Interfaces; using QuantConnect.Logging; using QuantConnect.Util; @@ -27,6 +26,17 @@ namespace QuantConnect.Lean.Engine.DataFeeds /// public class BacktestingFutureChainProvider : IFutureChainProvider { + private IDataProvider _dataProvider; + + /// + /// Creates a new instance + /// + /// The data provider instance to use + public BacktestingFutureChainProvider(IDataProvider dataProvider) + { + _dataProvider = dataProvider; + } + /// /// Gets the list of future contracts for a given underlying symbol /// @@ -44,24 +54,29 @@ public IEnumerable GetFutureContractList(Symbol symbol, DateTime date) // build the zip file name for open interest data var zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, symbol, date, Resolution.Minute, TickType.OpenInterest); - if (!File.Exists(zipFileName)) + var stream = _dataProvider.Fetch(zipFileName); + + // If the file isn't found lets give quote a chance - some futures do not have an open interest file + if (stream == null) { - // lets give quote a chance - some futures do not have an open interest file var zipFileNameQuote = LeanData.GenerateZipFilePath(Globals.DataFolder, symbol, date, Resolution.Minute, TickType.Quote); - if (!File.Exists(zipFileNameQuote)) + stream = _dataProvider.Fetch(zipFileNameQuote); + + if (stream == null) { Log.Error($"BacktestingFutureChainProvider.GetFutureContractList(): Failed, files not found: {zipFileName} {zipFileNameQuote}"); yield break; } - zipFileName = zipFileNameQuote; } // generate and return the contract symbol for each zip entry - var zipEntryNames = Compression.GetZipEntryFileNames(zipFileName); + var zipEntryNames = Compression.GetZipEntryFileNames(stream); foreach (var zipEntryName in zipEntryNames) { yield return LeanData.ReadSymbolFromZipEntry(symbol, Resolution.Minute, zipEntryName); } + + stream.DisposeSafely(); } } } diff --git a/Engine/DataFeeds/BacktestingOptionChainProvider.cs b/Engine/DataFeeds/BacktestingOptionChainProvider.cs index c1afa57b617d..1b85e92a3793 100644 --- a/Engine/DataFeeds/BacktestingOptionChainProvider.cs +++ b/Engine/DataFeeds/BacktestingOptionChainProvider.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -27,6 +27,17 @@ namespace QuantConnect.Lean.Engine.DataFeeds /// public class BacktestingOptionChainProvider : IOptionChainProvider { + private IDataProvider _dataProvider; + + /// + /// Creates a new instance + /// + /// The data provider instance to use + public BacktestingOptionChainProvider(IDataProvider dataProvider) + { + _dataProvider = dataProvider; + } + /// /// Gets the list of option contracts for a given underlying symbol /// @@ -51,8 +62,8 @@ public IEnumerable GetOptionContractList(Symbol underlyingSymbol, DateTi 0, SecurityIdentifier.DefaultDate); - var fileExists = false; var zipFileName = string.Empty; + Stream stream = null; // In order of trust-worthiness of containing the complete option chain, OpenInterest is guaranteed // to have the complete option chain. Quotes come after open-interest @@ -60,27 +71,30 @@ public IEnumerable GetOptionContractList(Symbol underlyingSymbol, DateTi // missing portions of the option chain, so we resort to it last. foreach (var tickType in new[] { TickType.OpenInterest, TickType.Quote, TickType.Trade }) { - // build the zip file name for open interest data + // build the zip file name and fetch it with our provider zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, tickType); - if (File.Exists(zipFileName)) + stream = _dataProvider.Fetch(zipFileName); + + if (stream != null) { - fileExists = true; break; } } - if (!fileExists) + if (stream == null) { Log.Trace($"BacktestingOptionChainProvider.GetOptionContractList(): File not found: {zipFileName}"); yield break; } // generate and return the contract symbol for each zip entry - var zipEntryNames = Compression.GetZipEntryFileNames(zipFileName); + var zipEntryNames = Compression.GetZipEntryFileNames(stream); foreach (var zipEntryName in zipEntryNames) { yield return LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName); } + + stream.DisposeSafely(); } } } diff --git a/Engine/DataFeeds/DefaultDataProvider.cs b/Engine/DataFeeds/DefaultDataProvider.cs index eba7745d786e..2f6bb7d752f5 100644 --- a/Engine/DataFeeds/DefaultDataProvider.cs +++ b/Engine/DataFeeds/DefaultDataProvider.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -29,7 +29,7 @@ public class DefaultDataProvider : IDataProvider, IDisposable /// /// A string representing where the data is stored /// A of the data requested - public Stream Fetch(string key) + public virtual Stream Fetch(string key) { try { diff --git a/Engine/DataFeeds/Enumerators/Factories/BaseDataCollectionSubscriptionEnumeratorFactory.cs b/Engine/DataFeeds/Enumerators/Factories/BaseDataCollectionSubscriptionEnumeratorFactory.cs index 7f28e8f02658..dff09523e196 100644 --- a/Engine/DataFeeds/Enumerators/Factories/BaseDataCollectionSubscriptionEnumeratorFactory.cs +++ b/Engine/DataFeeds/Enumerators/Factories/BaseDataCollectionSubscriptionEnumeratorFactory.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -64,7 +64,7 @@ public IEnumerator CreateEnumerator(SubscriptionRequest request, IData foreach (var date in tradableDays) { var source = sourceFactory.GetSource(configuration, date, false); - var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, configuration, date, false, sourceFactory); + var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, configuration, date, false, sourceFactory, dataProvider); var coarseFundamentalForDate = factory.Read(source); // shift all date of emitting the file forward one day to model emitting coarse midnight the next day. yield return new BaseDataCollection(date.AddDays(1), configuration.Symbol, coarseFundamentalForDate); @@ -72,4 +72,4 @@ public IEnumerator CreateEnumerator(SubscriptionRequest request, IData } } } -} \ No newline at end of file +} diff --git a/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactory.cs b/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactory.cs index 4adc0ed8e048..da28aca631d5 100644 --- a/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactory.cs +++ b/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactory.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -83,7 +83,7 @@ public IEnumerator CreateEnumerator(SubscriptionRequest request, IData } var source = sourceFactory.GetSource(request.Configuration, date, _isLiveMode); - var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, request.Configuration, date, _isLiveMode, sourceFactory); + var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, request.Configuration, date, _isLiveMode, sourceFactory, dataProvider); var entriesForDate = factory.Read(source); foreach (var entry in entriesForDate) { diff --git a/Engine/DataFeeds/Enumerators/Factories/FineFundamentalSubscriptionEnumeratorFactory.cs b/Engine/DataFeeds/Enumerators/Factories/FineFundamentalSubscriptionEnumeratorFactory.cs index 19b3115aaff7..4b5a9aa25034 100644 --- a/Engine/DataFeeds/Enumerators/Factories/FineFundamentalSubscriptionEnumeratorFactory.cs +++ b/Engine/DataFeeds/Enumerators/Factories/FineFundamentalSubscriptionEnumeratorFactory.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -71,7 +71,7 @@ public IEnumerator CreateEnumerator(SubscriptionRequest request, IData foreach (var date in tradableDays) { var fineFundamentalSource = GetSource(FineFundamental, fineFundamentalConfiguration, date); - var fineFundamentalFactory = SubscriptionDataSourceReader.ForSource(fineFundamentalSource, dataCacheProvider, fineFundamentalConfiguration, date, _isLiveMode, FineFundamental); + var fineFundamentalFactory = SubscriptionDataSourceReader.ForSource(fineFundamentalSource, dataCacheProvider, fineFundamentalConfiguration, date, _isLiveMode, FineFundamental, dataProvider); var fineFundamentalForDate = (FineFundamental)fineFundamentalFactory.Read(fineFundamentalSource).FirstOrDefault(); // directly do not emit null points. Null points won't happen when used with Coarse data since we are pre filtering based on Coarse.HasFundamentalData @@ -217,4 +217,4 @@ private SubscriptionDataSource DailyBackwardsLoop(FineFundamental fine, Subscrip return count == 0 ? null : source; } } -} \ No newline at end of file +} diff --git a/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs b/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs index 59c933187c7a..86f5fba7a324 100644 --- a/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs +++ b/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs @@ -138,7 +138,7 @@ private IEnumerator EnumerateDataSourceReader(SubscriptionDataConfig c using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider)) { var newLocalFrontier = localFrontier.Value; - var dataSourceReader = GetSubscriptionDataSourceReader(source, dataCacheProvider, config, localDate, baseDataInstance); + var dataSourceReader = GetSubscriptionDataSourceReader(source, dataCacheProvider, config, localDate, baseDataInstance, dataProvider); foreach (var datum in dataSourceReader.Read(source)) { // always skip past all times emitted on the previous invocation of this enumerator @@ -184,10 +184,11 @@ protected virtual ISubscriptionDataSourceReader GetSubscriptionDataSourceReader( IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, - BaseData baseDataInstance + BaseData baseDataInstance, + IDataProvider dataProvider ) { - return SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, true, baseDataInstance); + return SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, true, baseDataInstance, dataProvider); } private bool SourceRequiresFastForward(SubscriptionDataSource source) @@ -206,4 +207,4 @@ private static TimeSpan GetMaximumDataAge(TimeSpan increment) return TimeSpan.FromTicks(Math.Max(increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); } } -} \ No newline at end of file +} diff --git a/Engine/DataFeeds/Enumerators/Factories/SubscriptionDataReaderSubscriptionEnumeratorFactory.cs b/Engine/DataFeeds/Enumerators/Factories/SubscriptionDataReaderSubscriptionEnumeratorFactory.cs index 74f4ef8cdc45..c51622b5c6af 100644 --- a/Engine/DataFeeds/Enumerators/Factories/SubscriptionDataReaderSubscriptionEnumeratorFactory.cs +++ b/Engine/DataFeeds/Enumerators/Factories/SubscriptionDataReaderSubscriptionEnumeratorFactory.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -93,7 +93,8 @@ public IEnumerator CreateEnumerator(SubscriptionRequest request, IData _factorFileProvider, _tradableDaysProvider(request), _isLiveMode, - _zipDataCacheProvider + _zipDataCacheProvider, + dataProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); }; diff --git a/Engine/DataFeeds/IndexSubscriptionDataSourceReader.cs b/Engine/DataFeeds/IndexSubscriptionDataSourceReader.cs index 7d9ab17e4773..b0618c120817 100644 --- a/Engine/DataFeeds/IndexSubscriptionDataSourceReader.cs +++ b/Engine/DataFeeds/IndexSubscriptionDataSourceReader.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -32,6 +32,7 @@ public class IndexSubscriptionDataSourceReader : BaseSubscriptionDataSourceReade { private readonly SubscriptionDataConfig _config; private readonly DateTime _date; + private IDataProvider _dataProvider; private readonly IndexedBaseData _factory; /// @@ -46,11 +47,13 @@ public class IndexSubscriptionDataSourceReader : BaseSubscriptionDataSourceReade public IndexSubscriptionDataSourceReader(IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, - bool isLiveMode) + bool isLiveMode, + IDataProvider dataProvider) : base(dataCacheProvider, isLiveMode) { _config = config; _date = date; + _dataProvider = dataProvider; _factory = config.Type.GetBaseDataInstance() as IndexedBaseData; if (_factory == null) { @@ -105,7 +108,8 @@ public override IEnumerable Read(SubscriptionDataSource source) _config, _date, IsLiveMode, - _factory); + _factory, + _dataProvider); var enumerator = dataReader.Read(dataSource).GetEnumerator(); while (enumerator.MoveNext()) diff --git a/Engine/DataFeeds/SubscriptionDataReader.cs b/Engine/DataFeeds/SubscriptionDataReader.cs index f5023f15d6c1..80503235e4ed 100644 --- a/Engine/DataFeeds/SubscriptionDataReader.cs +++ b/Engine/DataFeeds/SubscriptionDataReader.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -15,20 +15,19 @@ */ using System; +using System.Linq; using System.Collections; -using System.Collections.Generic; +using QuantConnect.Data; using System.Globalization; -using System.Linq; +using QuantConnect.Logging; +using QuantConnect.Interfaces; +using QuantConnect.Data.Custom; +using System.Collections.Generic; using QuantConnect.Configuration; -using QuantConnect.Data; using QuantConnect.Data.Auxiliary; -using QuantConnect.Data.Custom; using QuantConnect.Data.Custom.Fred; using QuantConnect.Data.Custom.Tiingo; -using QuantConnect.Interfaces; using QuantConnect.Lean.Engine.DataFeeds.Enumerators; -using QuantConnect.Logging; -using QuantConnect.Securities.Option; namespace QuantConnect.Lean.Engine.DataFeeds { @@ -38,6 +37,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds /// The class accepts any subscription configuration and automatically makes it available to enumerate public class SubscriptionDataReader : IEnumerator, ITradableDatesNotifier, IDataProviderEvents { + private IDataProvider _dataProvider; private bool _initialized; // Source string to create memory stream: @@ -138,6 +138,7 @@ object IEnumerator.Current /// Used for caching files /// Defines the dates for which we'll request data, in order, in the security's data time zone /// True if we're in live mode, false otherwise + /// The data provider to use public SubscriptionDataReader(SubscriptionDataConfig config, DateTime periodStart, DateTime periodFinish, @@ -145,7 +146,8 @@ public SubscriptionDataReader(SubscriptionDataConfig config, IFactorFileProvider factorFileProvider, IEnumerable tradeableDates, bool isLiveMode, - IDataCacheProvider dataCacheProvider) + IDataCacheProvider dataCacheProvider, + IDataProvider dataProvider) { //Save configuration of data-subscription: _config = config; @@ -160,6 +162,7 @@ public SubscriptionDataReader(SubscriptionDataConfig config, //Save access to securities _isLiveMode = isLiveMode; _tradeableDates = tradeableDates.GetEnumerator(); + _dataProvider = dataProvider; } /// @@ -460,7 +463,7 @@ private bool UpdateDataEnumerator(bool endOfEnumerator) // save off for comparison next time _source = newSource; - var subscriptionFactory = CreateSubscriptionFactory(newSource, _dataFactory); + var subscriptionFactory = CreateSubscriptionFactory(newSource, _dataFactory, _dataProvider); _subscriptionFactoryEnumerator = subscriptionFactory.Read(newSource).GetEnumerator(); return true; } @@ -479,9 +482,9 @@ private bool UpdateDataEnumerator(bool endOfEnumerator) while (true); } - private ISubscriptionDataSourceReader CreateSubscriptionFactory(SubscriptionDataSource source, BaseData baseDataInstance) + private ISubscriptionDataSourceReader CreateSubscriptionFactory(SubscriptionDataSource source, BaseData baseDataInstance, IDataProvider dataProvider) { - var factory = SubscriptionDataSourceReader.ForSource(source, _dataCacheProvider, _config, _tradeableDates.Current, _isLiveMode, baseDataInstance); + var factory = SubscriptionDataSourceReader.ForSource(source, _dataCacheProvider, _config, _tradeableDates.Current, _isLiveMode, baseDataInstance, dataProvider); AttachEventHandlers(factory, source); return factory; } diff --git a/Engine/DataFeeds/SubscriptionDataSourceReader.cs b/Engine/DataFeeds/SubscriptionDataSourceReader.cs index 526b5852b6e1..dcf7875f97fd 100644 --- a/Engine/DataFeeds/SubscriptionDataSourceReader.cs +++ b/Engine/DataFeeds/SubscriptionDataSourceReader.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -39,8 +39,9 @@ public static class SubscriptionDataSourceReader /// The date to be processed /// True for live mode, false otherwise /// The base data instance factory + /// The data provider to use /// A new that can read the specified - public static ISubscriptionDataSourceReader ForSource(SubscriptionDataSource source, IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode, BaseData factory) + public static ISubscriptionDataSourceReader ForSource(SubscriptionDataSource source, IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode, BaseData factory, IDataProvider dataProvider) { ISubscriptionDataSourceReader reader; TextSubscriptionDataSourceReader textReader = null; @@ -55,11 +56,11 @@ public static ISubscriptionDataSourceReader ForSource(SubscriptionDataSource sou break; case FileFormat.ZipEntryName: - reader = new ZipEntryNameSubscriptionDataSourceReader(config, date, isLiveMode); + reader = new ZipEntryNameSubscriptionDataSourceReader(dataProvider, config, date, isLiveMode); break; case FileFormat.Index: - return new IndexSubscriptionDataSourceReader(dataCacheProvider, config, date, isLiveMode); + return new IndexSubscriptionDataSourceReader(dataCacheProvider, config, date, isLiveMode, dataProvider); default: throw new NotImplementedException("SubscriptionFactory.ForSource(" + source + ") has not been implemented yet."); diff --git a/Engine/DataFeeds/ZipEntryNameSubscriptionDataSourceReader.cs b/Engine/DataFeeds/ZipEntryNameSubscriptionDataSourceReader.cs index 01128801893f..90c45d7c2cac 100644 --- a/Engine/DataFeeds/ZipEntryNameSubscriptionDataSourceReader.cs +++ b/Engine/DataFeeds/ZipEntryNameSubscriptionDataSourceReader.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -14,9 +14,12 @@ */ using System; -using System.Collections.Generic; using Ionic.Zip; +using System.Linq; using QuantConnect.Data; +using QuantConnect.Util; +using QuantConnect.Interfaces; +using System.Collections.Generic; namespace QuantConnect.Lean.Engine.DataFeeds { @@ -25,6 +28,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds /// public class ZipEntryNameSubscriptionDataSourceReader : ISubscriptionDataSourceReader { + private readonly IDataProvider _dataProvider; private readonly SubscriptionDataConfig _config; private readonly DateTime _date; private readonly bool _isLiveMode; @@ -39,14 +43,16 @@ public class ZipEntryNameSubscriptionDataSourceReader : ISubscriptionDataSourceR /// /// Initializes a new instance of the class /// + /// Used to fetch data /// The subscription's configuration /// The date this factory was produced to read data for /// True if we're in live mode, false for backtesting - public ZipEntryNameSubscriptionDataSourceReader(SubscriptionDataConfig config, DateTime date, bool isLiveMode) + public ZipEntryNameSubscriptionDataSourceReader(IDataProvider dataProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode) { - _config = config; _date = date; + _config = config; _isLiveMode = isLiveMode; + _dataProvider = dataProvider; _factory = _factory = config.GetBaseDataInstance(); } @@ -57,13 +63,17 @@ public ZipEntryNameSubscriptionDataSourceReader(SubscriptionDataConfig config, D /// An that contains the data in the source public IEnumerable Read(SubscriptionDataSource source) { - ICollection entryNames; + List entryNames; try { - using (var zip = new ZipFile(source.Source)) + var stream = _dataProvider.Fetch(source.Source); + if (stream == null) { - entryNames = zip.EntryFileNames; + OnInvalidSource(source, new ArgumentException($"Failed to create source stream {source.Source}")); + yield break; } + entryNames = Compression.GetZipEntryFileNames(stream).ToList(); + stream.DisposeSafely(); } catch (ZipException err) { diff --git a/Engine/Engine.cs b/Engine/Engine.cs index 46b33256da38..5f2402f58150 100644 --- a/Engine/Engine.cs +++ b/Engine/Engine.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -115,6 +115,10 @@ public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemb AlgorithmHandlers.Setup.WorkerThread = workerThread; + // Initialize our factorfile and mapfile providers, before creating the algorithm which could require these + AlgorithmHandlers.FactorFileProvider.Initialize(AlgorithmHandlers.MapFileProvider, AlgorithmHandlers.DataProvider); + AlgorithmHandlers.MapFileProvider.Initialize(AlgorithmHandlers.DataProvider); + // Save algorithm to cache, load algorithm instance: algorithm = AlgorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath); @@ -133,6 +137,7 @@ public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemb // notify the user of any errors w/ object store persistence AlgorithmHandlers.ObjectStore.ErrorRaised += (sender, args) => algorithm.Debug($"ObjectStore Persistence Error: {args.Error.Message}"); + // Initialize the brokerage IBrokerageFactory factory; brokerage = AlgorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory); @@ -224,7 +229,8 @@ public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemb algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, SystemHandlers.Api); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. - initializeComplete = AlgorithmHandlers.Setup.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage, job, AlgorithmHandlers.Results, AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime, AlgorithmHandlers.ObjectStore)); + initializeComplete = AlgorithmHandlers.Setup.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage, job, AlgorithmHandlers.Results, + AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime, AlgorithmHandlers.ObjectStore, AlgorithmHandlers.DataProvider)); // set this again now that we've actually added securities AlgorithmHandlers.Results.SetAlgorithm(algorithm, AlgorithmHandlers.Setup.StartingPortfolioValue); diff --git a/Engine/HistoricalData/SubscriptionDataReaderHistoryProvider.cs b/Engine/HistoricalData/SubscriptionDataReaderHistoryProvider.cs index 3bede849768f..04eb0b11d86b 100644 --- a/Engine/HistoricalData/SubscriptionDataReaderHistoryProvider.cs +++ b/Engine/HistoricalData/SubscriptionDataReaderHistoryProvider.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -38,6 +38,7 @@ namespace QuantConnect.Lean.Engine.HistoricalData /// public class SubscriptionDataReaderHistoryProvider : SynchronizingHistoryProvider { + private IDataProvider _dataProvider; private IMapFileProvider _mapFileProvider; private IFactorFileProvider _factorFileProvider; private IDataCacheProvider _dataCacheProvider; @@ -57,6 +58,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) throw new InvalidOperationException("SubscriptionDataReaderHistoryProvider can only be initialized once"); } _initialized = true; + _dataProvider = parameters.DataProvider; _mapFileProvider = parameters.MapFileProvider; _dataCacheProvider = parameters.DataCacheProvider; _factorFileProvider = parameters.FactorFileProvider; @@ -136,7 +138,8 @@ private Subscription CreateSubscription(HistoryRequest request, DateTime startUt _factorFileProvider, tradableDates, false, - _dataCacheProvider + _dataCacheProvider, + _dataProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); }; diff --git a/Engine/Setup/BacktestingSetupHandler.cs b/Engine/Setup/BacktestingSetupHandler.cs index 043b233f2327..1eb156166a9e 100644 --- a/Engine/Setup/BacktestingSetupHandler.cs +++ b/Engine/Setup/BacktestingSetupHandler.cs @@ -171,10 +171,10 @@ public bool Setup(SetupHandlerParameters parameters) algorithm.Schedule.SetEventSchedule(parameters.RealTimeHandler); // set the option chain provider - algorithm.SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider())); + algorithm.SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider(parameters.DataProvider))); // set the future chain provider - algorithm.SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider())); + algorithm.SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider(parameters.DataProvider))); // set the object store algorithm.SetObjectStore(parameters.ObjectStore); diff --git a/Engine/Setup/SetupHandlerParameters.cs b/Engine/Setup/SetupHandlerParameters.cs index 389cf67f0d5b..7fb8fef7f9e6 100644 --- a/Engine/Setup/SetupHandlerParameters.cs +++ b/Engine/Setup/SetupHandlerParameters.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -68,6 +68,11 @@ public class SetupHandlerParameters /// public IObjectStore ObjectStore { get; } + /// + /// Gets the DataProvider + /// + public IDataProvider DataProvider { get; } + /// /// Creates a new instance /// @@ -79,6 +84,7 @@ public class SetupHandlerParameters /// The configured transaction handler /// The configured real time handler /// The configured object store + /// The configured data provider public SetupHandlerParameters(UniverseSelection universeSelection, IAlgorithm algorithm, IBrokerage brokerage, @@ -86,7 +92,8 @@ public SetupHandlerParameters(UniverseSelection universeSelection, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler, - IObjectStore objectStore + IObjectStore objectStore, + IDataProvider dataProvider ) { UniverseSelection = universeSelection; @@ -97,6 +104,7 @@ IObjectStore objectStore TransactionHandler = transactionHandler; RealTimeHandler = realTimeHandler; ObjectStore = objectStore; + DataProvider = dataProvider; } } } diff --git a/Launcher/config.json b/Launcher/config.json index 73e931b015dc..1cbd1df95a93 100644 --- a/Launcher/config.json +++ b/Launcher/config.json @@ -59,6 +59,7 @@ // To get your api access token go to quantconnect.com/account "job-user-id": "0", "api-access-token": "", + "job-organization-id": "", // live data configuration "live-data-url": "ws://www.quantconnect.com/api/v2/live/data/", @@ -187,7 +188,7 @@ }, // specify supported languages when running regression tests - "regression-test-languages": ["CSharp", "Python"], + "regression-test-languages": [ "CSharp", "Python" ], "environments": { diff --git a/Research/QuantBook.cs b/Research/QuantBook.cs index f5dea64d2cea..3fc4bbeb1fe5 100644 --- a/Research/QuantBook.cs +++ b/Research/QuantBook.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -118,6 +118,11 @@ public QuantBook() : base() var composer = new Composer(); var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer); var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(composer); + + // Initialize our factorfile and mapfile providers, before creating the algorithm which could require these + algorithmHandlers.FactorFileProvider.Initialize(algorithmHandlers.MapFileProvider, algorithmHandlers.DataProvider); + algorithmHandlers.MapFileProvider.Initialize(algorithmHandlers.DataProvider); + // init the API systemHandlers.Initialize(); systemHandlers.LeanManager.Initialize(systemHandlers, @@ -178,8 +183,8 @@ public QuantBook() : base() ) ); - SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider())); - SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider())); + SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider(_dataProvider))); + SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider(_dataProvider))); } catch (Exception exception) { diff --git a/Tests/Algorithm/AlgorithmDownloadTests.cs b/Tests/Algorithm/AlgorithmDownloadTests.cs index 6f6803af7874..8c23829cc44b 100644 --- a/Tests/Algorithm/AlgorithmDownloadTests.cs +++ b/Tests/Algorithm/AlgorithmDownloadTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * diff --git a/Tests/Algorithm/AlgorithmHistoryTests.cs b/Tests/Algorithm/AlgorithmHistoryTests.cs index 9a78520b517b..9e5fbff582a0 100644 --- a/Tests/Algorithm/AlgorithmHistoryTests.cs +++ b/Tests/Algorithm/AlgorithmHistoryTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -20,8 +20,8 @@ using NUnit.Framework; using QuantConnect.Algorithm; using QuantConnect.Data; -using QuantConnect.Data.Auxiliary; using QuantConnect.Data.Market; +using QuantConnect.Interfaces; using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.HistoricalData; using QuantConnect.Tests.Engine.DataFeeds; @@ -35,6 +35,9 @@ public class AlgorithmHistoryTests { private QCAlgorithm _algorithm; private TestHistoryProvider _testHistoryProvider; + private IDataProvider _dataProvider; + private IMapFileProvider _mapFileProvider; + private IFactorFileProvider _factorFileProvider; [SetUp] public void Setup() @@ -42,6 +45,10 @@ public void Setup() _algorithm = new QCAlgorithm(); _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm)); _algorithm.HistoryProvider = _testHistoryProvider = new TestHistoryProvider(); + + _dataProvider = TestGlobals.DataProvider; + _mapFileProvider = TestGlobals.MapFileProvider; + _factorFileProvider = TestGlobals.FactorFileProvider; } [Test] @@ -50,15 +57,14 @@ public void TickResolutionHistoryRequest() _algorithm = new QCAlgorithm(); _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm)); _algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider(); - var dataProvider = new DefaultDataProvider(); - var zipCacheProvider = new ZipDataCacheProvider(dataProvider); + var zipCacheProvider = new ZipDataCacheProvider(_dataProvider); _algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters( null, null, - dataProvider, + _dataProvider, zipCacheProvider, - new LocalDiskMapFileProvider(), - new LocalDiskFactorFileProvider(), + _mapFileProvider, + _factorFileProvider, null, false, new DataPermissionManager())); @@ -173,14 +179,15 @@ public void GetLastKnownPriceOfIlliquidAsset_RealData() var algorithm = new QCAlgorithm(); algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(algorithm)); algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider(); - var cacheProvider = new ZipDataCacheProvider(new DefaultDataProvider()); + var cacheProvider = new ZipDataCacheProvider(_dataProvider); + algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters( null, null, - new DefaultDataProvider(), + _dataProvider, cacheProvider, - new LocalDiskMapFileProvider(), - new LocalDiskFactorFileProvider(), + _mapFileProvider, + _factorFileProvider, null, false, new DataPermissionManager())); @@ -227,15 +234,14 @@ public void TickResolutionOpenInterestHistoryRequestIsNotFilteredWhenRequestedEx _algorithm = new QCAlgorithm(); _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm)); _algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider(); - var dataProvider = new DefaultDataProvider(); - var zipCacheProvider = new ZipDataCacheProvider(dataProvider); + var zipCacheProvider = new ZipDataCacheProvider(_dataProvider); _algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters( null, null, - dataProvider, + _dataProvider, zipCacheProvider, - new LocalDiskMapFileProvider(), - new LocalDiskFactorFileProvider(), + _mapFileProvider, + _factorFileProvider, null, false, new DataPermissionManager())); @@ -263,15 +269,14 @@ public void TickResolutionOpenInterestHistoryRequestIsFilteredByDefault_SingleSy _algorithm = new QCAlgorithm(); _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm)); _algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider(); - var dataProvider = new DefaultDataProvider(); - var zipCacheProvider = new ZipDataCacheProvider(dataProvider); + var zipCacheProvider = new ZipDataCacheProvider(_dataProvider); _algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters( null, null, - dataProvider, + _dataProvider, zipCacheProvider, - new LocalDiskMapFileProvider(), - new LocalDiskFactorFileProvider(), + _mapFileProvider, + _factorFileProvider, null, false, new DataPermissionManager())); @@ -297,15 +302,14 @@ public void TickResolutionOpenInterestHistoryRequestIsFilteredByDefault_Multiple _algorithm = new QCAlgorithm(); _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm)); _algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider(); - var dataProvider = new DefaultDataProvider(); - var zipCacheProvider = new ZipDataCacheProvider(dataProvider); + var zipCacheProvider = new ZipDataCacheProvider(_dataProvider); _algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters( null, null, - dataProvider, + _dataProvider, zipCacheProvider, - new LocalDiskMapFileProvider(), - new LocalDiskFactorFileProvider(), + _mapFileProvider, + _factorFileProvider, null, false, new DataPermissionManager())); diff --git a/Tests/Algorithm/Framework/Portfolio/MeanVarianceOptimizationPortfolioConstructionModelTests.cs b/Tests/Algorithm/Framework/Portfolio/MeanVarianceOptimizationPortfolioConstructionModelTests.cs index bac17a6e1f70..e0a348bca1aa 100644 --- a/Tests/Algorithm/Framework/Portfolio/MeanVarianceOptimizationPortfolioConstructionModelTests.cs +++ b/Tests/Algorithm/Framework/Portfolio/MeanVarianceOptimizationPortfolioConstructionModelTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -51,10 +51,10 @@ public virtual void SetUp() historyProvider.Initialize(new HistoryProviderInitializeParameters( new BacktestNodePacket(), new Api.Api(), - new DefaultDataProvider(), + TestGlobals.DataProvider, new SingleEntryDataCacheProvider(new DefaultDataProvider()), - new LocalDiskMapFileProvider(), - new LocalDiskFactorFileProvider(new LocalDiskMapFileProvider()), + TestGlobals.MapFileProvider, + TestGlobals.FactorFileProvider, i => { }, true, new DataPermissionManager())); diff --git a/Tests/Api/ApiTestBase.cs b/Tests/Api/ApiTestBase.cs new file mode 100644 index 000000000000..2c33cd8a93e4 --- /dev/null +++ b/Tests/Api/ApiTestBase.cs @@ -0,0 +1,47 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.Configuration; + +namespace QuantConnect.Tests.API +{ + /// + /// Base test class for Api tests, provides the setup needed for all api tests + /// + public class ApiTestBase + { + internal int TestAccount; + internal string TestToken; + internal string TestOrganization; + internal string DataFolder; + internal Api.Api ApiClient; + + /// + /// Run once before any RestApiTests + /// + [OneTimeSetUp] + public void Setup() + { + TestAccount = Config.GetInt("job-user-id", 1); + TestToken = Config.Get("api-access-token", "EnterTokenHere"); + TestOrganization = Config.Get("job-organization-id", "EnterOrgHere"); + DataFolder = Config.Get("data-folder"); + + ApiClient = new Api.Api(); + ApiClient.Initialize(TestAccount, TestToken, DataFolder); + } + } +} diff --git a/Tests/Api/ApiTests.cs b/Tests/Api/ApiTests.cs index 751d3543a859..6f3222c0a015 100644 --- a/Tests/Api/ApiTests.cs +++ b/Tests/Api/ApiTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -12,69 +12,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -using System; -using System.IO; -using System.Linq; -using System.Threading; -using System.Web; + using NUnit.Framework; using QuantConnect.Api; -using QuantConnect.Configuration; namespace QuantConnect.Tests.API { - [TestFixture, Explicit("These tests require QC User ID and API Token in the configuration")] - class RestApiTests + /// + /// API Object tests + /// Tests APIs ability to connect to Web API + /// + [TestFixture, Explicit("Requires configured api access")] + public class ApiTest : ApiTestBase { - private int _testAccount; - private string _testToken; - private string _dataFolder; - private Api.Api _api; - - /// - /// Run before test - /// - [OneTimeSetUp] - public void Setup() - { - _testAccount = Config.GetInt("job-user-id", 1); - _testToken = Config.Get("api-access-token", "ec87b337ac970da4cbea648f24f1c851"); - _dataFolder = Config.Get("data-folder"); - - _api = new Api.Api(); - _api.Initialize(_testAccount, _testToken, _dataFolder); - } - - /// - /// Test creating and deleting projects with the Api - /// - [Test] - public void Projects_CanBeCreatedAndDeleted_Successfully() - { - var name = "Test Project " + DateTime.Now.ToStringInvariant(); - - //Test create a new project successfully - var project = _api.CreateProject(name, Language.CSharp); - Assert.IsTrue(project.Success); - Assert.IsTrue(project.Projects.First().ProjectId > 0); - Assert.IsTrue(project.Projects.First().Name == name); - - // Delete the project - var deleteProject = _api.DeleteProject(project.Projects.First().ProjectId); - Assert.IsTrue(deleteProject.Success); - - // Make sure the project is really deleted - var projectList = _api.ListProjects(); - Assert.IsFalse(projectList.Projects.Any(p => p.ProjectId == project.Projects.First().ProjectId)); - } - /// /// Test successfully authenticating with the ApiConnection using valid credentials. /// [Test] public void ApiConnectionWillAuthenticate_ValidCredentials_Successfully() { - var connection = new ApiConnection(_testAccount, _testToken); + var connection = new ApiConnection(TestAccount, TestToken); Assert.IsTrue(connection.Connected); } @@ -85,7 +42,7 @@ public void ApiConnectionWillAuthenticate_ValidCredentials_Successfully() public void ApiWillAuthenticate_ValidCredentials_Successfully() { var api = new Api.Api(); - api.Initialize(_testAccount, _testToken, _dataFolder); + api.Initialize(TestAccount, TestToken, DataFolder); Assert.IsTrue(api.Connected); } @@ -95,7 +52,7 @@ public void ApiWillAuthenticate_ValidCredentials_Successfully() [Test] public void ApiConnectionWillAuthenticate_InvalidCredentials_Unsuccessfully() { - var connection = new ApiConnection(_testAccount, ""); + var connection = new ApiConnection(TestAccount, ""); Assert.IsFalse(connection.Connected); } @@ -106,322 +63,8 @@ public void ApiConnectionWillAuthenticate_InvalidCredentials_Unsuccessfully() public void ApiWillAuthenticate_InvalidCredentials_Unsuccessfully() { var api = new Api.Api(); - api.Initialize(_testAccount, "", _dataFolder); + api.Initialize(TestAccount, "", DataFolder); Assert.IsFalse(api.Connected); } - - /// - /// Test updating the files associated with a project - /// - [Test] - public void CRUD_ProjectFiles_Successfully() - { - var fakeFile = new ProjectFile - { - Name = "Hello.cs", - Code = HttpUtility.HtmlEncode("Hello, world!") - }; - - var realFile = new ProjectFile - { - Name = "main.cs", - Code = HttpUtility.HtmlEncode(File.ReadAllText("../../../Algorithm.CSharp/BasicTemplateAlgorithm.cs")) - }; - - var secondRealFile = new ProjectFile() - { - Name = "lol.cs", - Code = HttpUtility.HtmlEncode(File.ReadAllText("../../../Algorithm.CSharp/BubbleAlgorithm.cs")) - }; - - // Create a new project and make sure there are no files - var project = _api.CreateProject($"Test project - {DateTime.Now.ToStringInvariant()}", Language.CSharp); - Assert.IsTrue(project.Success); - Assert.IsTrue(project.Projects.First().ProjectId > 0); - - // Add random file - var randomAdd = _api.AddProjectFile(project.Projects.First().ProjectId, fakeFile.Name, fakeFile.Code); - Assert.IsTrue(randomAdd.Success); - Assert.IsTrue(randomAdd.Files.First().Code == fakeFile.Code); - Assert.IsTrue(randomAdd.Files.First().Name == fakeFile.Name); - // Update names of file - var updatedName = _api.UpdateProjectFileName(project.Projects.First().ProjectId, randomAdd.Files.First().Name, realFile.Name); - Assert.IsTrue(updatedName.Success); - - // Replace content of file - var updateContents = _api.UpdateProjectFileContent(project.Projects.First().ProjectId, realFile.Name, realFile.Code); - Assert.IsTrue(updateContents.Success); - - // Read single file - var readFile = _api.ReadProjectFile(project.Projects.First().ProjectId, realFile.Name); - Assert.IsTrue(readFile.Success); - Assert.IsTrue(readFile.Files.First().Code == realFile.Code); - Assert.IsTrue(readFile.Files.First().Name == realFile.Name); - - // Add a second file - var secondFile = _api.AddProjectFile(project.Projects.First().ProjectId, secondRealFile.Name, secondRealFile.Code); - Assert.IsTrue(secondFile.Success); - Assert.IsTrue(secondFile.Files.First().Code == secondRealFile.Code); - Assert.IsTrue(secondFile.Files.First().Name == secondRealFile.Name); - - // Read multiple files - var readFiles = _api.ReadProjectFiles(project.Projects.First().ProjectId); - Assert.IsTrue(readFiles.Success); - Assert.IsTrue(readFiles.Files.Count == 2); - - // Delete the second file - var deleteFile = _api.DeleteProjectFile(project.Projects.First().ProjectId, secondRealFile.Name); - Assert.IsTrue(deleteFile.Success); - - // Read files - var readFilesAgain = _api.ReadProjectFiles(project.Projects.First().ProjectId); - Assert.IsTrue(readFilesAgain.Success); - Assert.IsTrue(readFilesAgain.Files.Count == 1); - Assert.IsTrue(readFilesAgain.Files.First().Name == realFile.Name); - - - // Delete the project - var deleteProject = _api.DeleteProject(project.Projects.First().ProjectId); - Assert.IsTrue(deleteProject.Success); - } - - /// - /// Test downloading data that does not come with the repo (Oanda) - /// Requires that your account has this data; its free at quantconnect.com/data - /// - [Test, Ignore("Requires EURUSD daily data and minute data for 10/2013 in cloud data library")] - public void BacktestingData_CanBeDownloadedAndSaved_Successfully() - { - var minutePath = Path.Combine(_dataFolder, "forex/oanda/minute/eurusd/20131011_quote.zip"); - var dailyPath = Path.Combine(_dataFolder, "forex/oanda/daily/eurusd.zip"); - - if (File.Exists(dailyPath)) - File.Delete(dailyPath); - - if (File.Exists(minutePath)) - File.Delete(minutePath); - - var downloadedMinuteData = _api.DownloadData(new Symbol(SecurityIdentifier.GenerateForex("EURUSD", Market.Oanda), "EURUSD"), - Resolution.Minute, new DateTime(2013, 10, 11)); - var downloadedDailyData = _api.DownloadData(new Symbol(SecurityIdentifier.GenerateForex("EURUSD", Market.Oanda), "EURUSD"), - Resolution.Daily, new DateTime(2013, 10, 07)); - - Assert.IsTrue(downloadedMinuteData); - Assert.IsTrue(downloadedDailyData); - - Assert.IsTrue(File.Exists(dailyPath)); - Assert.IsTrue(File.Exists(minutePath)); - } - - /// - /// Test downloading non existent data - /// - [Test] - public void NonExistantData_WillBeDownloaded_Unsuccessfully() - { - var nonExistentData = _api.DownloadData(new Symbol(SecurityIdentifier.GenerateForex("EURUSD", Market.Oanda), "EURUSD"), - Resolution.Minute, new DateTime(1989, 10, 11)); - - Assert.IsFalse(nonExistentData); - } - - /// - /// Test creating, compiling and backtesting a C# project via the Api - /// - [Test] - public void CSharpProject_CreatedCompiledAndBacktested_Successully() - { - var language = Language.CSharp; - var code = File.ReadAllText("../../../Algorithm.CSharp/BasicTemplateAlgorithm.cs"); - var algorithmName = "main.cs"; - var projectName = $"{DateTime.UtcNow.ToStringInvariant("u")} Test {_testAccount} Lang {language}"; - - Perform_CreateCompileBackTest_Tests(projectName, language, algorithmName, code); - } - - /// - /// Test creating, compiling and backtesting a Python project via the Api - /// - [Test] - public void PythonProject_CreatedCompiledAndBacktested_Successully() - { - var language = Language.Python; - var code = File.ReadAllText("../../../Algorithm.Python/BasicTemplateAlgorithm.py"); - var algorithmName = "main.py"; - - var projectName = $"{DateTime.UtcNow.ToStringInvariant("u")} Test {_testAccount} Lang {language}"; - - Perform_CreateCompileBackTest_Tests(projectName, language, algorithmName, code); - } - - /// - /// Test getting links to forex data for FXCM - /// - [Test, Ignore("Requires configured FXCM account")] - public void FXCMDataLinks_CanBeRetrieved_Successfully() - { - var minuteDataLink = _api.ReadDataLink(new Symbol(SecurityIdentifier.GenerateForex("EURUSD", Market.FXCM), "EURUSD"), - Resolution.Minute, new DateTime(2013, 10, 07)); - var dailyDataLink = _api.ReadDataLink(new Symbol(SecurityIdentifier.GenerateForex("EURUSD", Market.FXCM), "EURUSD"), - Resolution.Daily, new DateTime(2013, 10, 07)); - - Assert.IsTrue(minuteDataLink.Success); - Assert.IsTrue(dailyDataLink.Success); - } - - /// - /// Test getting links to forex data for Oanda - /// - [Test, Ignore("Requires configured Oanda account")] - public void OandaDataLinks_CanBeRetrieved_Successfully() - { - var minuteDataLink = _api.ReadDataLink(new Symbol(SecurityIdentifier.GenerateForex("EURUSD", Market.Oanda), "EURUSD"), - Resolution.Minute, new DateTime(2013, 10, 07)); - var dailyDataLink = _api.ReadDataLink(new Symbol(SecurityIdentifier.GenerateForex("EURUSD", Market.Oanda), "EURUSD"), - Resolution.Daily, new DateTime(2013, 10, 07)); - - Assert.IsTrue(minuteDataLink.Success); - Assert.IsTrue(dailyDataLink.Success); - } - - [TestCase("organizationId")] - [TestCase("")] - public void ReadAccount(string organizationId) - { - var account = _api.ReadAccount(organizationId); - - Assert.IsTrue(account.Success); - Assert.IsNotEmpty(account.OrganizationId); - Assert.IsNotNull(account.Card); - Assert.AreNotEqual(default(DateTime),account.Card.Expiration); - Assert.IsNotEmpty(account.Card.Brand); - Assert.AreNotEqual(0, account.Card.LastFourDigits); - } - - private void Perform_CreateCompileBackTest_Tests(string projectName, Language language, string algorithmName, string code) - { - //Test create a new project successfully - var project = _api.CreateProject(projectName, language); - Assert.IsTrue(project.Success); - Assert.IsTrue(project.Projects.First().ProjectId > 0); - Assert.IsTrue(project.Projects.First().Name == projectName); - - // Make sure the project just created is now present - var projects = _api.ListProjects(); - Assert.IsTrue(projects.Success); - Assert.IsTrue(projects.Projects.Any(p => p.ProjectId == project.Projects.First().ProjectId)); - - // Test read back the project we just created - var readProject = _api.ReadProject(project.Projects.First().ProjectId); - Assert.IsTrue(readProject.Success); - Assert.IsTrue(readProject.Projects.First().Name == projectName); - - // Test set a project file for the project - var file = new ProjectFile { Name = algorithmName, Code = code }; - var addProjectFile = _api.AddProjectFile(project.Projects.First().ProjectId, file.Name, file.Code); - Assert.IsTrue(addProjectFile.Success); - - // Download the project again to validate its got the new file - var verifyRead = _api.ReadProject(project.Projects.First().ProjectId); - Assert.IsTrue(verifyRead.Success); - - // Compile the project we've created - var compileCreate = _api.CreateCompile(project.Projects.First().ProjectId); - Assert.IsTrue(compileCreate.Success); - Assert.IsTrue(compileCreate.State == CompileState.InQueue); - - // Read out the compile - var compileSuccess = WaitForCompilerResponse(project.Projects.First().ProjectId, compileCreate.CompileId); - Assert.IsTrue(compileSuccess.Success); - Assert.IsTrue(compileSuccess.State == CompileState.BuildSuccess); - - // Update the file, create a build error, test we get build error - file.Code += "[Jibberish at end of the file to cause a build error]"; - _api.UpdateProjectFileContent(project.Projects.First().ProjectId, file.Name, file.Code); - var compileError = _api.CreateCompile(project.Projects.First().ProjectId); - compileError = WaitForCompilerResponse(project.Projects.First().ProjectId, compileError.CompileId); - Assert.IsTrue(compileError.Success); // Successfully processed rest request. - Assert.IsTrue(compileError.State == CompileState.BuildError); //Resulting in build fail. - - // Using our successful compile; launch a backtest! - var backtestName = $"{DateTime.Now.ToStringInvariant("u")} API Backtest"; - var backtest = _api.CreateBacktest(project.Projects.First().ProjectId, compileSuccess.CompileId, backtestName); - Assert.IsTrue(backtest.Success); - - // Now read the backtest and wait for it to complete - var backtestRead = WaitForBacktestCompletion(project.Projects.First().ProjectId, backtest.BacktestId); - Assert.IsTrue(backtestRead.Success); - Assert.IsTrue(backtestRead.Progress == 1); - Assert.IsTrue(backtestRead.Name == backtestName); - Assert.IsTrue(backtestRead.Statistics["Total Trades"] == "1"); - Assert.IsTrue(backtestRead.Charts["Benchmark"].Series.Count > 0); - - // Verify we have the backtest in our project - var listBacktests = _api.ListBacktests(project.Projects.First().ProjectId); - Assert.IsTrue(listBacktests.Success); - Assert.IsTrue(listBacktests.Backtests.Count >= 1); - Assert.IsTrue(listBacktests.Backtests[0].Name == backtestName); - - // Update the backtest name and test its been updated - backtestName += "-Amendment"; - var renameBacktest = _api.UpdateBacktest(project.Projects.First().ProjectId, backtest.BacktestId, backtestName); - Assert.IsTrue(renameBacktest.Success); - backtestRead = _api.ReadBacktest(project.Projects.First().ProjectId, backtest.BacktestId); - Assert.IsTrue(backtestRead.Name == backtestName); - - //Update the note and make sure its been updated: - var newNote = DateTime.Now.ToStringInvariant("u"); - var noteBacktest = _api.UpdateBacktest(project.Projects.First().ProjectId, backtest.BacktestId, note: newNote); - Assert.IsTrue(noteBacktest.Success); - backtestRead = _api.ReadBacktest(project.Projects.First().ProjectId, backtest.BacktestId); - Assert.IsTrue(backtestRead.Note == newNote); - - // Delete the backtest we just created - var deleteBacktest = _api.DeleteBacktest(project.Projects.First().ProjectId, backtest.BacktestId); - Assert.IsTrue(deleteBacktest.Success); - - // Test delete the project we just created - var deleteProject = _api.DeleteProject(project.Projects.First().ProjectId); - Assert.IsTrue(deleteProject.Success); - } - - /// - /// Wait for the compiler to respond to a specified compile request - /// - /// Id of the project - /// Id of the compilation of the project - /// - private Compile WaitForCompilerResponse(int projectId, string compileId) - { - var compile = new Compile(); - var finish = DateTime.Now.AddSeconds(60); - while (DateTime.Now < finish) - { - compile = _api.ReadCompile(projectId, compileId); - if (compile.State == CompileState.BuildSuccess) break; - Thread.Sleep(1000); - } - return compile; - } - - /// - /// Wait for the backtest to complete - /// - /// Project id to scan - /// Backtest id previously started - /// Completed backtest object - private Backtest WaitForBacktestCompletion(int projectId, string backtestId) - { - var result = new Backtest(); - var finish = DateTime.Now.AddSeconds(60); - while (DateTime.Now < finish) - { - result = _api.ReadBacktest(projectId, backtestId); - if (result.Progress == 1) break; - if (!result.Success) break; - Thread.Sleep(1000); - } - return result; - } } } diff --git a/Tests/Api/DataTests.cs b/Tests/Api/DataTests.cs new file mode 100644 index 000000000000..59d24ece2b40 --- /dev/null +++ b/Tests/Api/DataTests.cs @@ -0,0 +1,140 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.IO; +using NUnit.Framework; +using QuantConnect.Api; +using QuantConnect.Util; + +namespace QuantConnect.Tests.API +{ + /// + /// API Data endpoint tests + /// + [TestFixture, Explicit("Requires configured api access, and also makes calls to data endpoints which are charging transactions")] + public class DataTests : ApiTestBase + { + private DataPricesList _pricesCache; + private static object[] validForexDataTestCases = + { + new object[] { "EURUSD", Market.Oanda, new DateTime(2013,10,07), Resolution.Daily, TickType.Quote }, + new object[] { "EURUSD", Market.Oanda, new DateTime(2013,10,07), Resolution.Minute, TickType.Quote } + }; + + /// + /// Test downloading data + /// + [TestCase("forex/oanda/minute/eurusd/20131011_quote.zip")] + [TestCase("forex/oanda/daily/eurusd.zip")] + public void DataDownloadedAndSaved(string fileToDownload) + { + var path = Path.Combine(DataFolder, fileToDownload); + + if (File.Exists(path)) + File.Delete(path); + + var downloadedData = ApiClient.DownloadData(path, TestOrganization); + + Assert.IsTrue(downloadedData); + Assert.IsTrue(File.Exists(path)); + } + + /// + /// Test attempting to fetch invalid data links + /// + [Test] + public void InvalidDataLinks() + { + var fakePath = Path.Combine(DataFolder, "forex/oanda/minute/eurusd/19891011_quote.zip"); + var nonExistentData = ApiClient.DownloadData(fakePath, TestOrganization); + + Assert.IsFalse(nonExistentData); + Assert.IsFalse(File.Exists(fakePath)); + } + + /// + /// Test getting links to forex data for Oanda + /// + [TestCaseSource(nameof(validForexDataTestCases))] + public void ValidForexDataLinks(string ticker, string market, DateTime date, Resolution resolution, TickType tickType) + { + var path = LeanData.GenerateRelativeZipFilePath( + new Symbol(SecurityIdentifier.GenerateForex(ticker, market), ticker), + date, resolution, tickType); + var dataLink = ApiClient.ReadDataLink(path, TestOrganization); + + Assert.IsTrue(dataLink.Success); + Assert.IsFalse(dataLink.Url.IsNullOrEmpty()); + } + + /// + /// Test getting price for file + /// + /// + [TestCase("forex/oanda/daily/eurusd.zip")] + [TestCase("crypto/gdax/daily/btcusd_quote.zip")] + [TestCase("\\index\\usa\\minute\\spx")] + public void GetPrices(string filePath) + { + if (_pricesCache == null) + { + _pricesCache = ApiClient.ReadDataPrices(TestOrganization); + } + + // Make sure we actually have these prices for the test to work + Assert.IsTrue(_pricesCache.Success); + + // Get the price + int price = _pricesCache.GetPrice(filePath); + Assert.IsTrue(price != 0); + } + + /// + /// Test regex implementation for DataPriceList price matching + /// + /// + /// + [TestCase("forex/oanda/daily/eurusd.zip", "/^(cfd|forex)\\/oanda\\/(daily|hour)\\/[^\\/]+.zip$/m")] + [TestCase("forex/oanda/daily/eurusd.zip", "/^(cfd|forex)\\/oanda\\/(daily|hour)\\/[^\\/]+.zip$")] + [TestCase("forex/oanda/daily/eurusd.zip", "^(cfd|forex)\\/oanda\\/(daily|hour)\\/[^\\/]+.zip$/")] + [TestCase("forex/oanda/daily/eurusd.zip", "^(cfd|forex)\\/oanda\\/(daily|hour)\\/[^\\/]+.zip$")] + public void DataPriceRegex(string dataFile, string matchingRegex) + { + var setPrice = 10; + var dataList = new DataPricesList + { + Prices = new List() { new PriceEntry() { Price = setPrice, RegEx = matchingRegex } } + }; + + int price = dataList.GetPrice(dataFile); + Assert.AreEqual(setPrice, price); + } + + /// + /// Test getting available data listings for directories + /// + /// + [TestCase("forex/oanda/minute/eurusd/")] + [TestCase("forex\\oanda\\minute\\eurusd\\")] + public void GetDataListings(string directory) + { + var dataList = ApiClient.ReadDataDirectory(directory); + Assert.IsTrue(dataList.Success); + Assert.IsTrue(dataList.AvailableData.Count > 0); + } + } +} diff --git a/Tests/Api/ApiLiveTradingTests.cs b/Tests/Api/LiveTradingTests.cs similarity index 87% rename from Tests/Api/ApiLiveTradingTests.cs rename to Tests/Api/LiveTradingTests.cs index 9525c8806466..24cab5919e77 100644 --- a/Tests/Api/ApiLiveTradingTests.cs +++ b/Tests/Api/LiveTradingTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -17,7 +17,6 @@ using System.IO; using System.Linq; using System.Threading; -using System.Web; using NUnit.Framework; using QuantConnect.Api; using QuantConnect.Brokerages; @@ -25,31 +24,13 @@ namespace QuantConnect.Tests.API { - [TestFixture, Ignore("These tests require configured and active accounts to Interactive Brokers, FXCM, Oanda, and Tradier " + - "as well as your Organization ID and available live nodes")] - public class ApiLiveTradingTests + /// + /// API Live endpoint tests + /// + [TestFixture, Explicit("Requires configured api access, a live node to run on, and brokerage configurations.")] + public class LiveTradingTests : ApiTestBase { - private int _testAccount; - private string _testToken; - private string _testOrganization; - private string _dataFolder; - private Api.Api _api; - private const bool stopLiveAlgos = true; - - /// - /// Run before test - /// - [OneTimeSetUp] - public void Setup() - { - _testAccount = Config.GetInt("job-user-id"); - _testToken = Config.Get("api-access-token"); - _testOrganization = Config.Get("job-organization-id", "EnterOrgHere"); //This org must be your preferred org - _dataFolder = Config.Get("data-folder"); - - _api = new Api.Api(); - _api.Initialize(_testAccount, _testToken, _dataFolder); - } + private const bool StopLiveAlgos = true; /// /// Live paper trading via Interactive Brokers @@ -319,11 +300,11 @@ public void LiveAlgorithmSettings_CanBeCreated_Successfully() /// - Get logs for the first algorithm returned /// Will there always be a live algorithm for the test user? /// - [Test, Ignore("Requires a live algorithm to be running")] + [Test] public void LiveAlgorithmsAndLiveLogs_CanBeRead_Successfully() { // Read all currently running algorithms - var liveAlgorithms = _api.ListLiveAlgorithms(AlgorithmStatus.Running); + var liveAlgorithms = ApiClient.ListLiveAlgorithms(AlgorithmStatus.Running); Assert.IsTrue(liveAlgorithms.Success); // There has to be at least one running algorithm @@ -331,7 +312,7 @@ public void LiveAlgorithmsAndLiveLogs_CanBeRead_Successfully() // Read the logs of the first live algorithm var firstLiveAlgo = liveAlgorithms.Algorithms[0]; - var liveLogs = _api.ReadLiveLogs(firstLiveAlgo.ProjectId, firstLiveAlgo.DeployId); + var liveLogs = ApiClient.ReadLiveLogs(firstLiveAlgo.ProjectId, firstLiveAlgo.DeployId); Assert.IsTrue(liveLogs.Success); Assert.IsTrue(liveLogs.Logs.Any()); @@ -345,14 +326,14 @@ public void LiveAlgorithmsAndLiveLogs_CanBeRead_Successfully() private void RunLiveAlgorithm(BaseLiveAlgorithmSettings settings, ProjectFile file) { // Create a new project - var project = _api.CreateProject($"Test project - {DateTime.Now.ToStringInvariant()}", Language.CSharp); + var project = ApiClient.CreateProject($"Test project - {DateTime.Now.ToStringInvariant()}", Language.CSharp, TestOrganization); // Add Project Files - var addProjectFile = _api.AddProjectFile(project.Projects.First().ProjectId, file.Name, file.Code); + var addProjectFile = ApiClient.AddProjectFile(project.Projects.First().ProjectId, file.Name, file.Code); Assert.IsTrue(addProjectFile.Success); // Create compile - var compile = _api.CreateCompile(project.Projects.First().ProjectId); + var compile = ApiClient.CreateCompile(project.Projects.First().ProjectId); Assert.IsTrue(compile.Success); // Wait at max 30 seconds for project to compile @@ -361,23 +342,23 @@ private void RunLiveAlgorithm(BaseLiveAlgorithmSettings settings, ProjectFile fi Assert.IsTrue(compileCheck.State == CompileState.BuildSuccess); // Get a live node to launch the algorithm on - var nodes = _api.ReadNodes(_testOrganization); + var nodes = ApiClient.ReadNodes(TestOrganization); Assert.IsTrue(nodes.Success); var freeNode = nodes.LiveNodes.Where(x => x.Busy == false); Assert.IsNotEmpty(freeNode, "No free Live Nodes found"); // Create live default algorithm - var createLiveAlgorithm = _api.CreateLiveAlgorithm(project.Projects.First().ProjectId, compile.CompileId, freeNode.FirstOrDefault().Id, settings); + var createLiveAlgorithm = ApiClient.CreateLiveAlgorithm(project.Projects.First().ProjectId, compile.CompileId, freeNode.FirstOrDefault().Id, settings); Assert.IsTrue(createLiveAlgorithm.Success); - if (stopLiveAlgos) + if (StopLiveAlgos) { // Liquidate live algorithm; will also stop algorithm - var liquidateLive = _api.LiquidateLiveAlgorithm(project.Projects.First().ProjectId); + var liquidateLive = ApiClient.LiquidateLiveAlgorithm(project.Projects.First().ProjectId); Assert.IsTrue(liquidateLive.Success); // Delete the project - var deleteProject = _api.DeleteProject(project.Projects.First().ProjectId); + var deleteProject = ApiClient.DeleteProject(project.Projects.First().ProjectId); Assert.IsTrue(deleteProject.Success); } } @@ -396,7 +377,7 @@ private Compile WaitForCompilerResponse(int projectId, string compileId, int sec while (DateTime.Now < finish) { Thread.Sleep(1000); - compile = _api.ReadCompile(projectId, compileId); + compile = ApiClient.ReadCompile(projectId, compileId); if (compile.State == CompileState.BuildSuccess) break; } return compile; diff --git a/Tests/Api/NodeTests.cs b/Tests/Api/NodeTests.cs index 63497cb851f1..486c1140840e 100644 --- a/Tests/Api/NodeTests.cs +++ b/Tests/Api/NodeTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -17,8 +17,6 @@ using System.Linq; using NUnit.Framework; using QuantConnect.Api; -using QuantConnect.Configuration; - namespace QuantConnect.Tests.API { @@ -26,30 +24,9 @@ namespace QuantConnect.Tests.API /// Test class for all nodes/ endpoints: /// create, read, update, delete, stop /// - [TestFixture, Ignore("These tests require an account, token, organization ID and billing permissions")] - public class NodeTests + [TestFixture, Explicit("Requires configured api access, account with credit, and node creation permission")] + public class NodeTests : ApiTestBase { - private int _testAccount; - private string _testToken; - private string _testOrganization; - private string _dataFolder; - private Api.Api _api; - - /// - /// Setup for this text fixture, will create API connection and load in configuration - /// - [OneTimeSetUp] - public void Setup() - { - _testAccount = Config.GetInt("job-user-id", 1); - _testToken = Config.Get("api-access-token", "EnterTokenHere"); - _testOrganization = Config.Get("job-organization-id", "EnterOrgHere"); - _dataFolder = Config.Get("data-folder"); - - _api = new Api.Api(); - _api.Initialize(_testAccount, _testToken, _dataFolder); - } - /// /// Create a new backtest node with 2 CPU cores and 8GB of RAM check for successful /// creation in API response. @@ -59,7 +36,7 @@ public void CreateNewNode() { var sku = new SKU(2, 8, NodeType.Backtest); var nodeName = $"{DateTime.UtcNow.Minute}:{DateTime.UtcNow.Second}-Pinocho"; - var newNode = _api.CreateNode(nodeName, _testOrganization, sku); + var newNode = ApiClient.CreateNode(nodeName, TestOrganization, sku); Assert.IsTrue(newNode.Success); } @@ -70,7 +47,7 @@ public void CreateNewNode() [Test] public void ReadNode() { - var result = _api.ReadNodes(_testOrganization); + var result = ApiClient.ReadNodes(TestOrganization); Assert.IsTrue(result.Success); } @@ -90,14 +67,14 @@ public void CRUDNodes(int cores, int memory, NodeType target) var nodeName2 = $"{DateTime.UtcNow.Minute}:{DateTime.UtcNow.Second}-Monstro"; // First create a new node - var createdNode = _api.CreateNode(nodeName, _testOrganization, sku); + var createdNode = ApiClient.CreateNode(nodeName, TestOrganization, sku); //Check for the nodes existance using the helper function var foundNode = FindNodeByName(nodeName); Assert.IsNotNull(foundNode); //Update that node with a new name - var updateNodeRequest = _api.UpdateNode(foundNode.Id, nodeName2, _testOrganization); + var updateNodeRequest = ApiClient.UpdateNode(foundNode.Id, nodeName2, TestOrganization); Assert.IsTrue(updateNodeRequest.Success); //Read again and check for the new name (nodeName2) @@ -105,7 +82,7 @@ public void CRUDNodes(int cores, int memory, NodeType target) Assert.IsNotNull(foundNode); //Delete this node - var deleteNodeRequest = _api.DeleteNode(foundNode.Id, _testOrganization); + var deleteNodeRequest = ApiClient.DeleteNode(foundNode.Id, TestOrganization); Assert.IsTrue(deleteNodeRequest.Success); //Read again and ensure the node does not exist. @@ -123,7 +100,7 @@ public void CRUDNodes(int cores, int memory, NodeType target) public Node FindNodeByName(string name) { Node result = null; - var readNodeRequest = _api.ReadNodes(_testOrganization); + var readNodeRequest = ApiClient.ReadNodes(TestOrganization); var allNodes = readNodeRequest.BacktestNodes.Concat(readNodeRequest.LiveNodes).Concat(readNodeRequest.ResearchNodes); foreach (var Node in allNodes) @@ -169,7 +146,7 @@ public void SkuIsGeneratedCorrectly(int cores, int memory, NodeType target, stri public void ReadAndStop() { // Then read the nodes from the org - var readNodeRequest = _api.ReadNodes(_testOrganization); + var readNodeRequest = ApiClient.ReadNodes(TestOrganization); Assert.IsTrue(readNodeRequest.Success); //Iterate through all nodes and stop them if they are running @@ -179,7 +156,7 @@ public void ReadAndStop() // If it is busy then stop it if (Node.Busy) { - var stopNodeRequest = _api.StopNode(Node.Id, _testOrganization); + var stopNodeRequest = ApiClient.StopNode(Node.Id, TestOrganization); Assert.IsTrue(stopNodeRequest.Success); } } diff --git a/Tests/Api/OrganizationTests.cs b/Tests/Api/OrganizationTests.cs new file mode 100644 index 000000000000..f41a8a8bdbdd --- /dev/null +++ b/Tests/Api/OrganizationTests.cs @@ -0,0 +1,54 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using NUnit.Framework; + +namespace QuantConnect.Tests.API +{ + /// + /// Tests API account and organizations endpoints + /// + [TestFixture, Explicit("Requires configured api access")] + public class OrganizationTests : ApiTestBase + { + [Test] + public void ReadAccount() + { + var account = ApiClient.ReadAccount(); + + Assert.IsTrue(account.Success); + Assert.IsNotEmpty(account.OrganizationId); + Assert.IsNotNull(account.Card); + Assert.AreNotEqual(default(DateTime), account.Card.Expiration); + Assert.IsNotEmpty(account.Card.Brand); + Assert.AreNotEqual(0, account.Card.LastFourDigits); + } + + [Test] + public void ListOrganization() + { + var list = ApiClient.ListOrganizations(); + Assert.IsTrue(list.Count > 0); + } + + [Test] + public void ReadOrganization() + { + var organization = ApiClient.ReadOrganization(TestOrganization); + Assert.IsNotNull(organization.Credit); + } + } +} diff --git a/Tests/Api/ProjectTests.cs b/Tests/Api/ProjectTests.cs new file mode 100644 index 000000000000..6384fec51bf8 --- /dev/null +++ b/Tests/Api/ProjectTests.cs @@ -0,0 +1,284 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Web; +using NUnit.Framework; +using QuantConnect.Api; + +namespace QuantConnect.Tests.API +{ + /// + /// API Project endpoints, includes some Backtest endpoints testing as well + /// + [TestFixture, Explicit("Requires configured api access and available backtest node to run on")] + public class ProjectTests : ApiTestBase + { + /// + /// Test creating and deleting projects with the Api + /// + [Test] + public void Projects_CanBeCreatedAndDeleted_Successfully() + { + var name = "Test Project " + DateTime.Now.ToStringInvariant(); + + //Test create a new project successfully + var project = ApiClient.CreateProject(name, Language.CSharp, TestOrganization); + Assert.IsTrue(project.Success); + Assert.IsTrue(project.Projects.First().ProjectId > 0); + Assert.IsTrue(project.Projects.First().Name == name); + + // Delete the project + var deleteProject = ApiClient.DeleteProject(project.Projects.First().ProjectId); + Assert.IsTrue(deleteProject.Success); + + // Make sure the project is really deleted + var projectList = ApiClient.ListProjects(); + Assert.IsFalse(projectList.Projects.Any(p => p.ProjectId == project.Projects.First().ProjectId)); + } + + /// + /// Test updating the files associated with a project + /// + [Test] + public void CRUD_ProjectFiles_Successfully() + { + var fakeFile = new ProjectFile + { + Name = "Hello.cs", + Code = HttpUtility.HtmlEncode("Hello, world!") + }; + + var realFile = new ProjectFile + { + Name = "main.cs", + Code = HttpUtility.HtmlEncode(File.ReadAllText("../../../Algorithm.CSharp/BasicTemplateAlgorithm.cs")) + }; + + var secondRealFile = new ProjectFile() + { + Name = "algorithm.cs", + Code = HttpUtility.HtmlEncode(File.ReadAllText("../../../Algorithm.CSharp/BubbleAlgorithm.cs")) + }; + + // Create a new project + var project = ApiClient.CreateProject($"Test project - {DateTime.Now.ToStringInvariant()}", Language.CSharp, TestOrganization); + Assert.IsTrue(project.Success); + Assert.IsTrue(project.Projects.First().ProjectId > 0); + + // Add random file + var randomAdd = ApiClient.AddProjectFile(project.Projects.First().ProjectId, fakeFile.Name, fakeFile.Code); + Assert.IsTrue(randomAdd.Success); + Assert.IsTrue(randomAdd.Files.First().Code == fakeFile.Code); + Assert.IsTrue(randomAdd.Files.First().Name == fakeFile.Name); + // Update names of file + var updatedName = ApiClient.UpdateProjectFileName(project.Projects.First().ProjectId, randomAdd.Files.First().Name, realFile.Name); + Assert.IsTrue(updatedName.Success); + + // Replace content of file + var updateContents = ApiClient.UpdateProjectFileContent(project.Projects.First().ProjectId, realFile.Name, realFile.Code); + Assert.IsTrue(updateContents.Success); + + // Read single file + var readFile = ApiClient.ReadProjectFile(project.Projects.First().ProjectId, realFile.Name); + Assert.IsTrue(readFile.Success); + Assert.IsTrue(readFile.Files.First().Code == realFile.Code); + Assert.IsTrue(readFile.Files.First().Name == realFile.Name); + + // Add a second file + var secondFile = ApiClient.AddProjectFile(project.Projects.First().ProjectId, secondRealFile.Name, secondRealFile.Code); + Assert.IsTrue(secondFile.Success); + Assert.IsTrue(secondFile.Files.First().Code == secondRealFile.Code); + Assert.IsTrue(secondFile.Files.First().Name == secondRealFile.Name); + + // Read multiple files + var readFiles = ApiClient.ReadProjectFiles(project.Projects.First().ProjectId); + Assert.IsTrue(readFiles.Success); + Assert.IsTrue(readFiles.Files.Count == 4); // 2 Added + 2 Automatic (Research.ipynb & Main.cs) + + // Delete the second file + var deleteFile = ApiClient.DeleteProjectFile(project.Projects.First().ProjectId, secondRealFile.Name); + Assert.IsTrue(deleteFile.Success); + + // Read files + var readFilesAgain = ApiClient.ReadProjectFiles(project.Projects.First().ProjectId); + Assert.IsTrue(readFilesAgain.Success); + Assert.IsTrue(readFilesAgain.Files.Count == 3); + Assert.IsTrue(readFilesAgain.Files.Any(x => x.Name == realFile.Name)); + + // Delete the project + var deleteProject = ApiClient.DeleteProject(project.Projects.First().ProjectId); + Assert.IsTrue(deleteProject.Success); + } + + /// + /// Test creating, compiling and backtesting a C# project via the Api + /// + [Test] + public void CSharpProject_CreatedCompiledAndBacktested_Successully() + { + var language = Language.CSharp; + var code = File.ReadAllText("../../../Algorithm.CSharp/BasicTemplateAlgorithm.cs"); + var algorithmName = "main.cs"; + var projectName = $"{DateTime.UtcNow.ToStringInvariant("u")} Test {TestAccount} Lang {language}"; + + Perform_CreateCompileBackTest_Tests(projectName, language, algorithmName, code); + } + + /// + /// Test creating, compiling and backtesting a Python project via the Api + /// + [Test] + public void PythonProject_CreatedCompiledAndBacktested_Successully() + { + var language = Language.Python; + var code = File.ReadAllText("../../../Algorithm.Python/BasicTemplateAlgorithm.py"); + var algorithmName = "main.py"; + + var projectName = $"{DateTime.UtcNow.ToStringInvariant("u")} Test {TestAccount} Lang {language}"; + + Perform_CreateCompileBackTest_Tests(projectName, language, algorithmName, code); + } + + private void Perform_CreateCompileBackTest_Tests(string projectName, Language language, string algorithmName, string code) + { + //Test create a new project successfully + var project = ApiClient.CreateProject(projectName, language, TestOrganization); + Assert.IsTrue(project.Success); + Assert.IsTrue(project.Projects.First().ProjectId > 0); + Assert.IsTrue(project.Projects.First().Name == projectName); + + // Make sure the project just created is now present + var projects = ApiClient.ListProjects(); + Assert.IsTrue(projects.Success); + Assert.IsTrue(projects.Projects.Any(p => p.ProjectId == project.Projects.First().ProjectId)); + + // Test read back the project we just created + var readProject = ApiClient.ReadProject(project.Projects.First().ProjectId); + Assert.IsTrue(readProject.Success); + Assert.IsTrue(readProject.Projects.First().Name == projectName); + + // Test set a project file for the project + var file = new ProjectFile { Name = algorithmName, Code = code }; + var addProjectFile = ApiClient.AddProjectFile(project.Projects.First().ProjectId, file.Name, file.Code); + Assert.IsTrue(addProjectFile.Success); + + // Download the project again to validate its got the new file + var verifyRead = ApiClient.ReadProject(project.Projects.First().ProjectId); + Assert.IsTrue(verifyRead.Success); + + // Compile the project we've created + var compileCreate = ApiClient.CreateCompile(project.Projects.First().ProjectId); + Assert.IsTrue(compileCreate.Success); + Assert.IsTrue(compileCreate.State == CompileState.InQueue); + + // Read out the compile + var compileSuccess = WaitForCompilerResponse(project.Projects.First().ProjectId, compileCreate.CompileId); + Assert.IsTrue(compileSuccess.Success); + Assert.IsTrue(compileSuccess.State == CompileState.BuildSuccess); + + // Update the file, create a build error, test we get build error + file.Code += "[Jibberish at end of the file to cause a build error]"; + ApiClient.UpdateProjectFileContent(project.Projects.First().ProjectId, file.Name, file.Code); + var compileError = ApiClient.CreateCompile(project.Projects.First().ProjectId); + compileError = WaitForCompilerResponse(project.Projects.First().ProjectId, compileError.CompileId); + Assert.IsTrue(compileError.Success); // Successfully processed rest request. + Assert.IsTrue(compileError.State == CompileState.BuildError); //Resulting in build fail. + + // Using our successful compile; launch a backtest! + var backtestName = $"{DateTime.Now.ToStringInvariant("u")} API Backtest"; + var backtest = ApiClient.CreateBacktest(project.Projects.First().ProjectId, compileSuccess.CompileId, backtestName); + Assert.IsTrue(backtest.Success); + + // Now read the backtest and wait for it to complete + var backtestRead = WaitForBacktestCompletion(project.Projects.First().ProjectId, backtest.BacktestId); + Assert.IsTrue(backtestRead.Success); + Assert.IsTrue(backtestRead.Progress == 1); + Assert.IsTrue(backtestRead.Name == backtestName); + Assert.IsTrue(backtestRead.Statistics["Total Trades"] == "1"); + Assert.IsTrue(backtestRead.Charts["Benchmark"].Series.Count > 0); + + // Verify we have the backtest in our project + var listBacktests = ApiClient.ListBacktests(project.Projects.First().ProjectId); + Assert.IsTrue(listBacktests.Success); + Assert.IsTrue(listBacktests.Backtests.Count >= 1); + Assert.IsTrue(listBacktests.Backtests[0].Name == backtestName); + + // Update the backtest name and test its been updated + backtestName += "-Amendment"; + var renameBacktest = ApiClient.UpdateBacktest(project.Projects.First().ProjectId, backtest.BacktestId, backtestName); + Assert.IsTrue(renameBacktest.Success); + backtestRead = ApiClient.ReadBacktest(project.Projects.First().ProjectId, backtest.BacktestId); + Assert.IsTrue(backtestRead.Name == backtestName); + + //Update the note and make sure its been updated: + var newNote = DateTime.Now.ToStringInvariant("u"); + var noteBacktest = ApiClient.UpdateBacktest(project.Projects.First().ProjectId, backtest.BacktestId, note: newNote); + Assert.IsTrue(noteBacktest.Success); + backtestRead = ApiClient.ReadBacktest(project.Projects.First().ProjectId, backtest.BacktestId); + Assert.IsTrue(backtestRead.Note == newNote); + + // Delete the backtest we just created + var deleteBacktest = ApiClient.DeleteBacktest(project.Projects.First().ProjectId, backtest.BacktestId); + Assert.IsTrue(deleteBacktest.Success); + + // Test delete the project we just created + var deleteProject = ApiClient.DeleteProject(project.Projects.First().ProjectId); + Assert.IsTrue(deleteProject.Success); + } + + /// + /// Wait for the compiler to respond to a specified compile request + /// + /// Id of the project + /// Id of the compilation of the project + /// + private Compile WaitForCompilerResponse(int projectId, string compileId) + { + var compile = new Compile(); + var finish = DateTime.Now.AddSeconds(60); + while (DateTime.Now < finish) + { + compile = ApiClient.ReadCompile(projectId, compileId); + if (compile.State == CompileState.BuildSuccess) break; + Thread.Sleep(1000); + } + return compile; + } + + /// + /// Wait for the backtest to complete + /// + /// Project id to scan + /// Backtest id previously started + /// Completed backtest object + private Backtest WaitForBacktestCompletion(int projectId, string backtestId) + { + var result = new Backtest(); + var finish = DateTime.Now.AddSeconds(60); + while (DateTime.Now < finish) + { + result = ApiClient.ReadBacktest(projectId, backtestId); + if (result.Progress == 1) break; + if (!result.Success) break; + Thread.Sleep(1000); + } + return result; + } + } +} diff --git a/Tests/AssemblyInitialize.cs b/Tests/AssemblyInitialize.cs index 3171df134b1c..3391d545bb8d 100644 --- a/Tests/AssemblyInitialize.cs +++ b/Tests/AssemblyInitialize.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -34,6 +34,7 @@ public class AssemblyInitialize public void InitializeTestEnvironment() { AdjustCurrentDirectory(); + TestGlobals.Initialize(); } public static void AdjustCurrentDirectory() diff --git a/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageAdditionalTests.cs b/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageAdditionalTests.cs index b805eded79e2..be5e5f2f3222 100644 --- a/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageAdditionalTests.cs +++ b/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageAdditionalTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -184,7 +184,7 @@ private InteractiveBrokersBrokerage GetBrokerage() new OrderProvider(_orders), securityProvider, new AggregationManager(), - new LocalDiskMapFileProvider()); + TestGlobals.MapFileProvider); brokerage.Connect(); return brokerage; diff --git a/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageDataQueueHandlerTest.cs b/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageDataQueueHandlerTest.cs index 54802b3f5802..2c4b19ec9b96 100644 --- a/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageDataQueueHandlerTest.cs +++ b/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageDataQueueHandlerTest.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -35,7 +35,7 @@ public class InteractiveBrokersBrokerageDataQueueHandlerTest [Test] public void GetsTickData() { - using (var ib = new InteractiveBrokersBrokerage(new QCAlgorithm(), new OrderProvider(), new SecurityProvider(), new AggregationManager(), new LocalDiskMapFileProvider())) + using (var ib = new InteractiveBrokersBrokerage(new QCAlgorithm(), new OrderProvider(), new SecurityProvider(), new AggregationManager(), TestGlobals.MapFileProvider)) { ib.Connect(); var gotUsdData = false; @@ -65,7 +65,7 @@ public void GetsTickData() [Test] public void GetsTickDataAfterDisconnectionConnectionCycle() { - using (var ib = new InteractiveBrokersBrokerage(new QCAlgorithm(), new OrderProvider(), new SecurityProvider(), new AggregationManager(), new LocalDiskMapFileProvider())) + using (var ib = new InteractiveBrokersBrokerage(new QCAlgorithm(), new OrderProvider(), new SecurityProvider(), new AggregationManager(), TestGlobals.MapFileProvider)) { ib.Connect(); var cancelationToken = new CancellationTokenSource(); diff --git a/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageTests.cs b/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageTests.cs index 14d0c5fbf54b..aba39060749e 100644 --- a/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageTests.cs +++ b/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -72,7 +72,7 @@ public void InitializeBrokerage() new OrderProvider(_orders), securityProvider, new AggregationManager(), - new LocalDiskMapFileProvider()); + TestGlobals.MapFileProvider); _interactiveBrokersBrokerage.Connect(); } @@ -756,4 +756,4 @@ private static Order AssertOrderOpened(bool orderFilled, InteractiveBrokersBroke return null; } } -} \ No newline at end of file +} diff --git a/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersFuturesTests.cs b/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersFuturesTests.cs index e393214dbcae..455b2beb3120 100644 --- a/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersFuturesTests.cs +++ b/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersFuturesTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -32,9 +32,9 @@ public class InteractiveBrokersFuturesTests [Test] public void CreatesExpectedFuturesContracts() { - var symbolMapper = new InteractiveBrokersSymbolMapper(new LocalDiskMapFileProvider()); + var symbolMapper = new InteractiveBrokersSymbolMapper(TestGlobals.MapFileProvider); - using (var ib = new InteractiveBrokersBrokerage(new QCAlgorithm(), new OrderProvider(), new SecurityProvider(), new AggregationManager(), new LocalDiskMapFileProvider())) + using (var ib = new InteractiveBrokersBrokerage(new QCAlgorithm(), new OrderProvider(), new SecurityProvider(), new AggregationManager(), TestGlobals.MapFileProvider)) { ib.Connect(); Assert.IsTrue(ib.IsConnected); diff --git a/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersOrderTests.cs b/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersOrderTests.cs index a4e92f64bbb3..a63b4903ca53 100644 --- a/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersOrderTests.cs +++ b/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersOrderTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -101,7 +101,7 @@ protected override decimal GetAskPrice(Symbol symbol) protected override IBrokerage CreateBrokerage(IOrderProvider orderProvider, ISecurityProvider securityProvider) { - return new InteractiveBrokersBrokerage(new QCAlgorithm(), orderProvider, securityProvider, new AggregationManager(), new LocalDiskMapFileProvider()); + return new InteractiveBrokersBrokerage(new QCAlgorithm(), orderProvider, securityProvider, new AggregationManager(), TestGlobals.MapFileProvider); } protected override void DisposeBrokerage(IBrokerage brokerage) diff --git a/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersSymbolMapperTests.cs b/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersSymbolMapperTests.cs index 025a8007a13b..a9eeb3c7eb5f 100644 --- a/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersSymbolMapperTests.cs +++ b/Tests/Brokerages/InteractiveBrokers/InteractiveBrokersSymbolMapperTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -20,7 +20,6 @@ using QuantConnect.Data.Auxiliary; using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Securities; -using QuantConnect.Securities.FutureOption; using IB = QuantConnect.Brokerages.InteractiveBrokers.Client; namespace QuantConnect.Tests.Brokerages.InteractiveBrokers @@ -31,7 +30,7 @@ public class InteractiveBrokersSymbolMapperTests [Test] public void ReturnsCorrectLeanSymbol() { - var mapper = new InteractiveBrokersSymbolMapper(new LocalDiskMapFileProvider()); + var mapper = new InteractiveBrokersSymbolMapper(TestGlobals.MapFileProvider); var symbol = mapper.GetLeanSymbol("EURUSD", SecurityType.Forex, Market.FXCM); Assert.AreEqual("EURUSD", symbol.Value); @@ -52,7 +51,7 @@ public void ReturnsCorrectLeanSymbol() [Test] public void ReturnsCorrectBrokerageSymbol() { - var mapper = new InteractiveBrokersSymbolMapper(new LocalDiskMapFileProvider()); + var mapper = new InteractiveBrokersSymbolMapper(TestGlobals.MapFileProvider); var symbol = Symbol.Create("EURUSD", SecurityType.Forex, Market.FXCM); var brokerageSymbol = mapper.GetBrokerageSymbol(symbol); @@ -180,8 +179,8 @@ public void MalformedContractSymbolCreatesFutureContract(string symbol) [TestCase(2021, 3, 25)] public void FuturesOptionsWithUnderlyingContractMonthMappedByRuleResolvesUnderlyingGetLeanSymbol(int year, int month, int day) { - var futuresChainProvider = new BacktestingFutureChainProvider(); - var mapper = new InteractiveBrokersSymbolMapper(new LocalDiskMapFileProvider()); + var futuresChainProvider = new BacktestingFutureChainProvider(TestGlobals.DataProvider); + var mapper = new InteractiveBrokersSymbolMapper(TestGlobals.MapFileProvider); var expectedUnderlyingSymbol = Symbol.CreateFuture("GC", Market.COMEX, new DateTime(2021, 4, 28)); var futureOption = mapper.GetLeanSymbol("OG", SecurityType.FutureOption, Market.COMEX, new DateTime(year, month, day)); diff --git a/Tests/Brokerages/Paper/PaperBrokerageTests.cs b/Tests/Brokerages/Paper/PaperBrokerageTests.cs index 644df8df58d1..68199b557302 100644 --- a/Tests/Brokerages/Paper/PaperBrokerageTests.cs +++ b/Tests/Brokerages/Paper/PaperBrokerageTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * diff --git a/Tests/Common/Data/Auxiliary/FactorFileTests.cs b/Tests/Common/Data/Auxiliary/FactorFileTests.cs index 095ef92329fe..5537b3dd4624 100644 --- a/Tests/Common/Data/Auxiliary/FactorFileTests.cs +++ b/Tests/Common/Data/Auxiliary/FactorFileTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -22,6 +22,7 @@ using QuantConnect.Data; using QuantConnect.Data.Auxiliary; using QuantConnect.Data.Market; +using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Logging; using QuantConnect.Securities; using QuantConnect.Util; @@ -34,10 +35,13 @@ public class FactorFileTests [Test] public void ReadsFactorFileWithoutInfValues() { - var factorFile = FactorFile.Read("AAPL", "usa"); + var PermTick = "AAPL"; + var Market = "usa"; + var _symbol = new Symbol(SecurityIdentifier.GenerateEquity(PermTick, Market), PermTick); - Assert.AreEqual(29, factorFile.SortedFactorFileData.Count); + var factorFile = TestGlobals.FactorFileProvider.Get(_symbol); + Assert.AreEqual(29, factorFile.SortedFactorFileData.Count); Assert.AreEqual(new DateTime(1998, 01, 01), factorFile.FactorFileMinimumDate.Value); } @@ -412,7 +416,7 @@ private static FactorFile GetTestFactorFile(string symbol, DateTime reference) private static FactorFile GetFactorFile(string permtick) { - return FactorFile.Read(permtick, QuantConnect.Market.USA); + return TestGlobals.FactorFileProvider.Get(permtick); } private static FactorFile GetFactorFile_LODE20191127() diff --git a/Tests/Common/Data/Auxiliary/LocalDiskFactorFileProviderTests.cs b/Tests/Common/Data/Auxiliary/LocalDiskFactorFileProviderTests.cs index 0b603b6bbc67..91885173e6f5 100644 --- a/Tests/Common/Data/Auxiliary/LocalDiskFactorFileProviderTests.cs +++ b/Tests/Common/Data/Auxiliary/LocalDiskFactorFileProviderTests.cs @@ -18,42 +18,46 @@ using System.Globalization; using System.IO; using NUnit.Framework; -using QuantConnect.Data.Auxiliary; +using QuantConnect.Interfaces; namespace QuantConnect.Tests.Common.Data.Auxiliary { [TestFixture] public class LocalDiskFactorFileProviderTests { + internal IFactorFileProvider FactorFileProvider; + + [OneTimeSetUp] + public virtual void Setup() + { + FactorFileProvider = TestGlobals.FactorFileProvider; + } + [Test] public void RetrievesFromDisk() { - var provider = new LocalDiskFactorFileProvider(); - var factorFile = provider.Get(Symbols.SPY); + var factorFile = FactorFileProvider.Get(Symbols.SPY); Assert.IsNotNull(factorFile); } [Test] public void CachesValueAndReturnsSameReference() { - var provider = new LocalDiskFactorFileProvider(); - var factorFile1 = provider.Get(Symbols.SPY); - var factorFile2 = provider.Get(Symbols.SPY); + var factorFile1 = FactorFileProvider.Get(Symbols.SPY); + var factorFile2 = FactorFileProvider.Get(Symbols.SPY); Assert.IsTrue(ReferenceEquals(factorFile1, factorFile2)); } [Test] public void ReturnsNullForNotFound() { - var provider = new LocalDiskFactorFileProvider(); - var factorFile = provider.Get(Symbol.Create("not-a-ticker", SecurityType.Equity, QuantConnect.Market.USA)); + var factorFile = FactorFileProvider.Get(Symbol.Create("not-a-ticker", SecurityType.Equity, QuantConnect.Market.USA)); Assert.IsNull(factorFile); } [Test, Ignore("This test is meant to be run manually")] public void FindsFactorFilesWithErrors() { - var provider = new LocalDiskFactorFileProvider(); var factorFileFolder = Path.Combine(Globals.DataFolder, "equity", QuantConnect.Market.USA, "factor_files"); foreach (var fileName in Directory.EnumerateFiles(factorFileFolder)) @@ -63,7 +67,7 @@ public void FindsFactorFilesWithErrors() try { - provider.Get(symbol); + FactorFileProvider.Get(symbol); } catch (Exception exception) { @@ -72,4 +76,4 @@ public void FindsFactorFilesWithErrors() } } } -} \ No newline at end of file +} diff --git a/Tests/Common/Data/Auxiliary/LocalDiskMapFileProviderTests.cs b/Tests/Common/Data/Auxiliary/LocalDiskMapFileProviderTests.cs index 3d2e5ad5468c..98f1fae11f86 100644 --- a/Tests/Common/Data/Auxiliary/LocalDiskMapFileProviderTests.cs +++ b/Tests/Common/Data/Auxiliary/LocalDiskMapFileProviderTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * diff --git a/Tests/Common/Data/Auxiliary/LocalZipFactorFileProviderTests.cs b/Tests/Common/Data/Auxiliary/LocalZipFactorFileProviderTests.cs new file mode 100644 index 000000000000..d45748c5b9ea --- /dev/null +++ b/Tests/Common/Data/Auxiliary/LocalZipFactorFileProviderTests.cs @@ -0,0 +1,108 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + + +using System; +using System.IO; +using NUnit.Framework; +using System.Threading; +using QuantConnect.Util; +using QuantConnect.Data.Auxiliary; +using QuantConnect.Lean.Engine.DataFeeds; + +namespace QuantConnect.Tests.Common.Data.Auxiliary +{ + [TestFixture] + public class LocalZipFactorFileProviderTests : LocalDiskFactorFileProviderTests + { + private string _zipFilePath; + + [OneTimeSetUp] + public override void Setup() + { + // Take our repo included factor files and zip them up for these tests + var date = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date.AddDays(-1); + var path = Path.Combine(Globals.DataFolder, $"equity/usa/factor_files/"); + var tmp = "./tmp.zip"; + + _zipFilePath = Path.Combine(Globals.DataFolder, $"equity/usa/factor_files/factor_files_{date:yyyyMMdd}.zip"); + + // Have to compress to tmp file or else it doesn't finish reading all the files in dir + QuantConnect.Compression.ZipDirectory(path, tmp); + File.Move(tmp, _zipFilePath, true); + + FactorFileProvider = new LocalZipFactorFileProvider(); + FactorFileProvider.Initialize(TestGlobals.MapFileProvider, TestGlobals.DataProvider); + } + + [OneTimeTearDown] + public void TearDown() + { + if (File.Exists(_zipFilePath)) + { + File.Delete(_zipFilePath); + } + } + + [Test] + public void CacheIsCleared() + { + var fileProviderTest = new LocalZipFactorFileProviderTest(); + var dataProviderTest = new DefaultDataProviderTest(); + + fileProviderTest.Initialize(TestGlobals.MapFileProvider, dataProviderTest); + + fileProviderTest.Get(Symbols.AAPL); + Assert.AreEqual(1, dataProviderTest.FetchCount); + Thread.Sleep(50); + fileProviderTest.Get(Symbols.AAPL); + Assert.AreEqual(1, dataProviderTest.FetchCount); + + Thread.Sleep(1000); + + fileProviderTest.Get(Symbols.AAPL); + Assert.AreEqual(2, dataProviderTest.FetchCount); + + fileProviderTest.Enabled = false; + dataProviderTest.DisposeSafely(); + } + + private class LocalZipFactorFileProviderTest : LocalZipFactorFileProvider + { + public bool Enabled = true; + protected override TimeSpan CacheRefreshPeriod => TimeSpan.FromMilliseconds(500); + + protected override void StartExpirationTask() + { + if (Enabled) + { + base.StartExpirationTask(); + } + } + } + + private class DefaultDataProviderTest : DefaultDataProvider + { + public int FetchCount { get; set; } + + public override Stream Fetch(string key) + { + FetchCount++; + return base.Fetch(key); + } + } + } +} diff --git a/Tests/Common/Data/Auxiliary/LocalZipMapFileProviderTests.cs b/Tests/Common/Data/Auxiliary/LocalZipMapFileProviderTests.cs new file mode 100644 index 000000000000..6d871bf52350 --- /dev/null +++ b/Tests/Common/Data/Auxiliary/LocalZipMapFileProviderTests.cs @@ -0,0 +1,117 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.IO; +using System.Threading; +using NUnit.Framework; +using QuantConnect.Data.Auxiliary; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Util; + +namespace QuantConnect.Tests.Common.Data.Auxiliary +{ + [TestFixture] + public class LocalZipMapFileProviderTests + { + private string _zipFilePath; + + [OneTimeSetUp] + public void Setup() + { + // Take our repo included map files and zip them up for these tests + var date = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date.AddDays(-1); + var path = Path.Combine(Globals.DataFolder, $"equity/usa/map_files/"); + var tmp = "./tmp.zip"; + + _zipFilePath = Path.Combine(Globals.DataFolder, $"equity/usa/map_files/map_files_{date:yyyyMMdd}.zip"); + + // Have to compress to tmp file or else it doesn't finish reading all the files in dir + QuantConnect.Compression.ZipDirectory(path, tmp); + File.Move(tmp, _zipFilePath, true); + } + + [OneTimeTearDown] + public void TearDown() + { + if (File.Exists(_zipFilePath)) + { + File.Delete(_zipFilePath); + } + } + + [Test] + public void Retrieves() + { + var fileProviderTest = new LocalZipMapFileProviderTest(); + var dataProviderTest = new DefaultDataProviderTest(); + fileProviderTest.Initialize(dataProviderTest); + + var mapFileResolver = fileProviderTest.Get(QuantConnect.Market.USA); + + fileProviderTest.Enabled = false; + dataProviderTest.DisposeSafely(); + + Assert.IsNotEmpty(mapFileResolver); + } + + [Test] + public void CacheIsCleared() + { + var fileProviderTest = new LocalZipMapFileProviderTest(); + var dataProviderTest = new DefaultDataProviderTest(); + fileProviderTest.Initialize(dataProviderTest); + + fileProviderTest.Get(QuantConnect.Market.USA); + Assert.AreEqual(1, dataProviderTest.FetchCount); + Thread.Sleep(50); + fileProviderTest.Get(QuantConnect.Market.USA); + Assert.AreEqual(1, dataProviderTest.FetchCount); + + Thread.Sleep(1000); + + fileProviderTest.Get(QuantConnect.Market.USA); + Assert.AreEqual(2, dataProviderTest.FetchCount); + + fileProviderTest.Enabled = false; + dataProviderTest.DisposeSafely(); + } + + private class LocalZipMapFileProviderTest : LocalZipMapFileProvider + { + public bool Enabled = true; + protected override TimeSpan CacheRefreshPeriod => TimeSpan.FromMilliseconds(500); + + protected override void StartExpirationTask() + { + if (Enabled) + { + base.StartExpirationTask(); + } + } + } + + private class DefaultDataProviderTest : DefaultDataProvider + { + public int FetchCount { get; set; } + + public override Stream Fetch(string key) + { + FetchCount++; + return base.Fetch(key); + } + } + } +} diff --git a/Tests/Common/Data/Custom/EstimizeTests.cs b/Tests/Common/Data/Custom/EstimizeTests.cs index ab2f289a7349..d88185351293 100644 --- a/Tests/Common/Data/Custom/EstimizeTests.cs +++ b/Tests/Common/Data/Custom/EstimizeTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -150,7 +150,7 @@ public void EstimizeReleaseReaderTest() var data = new EstimizeRelease(); var date = new DateTime(2019, 6, 10); var source = data.GetSource(config, date, false); - var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data); + var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data, TestGlobals.DataProvider); var rows = factory.Read(source).ToList(); @@ -234,7 +234,7 @@ public void EstimizeEstimateReaderTest() var data = new EstimizeEstimate(); var date = new DateTime(2019, 6, 10); var source = data.GetSource(config, date, false); - var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data); + var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data, TestGlobals.DataProvider); var rows = factory.Read(source).ToList(); @@ -261,7 +261,7 @@ public void EstimizeConsensusReaderTest() var data = new EstimizeConsensus(); var date = new DateTime(2019, 6, 10); var source = data.GetSource(config, date, false); - var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data); + var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data, TestGlobals.DataProvider); var rows = factory.Read(source).ToList(); @@ -390,4 +390,4 @@ public void NormalizesDefunctTickerValues(string ticker, string expectedTicker, Assert.AreEqual(expectedTicker, actualTicker); } } -} \ No newline at end of file +} diff --git a/Tests/Common/Data/Custom/QuandlTests.cs b/Tests/Common/Data/Custom/QuandlTests.cs index f9a3acd49be5..410432af8ee1 100644 --- a/Tests/Common/Data/Custom/QuandlTests.cs +++ b/Tests/Common/Data/Custom/QuandlTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -41,7 +41,7 @@ public void QuandlDownloadDoesNotThrow() var config = new SubscriptionDataConfig(typeof(HistoryAlgorithm.QuandlFuture), Symbol.Create(ticker, SecurityType.Base, QuantConnect.Market.USA), Resolution.Daily, DateTimeZone.Utc, DateTimeZone.Utc, false, false, false, true); var source = data.GetSource(config, date, false); var dataCacheProvider = new SingleEntryDataCacheProvider(new DefaultDataProvider()); - var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data); + var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data, TestGlobals.DataProvider); var rows = factory.Read(source).ToList(); diff --git a/Tests/Common/Data/Custom/QuiverTests.cs b/Tests/Common/Data/Custom/QuiverTests.cs index 1bfd3e5ac6b7..6fa4016f80ff 100644 --- a/Tests/Common/Data/Custom/QuiverTests.cs +++ b/Tests/Common/Data/Custom/QuiverTests.cs @@ -136,7 +136,7 @@ public void QuiverWikipediaReaderTest() var data = new QuiverWikipedia(); var date = new DateTime(2019, 6, 10); var source = data.GetSource(config, date, false); - var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data); + var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data, TestGlobals.DataProvider); var rows = factory.Read(source).ToList(); diff --git a/Tests/Common/Securities/BrokerageModelSecurityInitializerTests.cs b/Tests/Common/Securities/BrokerageModelSecurityInitializerTests.cs index d40a7b52c3be..530fa67cf12a 100644 --- a/Tests/Common/Securities/BrokerageModelSecurityInitializerTests.cs +++ b/Tests/Common/Securities/BrokerageModelSecurityInitializerTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -65,14 +65,15 @@ public void Setup() { _algo = new QCAlgorithm(); var historyProvider = new SubscriptionDataReaderHistoryProvider(); + historyProvider.Initialize( new HistoryProviderInitializeParameters( null, null, - new DefaultDataProvider(), + TestGlobals.DataProvider, new SingleEntryDataCacheProvider(new DefaultDataProvider()), - new LocalDiskMapFileProvider(), - new LocalDiskFactorFileProvider(), + TestGlobals.MapFileProvider, + TestGlobals.FactorFileProvider, null, true, new DataPermissionManager() diff --git a/Tests/Common/Securities/Options/OptionChainProviderTests.cs b/Tests/Common/Securities/Options/OptionChainProviderTests.cs index dd3cd4c982af..792d3486b668 100644 --- a/Tests/Common/Securities/Options/OptionChainProviderTests.cs +++ b/Tests/Common/Securities/Options/OptionChainProviderTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -32,7 +32,7 @@ public class OptionChainProviderTests [Test] public void BacktestingOptionChainProviderLoadsEquityOptionChain() { - var provider = new BacktestingOptionChainProvider(); + var provider = new BacktestingOptionChainProvider(TestGlobals.DataProvider); var twxOptionChain = provider.GetOptionContractList(Symbol.Create("TWX", SecurityType.Equity, Market.USA), new DateTime(2014, 6, 5)) .ToList(); @@ -44,7 +44,7 @@ public void BacktestingOptionChainProviderLoadsEquityOptionChain() [Test] public void BacktestingOptionChainProviderLoadsFutureOptionChain() { - var provider = new BacktestingOptionChainProvider(); + var provider = new BacktestingOptionChainProvider(TestGlobals.DataProvider); var esOptionChain = provider.GetOptionContractList( Symbol.CreateFuture( QuantConnect.Securities.Futures.Indices.SP500EMini, @@ -179,7 +179,7 @@ public void BacktestingOptionChainProviderReturnsMultipleContractsForZipFileCont strike, expiry); - var provider = new BacktestingOptionChainProvider(); + var provider = new BacktestingOptionChainProvider(TestGlobals.DataProvider); var contracts = provider.GetOptionContractList(underlying, new DateTime(2020, 1, 5)) .ToHashSet(); diff --git a/Tests/Common/Securities/SecurityIdentifierTests.cs b/Tests/Common/Securities/SecurityIdentifierTests.cs index 5d5ce7bb42d8..f828f6be3938 100644 --- a/Tests/Common/Securities/SecurityIdentifierTests.cs +++ b/Tests/Common/Securities/SecurityIdentifierTests.cs @@ -360,7 +360,7 @@ public void ThrowsOnInvalidSymbolCharacters(string input) public void GenerateEquityWithTickerUsingMapFile() { var expectedFirstDate = new DateTime(1998, 1, 2); - var sid = SecurityIdentifier.GenerateEquity("TWX", Market.USA, mapSymbol: true, mapFileProvider: new LocalDiskMapFileProvider()); + var sid = SecurityIdentifier.GenerateEquity("TWX", Market.USA, mapSymbol: true, mapFileProvider: TestGlobals.MapFileProvider); Assert.AreEqual(sid.Date, expectedFirstDate); Assert.AreEqual(sid.Symbol, "AOL"); @@ -535,7 +535,7 @@ public void HighPrecisionNumberThrows(double strike) [Test, Ignore("Requires complete option data to validate chain")] public void ValidateAAPLOptionChainSecurityIdentifiers() { - var chainProvider = new BacktestingOptionChainProvider(); + var chainProvider = new BacktestingOptionChainProvider(TestGlobals.DataProvider); var aapl = Symbol.Create("AAPL", SecurityType.Equity, Market.USA); var chains = new HashSet(); var expectedChains = File.ReadAllLines("TestData/aapl_chain.csv") diff --git a/Tests/Common/Util/FactorFileGeneratorTests.cs b/Tests/Common/Util/FactorFileGeneratorTests.cs index d9d80b84b95c..0b72c71a3251 100644 --- a/Tests/Common/Util/FactorFileGeneratorTests.cs +++ b/Tests/Common/Util/FactorFileGeneratorTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -19,6 +19,7 @@ using NUnit.Framework; using QuantConnect.Configuration; using QuantConnect.Data.Auxiliary; +using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.ToolBox; using QuantConnect.ToolBox.YahooDownloader; using QuantConnect.Util; @@ -78,7 +79,7 @@ public void FactorFiles_CanBeGenerated_Accurately() throw new ArgumentException("This test requires an already calculated factor file." + "Try using one of the pre-existing factor files "); - var originalFactorFileInstance = FactorFile.Read(PermTick, Market); + var originalFactorFileInstance = TestGlobals.FactorFileProvider.Get(_symbol); // we limit events to the penultimate time in our factor file (last one is 2050) var lastValidRow = originalFactorFileInstance.SortedFactorFileData.Reverse().Skip(1).First(); @@ -114,4 +115,4 @@ public void FactorFiles_CanBeGenerated_Accurately() } } } -} \ No newline at end of file +} diff --git a/Tests/Compression/CompressionTests.cs b/Tests/Compression/CompressionTests.cs index 6144e1f66de6..6735c5090a45 100644 --- a/Tests/Compression/CompressionTests.cs +++ b/Tests/Compression/CompressionTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * diff --git a/Tests/Engine/AlgorithmManagerTests.cs b/Tests/Engine/AlgorithmManagerTests.cs index ef10cd9598aa..152c95ac7c04 100644 --- a/Tests/Engine/AlgorithmManagerTests.cs +++ b/Tests/Engine/AlgorithmManagerTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * diff --git a/Tests/Engine/DataFeeds/BacktestingFutureChainProviderTests.cs b/Tests/Engine/DataFeeds/BacktestingFutureChainProviderTests.cs index d2741e32e7fe..0d1b13159b78 100644 --- a/Tests/Engine/DataFeeds/BacktestingFutureChainProviderTests.cs +++ b/Tests/Engine/DataFeeds/BacktestingFutureChainProviderTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -33,7 +33,7 @@ public void SetUp() { // Store initial Log Handler _logHandler = Log.LogHandler; - _provider = new BacktestingFutureChainProvider(); + _provider = new BacktestingFutureChainProvider(TestGlobals.DataProvider); } [OneTimeTearDown] diff --git a/Tests/Engine/DataFeeds/CustomLiveDataFeedTests.cs b/Tests/Engine/DataFeeds/CustomLiveDataFeedTests.cs index adc7d93594e2..26c15fc8699f 100644 --- a/Tests/Engine/DataFeeds/CustomLiveDataFeedTests.cs +++ b/Tests/Engine/DataFeeds/CustomLiveDataFeedTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -313,9 +313,8 @@ private void RunLiveDataFeed( _synchronizer = new TestableLiveSynchronizer(timeProvider); _synchronizer.Initialize(algorithm, dataManager); - var mapFileProvider = new LocalDiskMapFileProvider(); _feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(), - mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), dataManager, _synchronizer, new DataChannelProvider()); + TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, dataManager, _synchronizer, new DataChannelProvider()); foreach (var symbol in symbols) { @@ -348,4 +347,4 @@ public static string GetLocalFileName(string ticker, string fileExtension) } } } -} \ No newline at end of file +} diff --git a/Tests/Engine/DataFeeds/Enumerators/DividendEventProviderTests.cs b/Tests/Engine/DataFeeds/Enumerators/DividendEventProviderTests.cs index 121511e65601..5d3d3dbb2ce5 100644 --- a/Tests/Engine/DataFeeds/Enumerators/DividendEventProviderTests.cs +++ b/Tests/Engine/DataFeeds/Enumerators/DividendEventProviderTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -20,6 +20,7 @@ using System.Globalization; using QuantConnect.Data.Market; using QuantConnect.Data.Auxiliary; +using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.DataFeeds.Enumerators; namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators @@ -56,8 +57,9 @@ public void DividendsDistribution(string exDividendDateStr, decimal expectedDist var dividendProvider = new DividendEventProvider(); var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.AAPL, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork, false, false, false); - var factorFile = new LocalDiskFactorFileProvider(); - var mapFile = new LocalDiskMapFileProvider(); + + var mapFile = TestGlobals.MapFileProvider; + var factorFile = TestGlobals.FactorFileProvider; var start = new DateTime(1998, 01, 02); dividendProvider.Initialize(config, factorFile.Get(Symbols.AAPL), mapFile.Get(Market.USA).ResolveMapFile(Symbols.AAPL, start), start); diff --git a/Tests/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactoryTests.cs b/Tests/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactoryTests.cs index 8cd6da84f1d3..b3dac95250e9 100644 --- a/Tests/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactoryTests.cs +++ b/Tests/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactoryTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -46,10 +46,10 @@ public void DoesNotLeakMemory() new SecurityCache() ); - var mapFileProvider = new LocalDiskMapFileProvider(); - var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); + var mapFileProvider = TestGlobals.MapFileProvider; + var fileProvider = TestGlobals.DataProvider; + var factorFileProvider = TestGlobals.FactorFileProvider; var mapFileResolver = mapFileProvider.Get(security.Symbol.ID.Market); - var fileProvider = new DefaultDataProvider(); var factory = new BaseDataSubscriptionEnumeratorFactory(false, mapFileResolver, factorFileProvider); diff --git a/Tests/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactoryTests.cs b/Tests/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactoryTests.cs index 950613022668..330fce5a4404 100644 --- a/Tests/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactoryTests.cs +++ b/Tests/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactoryTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -517,7 +517,8 @@ protected override ISubscriptionDataSourceReader GetSubscriptionDataSourceReader IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, - BaseData baseData) + BaseData baseData, + IDataProvider dataProvider) { return _dataSourceReader; } diff --git a/Tests/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactoryTests.cs b/Tests/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactoryTests.cs index cb8833373a79..daa138c8bd3e 100644 --- a/Tests/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactoryTests.cs +++ b/Tests/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactoryTests.cs @@ -70,11 +70,12 @@ public void DoesNotEmitInvalidData() null ); - var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(false, MapFileResolver.Create(Globals.DataFolder, Market.USA), new LocalDiskFactorFileProvider(new LocalDiskMapFileProvider())); + var dataProvider = TestGlobals.DataProvider; + var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(false, MapFileResolver.Create(Globals.DataFolder, Market.USA), TestGlobals.FactorFileProvider); var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan()); Func> underlyingEnumeratorFunc = (req) => { - var input = enumeratorFactory.CreateEnumerator(req, new DefaultDataProvider()); + var input = enumeratorFactory.CreateEnumerator(req, dataProvider); input = new BaseDataCollectionAggregatorEnumerator(input, req.Configuration.Symbol); return new FillForwardEnumerator( diff --git a/Tests/Engine/DataFeeds/Enumerators/LiveDataBasedDelistingEventProviderTests.cs b/Tests/Engine/DataFeeds/Enumerators/LiveDataBasedDelistingEventProviderTests.cs index 14e4c3705748..917bb3ed7bb1 100644 --- a/Tests/Engine/DataFeeds/Enumerators/LiveDataBasedDelistingEventProviderTests.cs +++ b/Tests/Engine/DataFeeds/Enumerators/LiveDataBasedDelistingEventProviderTests.cs @@ -57,7 +57,7 @@ public void EmitsBasedOnData() return Enumerable.Empty(); }, timeProvider); var provider = new LiveDataBasedDelistingEventProvider(config, dataQueueHandler); - var mapFile = new LocalDiskMapFileProvider().Get(config.Symbol.ID.Market).ResolveMapFile(config.Symbol, config.Type); + var mapFile = TestGlobals.MapFileProvider.Get(config.Symbol.ID.Market).ResolveMapFile(config.Symbol, config.Type); provider.Initialize(config, null, mapFile, time); Assert.IsTrue(autoResetEvent.WaitOne(TimeSpan.FromMilliseconds(100))); diff --git a/Tests/Engine/DataFeeds/Enumerators/LiveDelistingEventProviderEnumeratorTests.cs b/Tests/Engine/DataFeeds/Enumerators/LiveDelistingEventProviderEnumeratorTests.cs index e14e76d85042..677821bd635c 100644 --- a/Tests/Engine/DataFeeds/Enumerators/LiveDelistingEventProviderEnumeratorTests.cs +++ b/Tests/Engine/DataFeeds/Enumerators/LiveDelistingEventProviderEnumeratorTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -19,7 +19,6 @@ using QuantConnect.Securities; using QuantConnect.Data.Market; using System.Collections.Generic; -using QuantConnect.Data.Auxiliary; using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.DataFeeds.Enumerators; @@ -46,7 +45,7 @@ public void EmitsDelistingEventsBasedOnCurrentTime() var timeProvider = new ManualTimeProvider(time); IEnumerator enumerator; - Assert.IsTrue(LiveDelistingEventProviderEnumerator.TryCreate(config, timeProvider, null, cache, new LocalDiskMapFileProvider(), out enumerator)); + Assert.IsTrue(LiveDelistingEventProviderEnumerator.TryCreate(config, timeProvider, null, cache, TestGlobals.MapFileProvider, out enumerator)); Assert.IsFalse(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); @@ -78,4 +77,4 @@ public void EmitsDelistingEventsBasedOnCurrentTime() Assert.IsNull(enumerator.Current); } } -} \ No newline at end of file +} diff --git a/Tests/Engine/DataFeeds/Enumerators/PriceScaleFactorEnumeratorTests.cs b/Tests/Engine/DataFeeds/Enumerators/PriceScaleFactorEnumeratorTests.cs index acd9d7d1ce67..883ad7fd28c2 100644 --- a/Tests/Engine/DataFeeds/Enumerators/PriceScaleFactorEnumeratorTests.cs +++ b/Tests/Engine/DataFeeds/Enumerators/PriceScaleFactorEnumeratorTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -21,6 +21,7 @@ using QuantConnect.Data; using QuantConnect.Data.Auxiliary; using QuantConnect.Data.Market; +using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.DataFeeds.Enumerators; using Tick = QuantConnect.Data.Market.Tick; @@ -44,8 +45,8 @@ public void Setup() true, true, false); - _factorFile = FactorFile.Read( - _config.Symbol.Value, _config.Symbol.ID.Market); + + _factorFile = TestGlobals.FactorFileProvider.Get(_config.Symbol); _rawDataEnumerator = new RawDataEnumerator(); } diff --git a/Tests/Engine/DataFeeds/FileSystemDataFeedTests.cs b/Tests/Engine/DataFeeds/FileSystemDataFeedTests.cs index cc07a77d280d..0aedf33ca071 100644 --- a/Tests/Engine/DataFeeds/FileSystemDataFeedTests.cs +++ b/Tests/Engine/DataFeeds/FileSystemDataFeedTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -39,9 +39,6 @@ public void TestsFileSystemDataFeedSpeed() { var job = new BacktestNodePacket(); var resultHandler = new BacktestingResultHandler(); - var mapFileProvider = new LocalDiskMapFileProvider(); - var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); - var dataProvider = new DefaultDataProvider(); var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second; var feed = new FileSystemDataFeed(); @@ -64,7 +61,7 @@ public void TestsFileSystemDataFeedSpeed() var synchronizer = new Synchronizer(); synchronizer.Initialize(algorithm, dataManager); - feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager, synchronizer, dataPermissionManager.DataChannelProvider); + feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, dataManager, synchronizer, dataPermissionManager.DataChannelProvider); algorithm.Initialize(); algorithm.PostInitialize(); @@ -97,17 +94,14 @@ public void TestDataFeedEnumeratorStackSpeed() algorithm.Initialize(); algorithm.PostInitialize(); - var dataProvider = new DefaultDataProvider(); var resultHandler = new BacktestingResultHandler(); - var mapFileProvider = new LocalDiskMapFileProvider(); - var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); - var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, mapFileProvider, factorFileProvider, dataProvider, enablePriceScaling: false); + var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, enablePriceScaling: false); var universe = algorithm.UniverseManager.Single().Value; var security = algorithm.Securities.Single().Value; var securityConfig = security.Subscriptions.First(); var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, algorithm.StartDate, algorithm.EndDate); - var enumerator = factory.CreateEnumerator(subscriptionRequest, dataProvider); + var enumerator = factory.CreateEnumerator(subscriptionRequest, TestGlobals.DataProvider); var count = 0; var stopwatch = Stopwatch.StartNew(); @@ -147,9 +141,7 @@ public void ChecksMapFileFirstDate() var dataProvider = new DefaultDataProvider(); var resultHandler = new TestResultHandler(); - var mapFileProvider = new LocalDiskMapFileProvider(); - var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); - var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, mapFileProvider, factorFileProvider, dataProvider, enablePriceScaling: false); + var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, enablePriceScaling: false); var universe = algorithm.UniverseManager.Single().Value; var security = algorithm.AddEquity("AAA", Resolution.Daily); diff --git a/Tests/Engine/DataFeeds/IndexSubscriptionDataSourceReaderTests.cs b/Tests/Engine/DataFeeds/IndexSubscriptionDataSourceReaderTests.cs index 43dd149e4bdc..ac4fecc25a61 100644 --- a/Tests/Engine/DataFeeds/IndexSubscriptionDataSourceReaderTests.cs +++ b/Tests/Engine/DataFeeds/IndexSubscriptionDataSourceReaderTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -54,7 +54,8 @@ public void ThrowsIfDataIsNotIndexBased() _dataCacheProvider, config, _initialDate, - false)); + false, + TestGlobals.DataProvider)); } [Test] @@ -74,7 +75,8 @@ public void GetsIndexAndSource() _dataCacheProvider, config, _initialDate, - false); + false, + TestGlobals.DataProvider); var source = (new TradeBar()).GetSource(config, _initialDate, false); _dataCacheProvider.Data = "20000101 00:00,2,2,2,2,2"; var dataBars = reader.Read(source).First(); diff --git a/Tests/Engine/DataFeeds/InternalSubscriptionManagerTests.cs b/Tests/Engine/DataFeeds/InternalSubscriptionManagerTests.cs index 318d4b95141f..3d09690d80f0 100644 --- a/Tests/Engine/DataFeeds/InternalSubscriptionManagerTests.cs +++ b/Tests/Engine/DataFeeds/InternalSubscriptionManagerTests.cs @@ -327,6 +327,8 @@ private void SetupImpl(IDataQueueHandler dataQueueHandler, Synchronizer synchron _dataFeed = new TestableLiveTradingDataFeed(dataQueueHandler ?? new FakeDataQueue(dataAggregator ?? new AggregationManager())); _algorithm = new AlgorithmStub(createDataManager: false); _synchronizer = synchronizer ?? new LiveSynchronizer(); + + var registeredTypesProvider = new RegisteredSecurityDataTypesProvider(); var securityService = new SecurityService(_algorithm.Portfolio.CashBook, MarketHoursDatabase.FromDataFolder(), @@ -338,7 +340,7 @@ private void SetupImpl(IDataQueueHandler dataQueueHandler, Synchronizer synchron _algorithm, securityService, new DataPermissionManager(), - new DefaultDataProvider(), + TestGlobals.DataProvider, Resolution.Second); _dataManager = new DataManager(_dataFeed, universeSelection, _algorithm, new TimeKeeper(DateTime.UtcNow, TimeZones.NewYork), MarketHoursDatabase.FromDataFolder(), @@ -350,9 +352,9 @@ private void SetupImpl(IDataQueueHandler dataQueueHandler, Synchronizer synchron _dataFeed.Initialize(_algorithm, new LiveNodePacket(), _resultHandler, - new LocalDiskMapFileProvider(), - new LocalDiskFactorFileProvider(), - new DefaultDataProvider(), + TestGlobals.MapFileProvider, + TestGlobals.FactorFileProvider, + TestGlobals.DataProvider, _dataManager, _synchronizer, new DataChannelProvider()); diff --git a/Tests/Engine/DataFeeds/LiveCoarseUniverseTests.cs b/Tests/Engine/DataFeeds/LiveCoarseUniverseTests.cs index efe4def35338..61292d09ac72 100644 --- a/Tests/Engine/DataFeeds/LiveCoarseUniverseTests.cs +++ b/Tests/Engine/DataFeeds/LiveCoarseUniverseTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -70,9 +70,8 @@ public void CoarseUniverseRotatesActiveSecurity() var synchronizer = new TestableLiveSynchronizer(timeProvider); synchronizer.Initialize(algorithm, algorithm.DataManager); - var mapFileProvider = new LocalDiskMapFileProvider(); feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(), - mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), algorithm.DataManager, synchronizer, new DataChannelProvider()); + TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, new DataChannelProvider()); var symbolIndex = 0; var coarseUniverseSelectionCount = 0; @@ -157,4 +156,4 @@ public void CoarseUniverseRotatesActiveSecurity() Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount, message: "coarseUniverseSelectionCount"); } } -} \ No newline at end of file +} diff --git a/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs b/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs index fba22634e4a0..a5b4d72ee2df 100644 --- a/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs +++ b/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs @@ -1000,13 +1000,13 @@ public void FastExitsDoNotThrowUnhandledExceptions() algorithm.AddSecurities(Resolution.Tick, Enumerable.Range(0, 20).Select(x => x.ToStringInvariant()).ToList()); var getNextTicksFunction = Enumerable.Range(0, 20).Select(x => new Tick { Symbol = SymbolCache.GetSymbol(x.ToStringInvariant()) }).ToList(); _feed.DataQueueHandler = new FuncDataQueueHandler(handler => getNextTicksFunction, new RealTimeProvider()); - var mapFileProvider = new LocalDiskMapFileProvider(); + _feed.Initialize( algorithm, job, resultHandler, - mapFileProvider, - new LocalDiskFactorFileProvider(mapFileProvider), + TestGlobals.MapFileProvider, + TestGlobals.FactorFileProvider, fileProvider, dataManager, _synchronizer, @@ -1278,7 +1278,6 @@ private IDataFeed RunDataFeed(Resolution resolution = Resolution.Second, List( break; } - var mapFileProvider = new LocalDiskMapFileProvider(); _feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(), - mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, + TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, fileProvider, dataManager, _synchronizer, new TestDataChannelProvider()); if (!dataQueueStarted.WaitOne(TimeSpan.FromMilliseconds(5000))) @@ -2164,9 +2162,8 @@ public void HandlesFutureAndOptionChainUniverse(SecurityType securityType) throw new NotSupportedException($"Unsupported security type: {securityType}"); } - var mapFileProvider = new LocalDiskMapFileProvider(); _feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(), - mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), dataProvider, + TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, dataProvider, dataManager, _synchronizer, new TestDataChannelProvider()); var cancellationTokenSource = new CancellationTokenSource(); diff --git a/Tests/Engine/DataFeeds/SubscriptionDataReaderTests.cs b/Tests/Engine/DataFeeds/SubscriptionDataReaderTests.cs index 1cee9a92e874..d149e10cd492 100644 --- a/Tests/Engine/DataFeeds/SubscriptionDataReaderTests.cs +++ b/Tests/Engine/DataFeeds/SubscriptionDataReaderTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -46,7 +46,7 @@ public void DoesNotEmitDataBeyondTradableDate(string data, bool shouldEmitSecond var start = new DateTime(2019, 12, 9); var end = new DateTime(2019, 12, 12); - var mapFileProvider = new LocalDiskMapFileProvider(); + var mapFileProvider = TestGlobals.MapFileProvider; var mapFileResolver = new MapFileResolver(mapFileProvider.Get(Market.USA)); var dataReader = new SubscriptionDataReader( @@ -61,11 +61,12 @@ public void DoesNotEmitDataBeyondTradableDate(string data, bool shouldEmitSecond start, end, mapFileResolver, - new LocalDiskFactorFileProvider(mapFileProvider), + TestGlobals.FactorFileProvider, LinqExtensions.Range(start, end, time => time + TimeSpan.FromDays(1)), false, new TestDataCacheProvider - { Data = data } + { Data = data }, + TestGlobals.DataProvider ); Assert.IsTrue(dataReader.MoveNext()); diff --git a/Tests/Engine/DataFeeds/SubscriptionUtilsTests.cs b/Tests/Engine/DataFeeds/SubscriptionUtilsTests.cs index 13448e71ac13..14fcf3d7c45b 100644 --- a/Tests/Engine/DataFeeds/SubscriptionUtilsTests.cs +++ b/Tests/Engine/DataFeeds/SubscriptionUtilsTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -37,6 +37,7 @@ public class SubscriptionUtilsTests { private Security _security; private SubscriptionDataConfig _config; + private IFactorFileProvider _factorFileProvider; [SetUp] public void SetUp() @@ -56,6 +57,8 @@ public void SetUp() TimeZones.NewYork, TimeZones.NewYork, true, true, false); + + _factorFileProvider = TestGlobals.FactorFileProvider; } [Test] @@ -63,8 +66,6 @@ public void SubscriptionIsDisposed() { var dataPoints = 10; var enumerator = new TestDataEnumerator { MoveNextTrueCount = dataPoints }; - var factorFileProfider = new Mock(); - factorFileProfider.Setup(s => s.Get(It.IsAny())).Returns(FactorFile.Read(_security.Symbol.Value, _config.Market)); var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( @@ -76,7 +77,7 @@ public void SubscriptionIsDisposed() Time.EndOfTime ), enumerator, - factorFileProfider.Object, + _factorFileProvider, false); var count = 0; @@ -97,8 +98,6 @@ public void SubscriptionIsDisposed() public void ThrowingEnumeratorStackDisposesOfSubscription() { var enumerator = new TestDataEnumerator { MoveNextTrueCount = 10, ThrowException = true}; - var factorFileProfider = new Mock(); - factorFileProfider.Setup(s => s.Get(It.IsAny())).Returns(FactorFile.Read(_security.Symbol.Value, _config.Market)); var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( @@ -110,7 +109,7 @@ public void ThrowingEnumeratorStackDisposesOfSubscription() Time.EndOfTime ), enumerator, - factorFileProfider.Object, + _factorFileProvider, false); var count = 0; @@ -147,8 +146,6 @@ public void ConsumerDoesNotHang() var dataPoints = 10; var enumerator = new TestDataEnumerator {MoveNextTrueCount = dataPoints}; - var factorFileProfider = new Mock(); - factorFileProfider.Setup(s => s.Get(It.IsAny())).Returns(FactorFile.Read(_security.Symbol.Value, _config.Market)); var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( @@ -160,7 +157,7 @@ public void ConsumerDoesNotHang() Time.EndOfTime ), enumerator, - factorFileProfider.Object, + _factorFileProvider, false); for (var j = 0; j < dataPoints; j++) @@ -272,9 +269,7 @@ public void PriceScaleDoesNotUpdateForFillForwardBar() [TestCase(typeof(QuoteBar), false)] public void SubscriptionEmitsAuxData(Type typeOfConfig, bool shouldReceiveAuxData) { - var factorFileProvider = new Mock(); var config = new SubscriptionDataConfig(typeOfConfig, _security.Symbol, Resolution.Hour, TimeZones.NewYork, TimeZones.NewYork, true, true, false); - factorFileProvider.Setup(s => s.Get(It.IsAny())).Returns(FactorFile.Read(_security.Symbol.Value, config.Market)); var totalPoints = 8; var time = new DateTime(2010, 1, 1); @@ -289,7 +284,7 @@ public void SubscriptionEmitsAuxData(Type typeOfConfig, bool shouldReceiveAuxDat Time.EndOfTime ), enumerator, - factorFileProvider.Object, + _factorFileProvider, false); // Test our subscription stream to see if it emits the aux data it should be filtered diff --git a/Tests/Engine/DataFeeds/Transport/RemoteFileSubscriptionStreamReaderTests.cs b/Tests/Engine/DataFeeds/Transport/RemoteFileSubscriptionStreamReaderTests.cs index a4578a6dd9fd..2be236594042 100644 --- a/Tests/Engine/DataFeeds/Transport/RemoteFileSubscriptionStreamReaderTests.cs +++ b/Tests/Engine/DataFeeds/Transport/RemoteFileSubscriptionStreamReaderTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -122,7 +122,7 @@ public void InvalidDataSource() remoteReader.DisposeSafely(); } - private class TestDownloadProvider : Api.Api + private class TestDownloadProvider : QuantConnect.Api.Api { public static int DownloadCount { get; set; } static TestDownloadProvider() diff --git a/Tests/Engine/DataFeeds/ZipEntryNameSubscriptionFactoryTests.cs b/Tests/Engine/DataFeeds/ZipEntryNameSubscriptionFactoryTests.cs index a14fd8dfa9bb..45842d605f3f 100644 --- a/Tests/Engine/DataFeeds/ZipEntryNameSubscriptionFactoryTests.cs +++ b/Tests/Engine/DataFeeds/ZipEntryNameSubscriptionFactoryTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -33,7 +33,7 @@ public void ReadsZipEntryNames() var source = Path.Combine("TestData", "20151224_quote_american.zip"); var config = new SubscriptionDataConfig(typeof (ZipEntryName), Symbol.Create("XLRE", SecurityType.Option, Market.USA), Resolution.Tick, TimeZones.NewYork, TimeZones.NewYork, false, false, false); - var factory = new ZipEntryNameSubscriptionDataSourceReader(config, time, false); + var factory = new ZipEntryNameSubscriptionDataSourceReader(TestGlobals.DataProvider, config, time, false); var expected = new[] { Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Call, 21m, new DateTime(2016, 08, 19)), diff --git a/Tests/Engine/DataProviders/ApiDataProviderTests.cs b/Tests/Engine/DataProviders/ApiDataProviderTests.cs index f5e5aa4c7f3a..f8dafac3c1d5 100644 --- a/Tests/Engine/DataProviders/ApiDataProviderTests.cs +++ b/Tests/Engine/DataProviders/ApiDataProviderTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -21,17 +21,9 @@ namespace QuantConnect.Tests.Engine.DataProviders { - [TestFixture] + [TestFixture, Explicit("Requires configured api access, and also makes calls to data endpoints which are charging transactions")] public class ApiDataProviderTests { - private ApiDataProvider _apiDataProvider; - - [OneTimeSetUp] - public void Setup() - { - _apiDataProvider = new ApiDataProvider(); - } - [TestCase(Resolution.Daily, 6, true)] [TestCase(Resolution.Daily, 3, false)] [TestCase(Resolution.Hour, 6, true)] @@ -51,5 +43,34 @@ public void OutOfDateTest(Resolution resolution, int days, bool expected) Assert.AreEqual(expected, test); File.Delete(path); } + + [TestCase("forex/oanda/minute/eurusd/20131011_quote.zip")] + [TestCase("equity/usa/factor_files/tsla.csv")] + [TestCase("equity/usa/factor_files/factor_files_20210601.zip")] + [TestCase("equity/usa/map_files/googl.csv")] + [TestCase("equity/usa/map_files/map_files_20210601.zip")] + [TestCase("crypto/gdax/daily/btcusd_quote.zip")] + [TestCase("crypto/bitfinex/hour/ethusd_quote.zip")] + [TestCase("option/usa/minute/aapl/20210601_openinterest_american.zip")] + [TestCase("future/sgx/margins/IN.csv")] + public void DownloadFiles(string file) + { + var dataProvider = new ApiDataProvider(); + var path = Path.Combine(Globals.DataFolder, file); + + // Delete it if we already have it + if (File.Exists(path)) + { + File.Delete(path); + Assert.IsFalse(File.Exists(path)); + } + + // Go get the file + var stream = dataProvider.Fetch(path); + Assert.IsNotNull(stream); + stream.Dispose(); + + Assert.IsTrue(File.Exists(path)); + } } } diff --git a/Tests/Engine/DefaultBrokerageMessageHandler.cs b/Tests/Engine/DefaultBrokerageMessageHandler.cs index e75e57d91426..89dfae5b035b 100644 --- a/Tests/Engine/DefaultBrokerageMessageHandler.cs +++ b/Tests/Engine/DefaultBrokerageMessageHandler.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * diff --git a/Tests/Engine/HistoricalData/SubscriptionDataReaderHistoryProviderTests.cs b/Tests/Engine/HistoricalData/SubscriptionDataReaderHistoryProviderTests.cs index aaefabb73b27..73a86da8fcd4 100644 --- a/Tests/Engine/HistoricalData/SubscriptionDataReaderHistoryProviderTests.cs +++ b/Tests/Engine/HistoricalData/SubscriptionDataReaderHistoryProviderTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -36,13 +36,14 @@ public void OptionsAreMappedCorrectly() { var historyProvider = new SubscriptionDataReaderHistoryProvider(); var zipCache = new ZipDataCacheProvider(new DefaultDataProvider()); + historyProvider.Initialize(new HistoryProviderInitializeParameters( null, null, - new DefaultDataProvider(), + TestGlobals.DataProvider, zipCache, - new LocalDiskMapFileProvider(), - new LocalDiskFactorFileProvider(), + TestGlobals.MapFileProvider, + TestGlobals.FactorFileProvider, null, false, new DataPermissionManager())); diff --git a/Tests/Engine/Results/LiveTradingResultHandlerTests.cs b/Tests/Engine/Results/LiveTradingResultHandlerTests.cs index 14a98c1770b5..08bce43e32a6 100644 --- a/Tests/Engine/Results/LiveTradingResultHandlerTests.cs +++ b/Tests/Engine/Results/LiveTradingResultHandlerTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * diff --git a/Tests/Engine/Setup/BaseSetupHandlerTests.cs b/Tests/Engine/Setup/BaseSetupHandlerTests.cs index 10a407627ed9..6b0adad9324e 100644 --- a/Tests/Engine/Setup/BaseSetupHandlerTests.cs +++ b/Tests/Engine/Setup/BaseSetupHandlerTests.cs @@ -37,13 +37,14 @@ public void CurrencyConversionRateResolved() // Setup history provider and algorithm var historyProvider = new SubscriptionDataReaderHistoryProvider(); var zipCache = new ZipDataCacheProvider(new DefaultDataProvider()); + historyProvider.Initialize(new HistoryProviderInitializeParameters( null, null, - new DefaultDataProvider(), + TestGlobals.DataProvider, zipCache, - new LocalDiskMapFileProvider(), - new LocalDiskFactorFileProvider(), + TestGlobals.MapFileProvider, + TestGlobals.FactorFileProvider, null, false, new DataPermissionManager())); diff --git a/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs b/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs index 40d81e2d3750..f92f6e42c8ab 100644 --- a/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs +++ b/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs @@ -147,7 +147,7 @@ public void ExistingHoldingsAndOrdersResolution(Func> getHoldings, setupHandler.CreateBrokerage(job, algorithm, out factory); var result = setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object, - transactionHandler.Object, realTimeHandler.Object, objectStore.Object)); + transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider)); Assert.AreEqual(expected, result); @@ -191,7 +191,7 @@ public void ExistingHoldingsAndOrdersUniverseSettings(Func> getHol setupHandler.CreateBrokerage(job, algorithm, out factory); var result = setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object, - transactionHandler.Object, realTimeHandler.Object, objectStore.Object)); + transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider)); if (result != expected) { @@ -246,7 +246,7 @@ public void LoadsExistingHoldingsAndOrders(Func> getHoldings, Func setupHandler.CreateBrokerage(job, algorithm, out factory); var result = setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object, - transactionHandler.Object, realTimeHandler.Object, objectStore.Object)); + transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider)); Assert.AreEqual(expected, result); @@ -291,7 +291,7 @@ public void EnforcesTotalPortfolioValue(bool fails) setupHandler.CreateBrokerage(job, algorithm, out factory); Assert.AreEqual(!fails, setupHandler.Setup(new SetupHandlerParameters(algorithm.DataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object, - transactionHandler.Object, realTimeHandler.Object, objectStore.Object))); + transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider))); Assert.AreEqual(10000, algorithm.Portfolio.CashBook[Currencies.USD].Amount); Assert.AreEqual(11, algorithm.Portfolio.CashBook[Currencies.GBP].Amount); @@ -326,7 +326,7 @@ public void EnforcesAccountCurrency(bool enforceAccountCurrency) setupHandler.CreateBrokerage(job, algorithm, out factory); Assert.IsTrue(setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object, - transactionHandler.Object, realTimeHandler.Object, objectStore.Object))); + transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider))); Assert.AreEqual(enforceAccountCurrency ? Currencies.USD : Currencies.EUR, algorithm.AccountCurrency); } @@ -363,7 +363,7 @@ public void LoadsHoldingsForExpectedMarket() setupHandler.CreateBrokerage(job, algorithm, out factory); Assert.IsTrue(setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object, - transactionHandler.Object, realTimeHandler.Object, objectStore.Object))); + transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider))); Security security; Assert.IsTrue(algorithm.Portfolio.Securities.TryGetValue(symbol, out security)); @@ -402,7 +402,7 @@ public void SeedsSecurityCorrectly() setupHandler.CreateBrokerage(job, algorithm, out factory); Assert.IsTrue(setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object, - transactionHandler.Object, realTimeHandler.Object, objectStore.Object))); + transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider))); Security security; Assert.IsTrue(algorithm.Portfolio.Securities.TryGetValue(symbol, out security)); @@ -447,7 +447,7 @@ public void AlgorithmTimeIsSetToUtcNowBeforePostInitialize() setupHandler.CreateBrokerage(job, algorithm, out factory); Assert.IsTrue(setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object, - transactionHandler.Object, realTimeHandler.Object, objectStore.Object))); + transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider))); Assert.Greater(algorithm.UtcTime, time); } @@ -497,7 +497,7 @@ public void HasErrorWithZeroTotalPortfolioValue(bool hasCashBalance, bool hasHol var dataManager = new DataManagerStub(algorithm, new MockDataFeed(), true); Assert.IsTrue(setupHandler.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object, - transactionHandler.Object, realTimeHandler.Object, objectStore.Object))); + transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider))); if (!hasCashBalance && !hasHoldings) { diff --git a/Tests/RegressionTests.cs b/Tests/RegressionTests.cs index 78435614b73a..d28b644dc48b 100644 --- a/Tests/RegressionTests.cs +++ b/Tests/RegressionTests.cs @@ -70,6 +70,8 @@ public void AlgorithmStatisticsRegression(AlgorithmStatisticsTestParameters para private static TestCaseData[] GetRegressionTestParameters() { + TestGlobals.Initialize(); + // since these are static test cases, they are executed before test setup AssemblyInitialize.AdjustCurrentDirectory(); diff --git a/Tests/TestGlobals.cs b/Tests/TestGlobals.cs new file mode 100644 index 000000000000..1f61fc00c2a0 --- /dev/null +++ b/Tests/TestGlobals.cs @@ -0,0 +1,44 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Util; +using QuantConnect.Interfaces; +using QuantConnect.Configuration; + +namespace QuantConnect.Tests +{ + /// + /// Global static class for some core providers used by various tests + /// + public static class TestGlobals + { + public static IDataProvider DataProvider + = Composer.Instance.GetExportedValueByTypeName(Config.Get("data-provider", "DefaultDataProvider")); + public static IMapFileProvider MapFileProvider + = Composer.Instance.GetExportedValueByTypeName(Config.Get("map-file-provider", "LocalDiskMapFileProvider")); + public static IFactorFileProvider FactorFileProvider + = Composer.Instance.GetExportedValueByTypeName(Config.Get("factor-file-provider", "LocalDiskFactorFileProvider")); + + /// + /// Initialize our providers, called by AssemblyInitialize.cs so all tests + /// can access initialized providers + /// + public static void Initialize() + { + MapFileProvider.Initialize(DataProvider); + FactorFileProvider.Initialize(MapFileProvider, DataProvider); + } + } +} diff --git a/Tests/ToolBox/LeanDataReaderTests.cs b/Tests/ToolBox/LeanDataReaderTests.cs index 3312374e194b..310cbcce82e6 100644 --- a/Tests/ToolBox/LeanDataReaderTests.cs +++ b/Tests/ToolBox/LeanDataReaderTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -179,11 +179,11 @@ private List LoadFutureChain(Symbol baseFuture, DateTime date, TickType //load future chain first var config = new SubscriptionDataConfig(typeof(ZipEntryName), baseFuture, res, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, tickType); + var factory = new ZipEntryNameSubscriptionDataSourceReader(TestGlobals.DataProvider, config, date, false); - var factory = new ZipEntryNameSubscriptionDataSourceReader(config, date, false); - - return factory.Read(new SubscriptionDataSource(filePath, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName)) + var result = factory.Read(new SubscriptionDataSource(filePath, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName)) .Select(s => s.Symbol).ToList(); + return result; } private string LoadFutureData(Symbol future, TickType tickType, Resolution res) diff --git a/ToolBox/CoarseUniverseGenerator/CoarseUniverseGeneratorProgram.cs b/ToolBox/CoarseUniverseGenerator/CoarseUniverseGeneratorProgram.cs index 2cb0e320b17d..256bf8519dd4 100644 --- a/ToolBox/CoarseUniverseGenerator/CoarseUniverseGeneratorProgram.cs +++ b/ToolBox/CoarseUniverseGenerator/CoarseUniverseGeneratorProgram.cs @@ -26,6 +26,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using QuantConnect.Lean.Engine.DataFeeds; using DateTime = System.DateTime; using Log = QuantConnect.Logging.Log; @@ -55,8 +56,11 @@ public static bool CoarseUniverseGenerator() var fineFundamentalFolder = new DirectoryInfo(Path.Combine(dailyDataFolder.Parent.FullName, "fundamental", "fine")); var blackListedTickersFile = new FileInfo("blacklisted-tickers.txt"); var reservedWordPrefix = Config.Get("reserved-words-prefix", "quantconnect-"); + var dataProvider = new DefaultDataProvider(); var mapFileProvider = new LocalDiskMapFileProvider(); - var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); + mapFileProvider.Initialize(dataProvider); + var factorFileProvider = new LocalDiskFactorFileProvider(); + factorFileProvider.Initialize(mapFileProvider, dataProvider); var generator = new CoarseUniverseGeneratorProgram(dailyDataFolder, destinationFolder, fineFundamentalFolder, Market.USA, blackListedTickersFile, reservedWordPrefix, mapFileProvider, factorFileProvider); return generator.Run(); }