From 41636c94c30f7c3a4f80a692ec95433fbd926e52 Mon Sep 17 00:00:00 2001 From: Martin-Molinero Date: Thu, 29 Jul 2021 21:14:17 -0300 Subject: [PATCH] Common engine initialization (#5809) - Create Lean common engine initialization class, which will be called by research and Lean launcher - Fix backtest deserialization bug due to failing to handle decimal value --- Common/AlgorithmImports.py | 1 + Common/AlphaRuntimeStatistics.cs | 18 ++++---- Engine/Initializer.cs | 72 ++++++++++++++++++++++++++++++++ Launcher/Program.cs | 39 ++--------------- Research/QuantBook.cs | 3 -- Research/QuantConnect.csx | 8 ++-- Research/start.py | 10 ++--- 7 files changed, 93 insertions(+), 58 deletions(-) create mode 100644 Engine/Initializer.cs diff --git a/Common/AlgorithmImports.py b/Common/AlgorithmImports.py index 0fae28220cc9..d7b78f7369d4 100644 --- a/Common/AlgorithmImports.py +++ b/Common/AlgorithmImports.py @@ -48,6 +48,7 @@ from QuantConnect.Orders.Fees import * from QuantConnect.Data.Custom import * from QuantConnect.Data.Market import * +from QuantConnect.Lean.Engine import * from QuantConnect.Orders.Fills import * from QuantConnect.Configuration import * from QuantConnect.Notifications import * diff --git a/Common/AlphaRuntimeStatistics.cs b/Common/AlphaRuntimeStatistics.cs index a863749615eb..11b5357e3441 100644 --- a/Common/AlphaRuntimeStatistics.cs +++ b/Common/AlphaRuntimeStatistics.cs @@ -86,7 +86,7 @@ public AlphaRuntimeStatistics() /// /// The total accumulated estimated value of trading all insights /// - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore), JsonConverter(typeof(StringDecimalJsonConverter), true)] public decimal TotalAccumulatedEstimatedAlphaValue { get; set; } /// @@ -94,7 +94,7 @@ public AlphaRuntimeStatistics() /// /// See https://www.quantconnect.com/forum/discussion/6194/insight-scoring-metric/p1. /// For performance we only truncate when the value is gotten - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore), JsonConverter(typeof(StringDecimalJsonConverter), true)] public decimal KellyCriterionEstimate { get @@ -112,7 +112,7 @@ public decimal KellyCriterionEstimate /// /// See https://www.quantconnect.com/forum/discussion/6194/insight-scoring-metric/p1. /// For performance we only truncate when the value is gotten - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore), JsonConverter(typeof(StringDecimalJsonConverter), true)] public decimal KellyCriterionProbabilityValue { get @@ -130,7 +130,7 @@ public decimal KellyCriterionProbabilityValue /// /// See https://www.quantconnect.com/research/3bc40ecee68d36a9424fbd1b338eb227. /// For performance we only truncate when the value is gotten - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore), JsonConverter(typeof(StringDecimalJsonConverter), true)] public decimal FitnessScore { get @@ -148,7 +148,7 @@ public decimal FitnessScore /// Calculated as the sales volume with respect to the average total portfolio value. /// /// For performance we only truncate when the value is gotten - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore), JsonConverter(typeof(StringDecimalJsonConverter), true)] public decimal PortfolioTurnover { get @@ -166,7 +166,7 @@ public decimal PortfolioTurnover /// It is calculated by dividing the Portfolio Annualized Return by the Maximum Drawdown seen during the backtest. /// /// For performance we only truncate when the value is gotten - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore), JsonConverter(typeof(StringDecimalJsonConverter), true)] public decimal ReturnOverMaxDrawdown { get @@ -200,7 +200,7 @@ public decimal SortinoRatio /// /// Suggested Value of the Alpha On A Monthly Basis For Licensing /// - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore), JsonConverter(typeof(StringDecimalJsonConverter), true)] public decimal EstimatedMonthlyAlphaValue { get @@ -235,7 +235,7 @@ public decimal EstimatedMonthlyAlphaValue /// /// Gets the mean estimated insight value /// - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore), JsonConverter(typeof(StringDecimalJsonConverter), true)] public decimal MeanPopulationEstimatedInsightValue => TotalInsightsClosed > 0 ? TotalAccumulatedEstimatedAlphaValue / TotalInsightsClosed : 0; /// @@ -292,4 +292,4 @@ private static string Invariant(IConvertible obj) return obj.ToStringInvariant(); } } -} \ No newline at end of file +} diff --git a/Engine/Initializer.cs b/Engine/Initializer.cs new file mode 100644 index 000000000000..2b418640dda5 --- /dev/null +++ b/Engine/Initializer.cs @@ -0,0 +1,72 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.IO; +using QuantConnect.Util; +using QuantConnect.Logging; +using QuantConnect.Configuration; + +namespace QuantConnect.Lean.Engine +{ + /// + /// Helper class to initialize a Lean engine + /// + public static class Initializer + { + /// + /// The current Lean Engine System handlers + /// + public static LeanEngineSystemHandlers LeanEngineSystemHandlers { get; private set; } + + /// + /// The current Lean Engine Algorithm handlers + /// + public static LeanEngineAlgorithmHandlers LeanEngineAlgorithmHandlers { get; private set; } + + /// + /// Basic common Lean initialization + /// + public static void Start() + { + try + { + var mode = "RELEASE"; + #if DEBUG + mode = "DEBUG"; + #endif + + Log.DebuggingEnabled = Config.GetBool("debug-mode"); + Log.FilePath = Path.Combine(Config.Get("results-destination-folder"), "log.txt"); + Log.LogHandler = Composer.Instance.GetExportedValueByTypeName(Config.Get("log-handler", "CompositeLogHandler")); + + Log.Trace($"Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v{Globals.Version} Mode: {mode} ({(Environment.Is64BitProcess ? "64" : "32")}bit) Host: {Environment.MachineName}"); + Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString()); + + LeanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance); + + //Setup packeting, queue and controls system: These don't do much locally. + LeanEngineSystemHandlers.Initialize(); + + LeanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance); + } + catch (Exception e) + { + Log.Error(e); + throw; + } + } + } +} diff --git a/Launcher/Program.cs b/Launcher/Program.cs index f9fa5172ab37..dbb6fac58610 100644 --- a/Launcher/Program.cs +++ b/Launcher/Program.cs @@ -47,12 +47,6 @@ static Program() static void Main(string[] args) { - //Initialize: - var mode = "RELEASE"; - #if DEBUG - mode = "DEBUG"; - #endif - if (OS.IsWindows) { Console.OutputEncoding = System.Text.Encoding.UTF8; @@ -65,29 +59,12 @@ static void Main(string[] args) } var liveMode = Config.GetBool("live-mode"); - Log.DebuggingEnabled = Config.GetBool("debug-mode"); - Log.FilePath = Path.Combine(Config.Get("results-destination-folder"), "log.txt"); - Log.LogHandler = Composer.Instance.GetExportedValueByTypeName(Config.Get("log-handler", "CompositeLogHandler")); - //Name thread for the profiler: Thread.CurrentThread.Name = "Algorithm Analysis Thread"; - Log.Trace($"Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v{Globals.Version} Mode: {mode} ({(Environment.Is64BitProcess ? "64" : "32")}bit) Host: {Environment.MachineName}"); - Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString()); - //Import external libraries specific to physical server location (cloud/local) - - try - { - leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance); - } - catch (CompositionException compositionException) - { - Log.Error("Engine.Main(): Failed to load library: " + compositionException); - throw; - } - - //Setup packeting, queue and controls system: These don't do much locally. - leanEngineSystemHandlers.Initialize(); + Initializer.Start(); + leanEngineSystemHandlers = Initializer.LeanEngineSystemHandlers; + leanEngineAlgorithmHandlers = Initializer.LeanEngineAlgorithmHandlers; //-> Pull job from QuantConnect job queue, or, pull local build: string assemblyPath; @@ -100,16 +77,6 @@ static void Main(string[] args) throw new ArgumentException(jobNullMessage); } - try - { - leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance); - } - catch (CompositionException compositionException) - { - Log.Error("Engine.Main(): Failed to load library: " + compositionException); - throw; - } - // if the job version doesn't match this instance version then we can't process it // we also don't want to reprocess redelivered jobs if (job.Redelivered) diff --git a/Research/QuantBook.cs b/Research/QuantBook.cs index 78b38ad2cab1..7bd38d97d825 100644 --- a/Research/QuantBook.cs +++ b/Research/QuantBook.cs @@ -52,9 +52,6 @@ public class QuantBook : QCAlgorithm static QuantBook() { - Logging.Log.LogHandler = - Composer.Instance.GetExportedValueByTypeName(Config.Get("log-handler", "CompositeLogHandler")); - //Determine if we are in a Python Notebook try { diff --git a/Research/QuantConnect.csx b/Research/QuantConnect.csx index 0138562517d1..65fc67b8084a 100644 --- a/Research/QuantConnect.csx +++ b/Research/QuantConnect.csx @@ -80,9 +80,7 @@ using QuantConnect.Securities.Equity; using QuantConnect.Securities.Forex; using QuantConnect.Securities.Interfaces; using QuantConnect.Configuration; +using QuantConnect.Lean.Engine; -// Loads up a connection to our API for use in the research environment -Api api = new Api(); -api.Initialize(Config.GetInt("job-user-id", 1), - Config.Get("api-access-token", "default"), - Config.Get("data-folder")); +Initializer.Start(); +Api api = (Api)Initializer.LeanEngineSystemHandlers.Api; diff --git a/Research/start.py b/Research/start.py index bde3ee2aa9a7..968acc2cbf13 100644 --- a/Research/start.py +++ b/Research/start.py @@ -29,10 +29,10 @@ from AlgorithmImports import * -# Start an instance of an API class -api = Api() -api.Initialize(Config.GetInt("job-user-id", 1), - Config.Get("api-access-token", "default"), - Config.Get("data-folder")) +# Used by pythonNet +AddReference("Fasterflect") + +Initializer.Start() +api = Initializer.LeanEngineSystemHandlers.Api get_ipython().run_line_magic('matplotlib', 'inline')