Handle/filter new brokerage-side orders (QuantConnect#7706)

* Handle/filter brokerage side orders

* Add unit tests

* Minor fix

* Add unrequested security for new brokerage-side orders

* Add unit test for algorithms overriding the brokerage message handler

* Add unit test for python algorithm overriding the brokerage message handler

* Minor change

* Address peer review

Algorithm.CSharp/CustomBrokerageMessageHandlerAlgorithm.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -59,11 +59,21 @@ public class CustomBrokerageMessageHandler : IBrokerageMessageHandler
         /// Process the brokerage message event. Trigger any actions in the algorithm or notifications system required.
         /// </summary>
         /// <param name="message">Message object</param>
-        public void Handle(BrokerageMessageEvent message)
+        public void HandleMessage(BrokerageMessageEvent message)
         {
             var toLog = $"{_algo.Time.ToStringInvariant("o")} Event: {message.Message}";
             _algo.Debug(toLog);
             _algo.Log(toLog);
         }
+
+        /// <summary>
+        /// Handles a new order placed manually in the brokerage side
+        /// </summary>
+        /// <param name="eventArgs">The new order event</param>
+        /// <returns>Whether the order should be added to the transaction handler</returns>
+        public bool HandleOrder(NewBrokerageOrderNotificationEventArgs eventArgs)
+        {
+            return true;
+        }
     }
-}
+}

Algorithm.CSharp/CustomBrokerageSideOrderHandlingRegressionAlgorithm.cs

@@ -0,0 +1,93 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Globalization;
+using QuantConnect.Brokerages;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Algorithm demonstrating the usage of custom brokerage message handler and the new brokerage-side order handling/filtering.
+    /// This test is supposed to be ran by the CustomBrokerageMessageHandlerTests unit test fixture.
+    ///
+    /// All orders are sent from the brokerage, none of them will be placed by the algorithm.
+    /// </summary>
+    public class CustomBrokerageSideOrderHandlingRegressionAlgorithm : QCAlgorithm
+    {
+        private Symbol _spy = QuantConnect.Symbol.Create("SPY", SecurityType.Equity, Market.USA);
+
+        public override void Initialize()
+        {
+            SetStartDate(2013, 10, 07);
+            SetEndDate(2013, 10, 11);
+            SetCash(100000);
+
+            SetBrokerageMessageHandler(new CustomBrokerageMessageHandler(this));
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            // The security should have been added
+            if (!Securities.ContainsKey(_spy))
+            {
+                throw new Exception("Expected security to have been added");
+            }
+
+            if (Transactions.OrdersCount == 0)
+            {
+                throw new Exception("Expected orders to be added from brokerage side");
+            }
+
+            if (Portfolio.Positions.Groups.Count != 1)
+            {
+                throw new Exception("Expected only one position");
+            }
+        }
+
+        public class CustomBrokerageMessageHandler : IBrokerageMessageHandler
+        {
+            private readonly IAlgorithm _algorithm;
+            public CustomBrokerageMessageHandler(IAlgorithm algo) { _algorithm = algo; }
+
+            /// <summary>
+            /// Process the brokerage message event. Trigger any actions in the algorithm or notifications system required.
+            /// </summary>
+            /// <param name="message">Message object</param>
+            public void HandleMessage(BrokerageMessageEvent message)
+            {
+                _algorithm.Debug($"{_algorithm.Time.ToStringInvariant("o")} Event: {message.Message}");
+            }
+
+            /// <summary>
+            /// Handles a new order placed manually in the brokerage side
+            /// </summary>
+            /// <param name="eventArgs">The new order event</param>
+            /// <returns>Whether the order should be added to the transaction handler</returns>
+            public bool HandleOrder(NewBrokerageOrderNotificationEventArgs eventArgs)
+            {
+                var order = eventArgs.Order;
+                if (string.IsNullOrEmpty(order.Tag) || !int.TryParse(order.Tag, NumberStyles.Integer, CultureInfo.InvariantCulture, out var value))
+                {
+                    throw new Exception("Expected all new brokerage-side orders to have a valid tag");
+                }
+
+                // We will only process orders with even tags
+                return value % 2 == 0;
+            }
+        }
+    }
+}

Algorithm.Python/CustomBrokerageSideOrderHandlingRegressionAlgorithm.py

@@ -0,0 +1,63 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Algorithm demonstrating the usage of custom brokerage message handler and the new brokerage-side order handling/filtering.
+### This test is supposed to be ran by the CustomBrokerageMessageHandlerTests unit test fixture.
+###
+### All orders are sent from the brokerage, none of them will be placed by the algorithm.
+### </summary>
+class CustomBrokerageSideOrderHandlingRegressionAlgorithm(QCAlgorithm):
+    '''Algorithm demonstrating the usage of custom brokerage message handler and the new brokerage-side order handling/filtering.
+     This test is supposed to be ran by the CustomBrokerageMessageHandlerTests unit test fixture.
+
+     All orders are sent from the brokerage, none of them will be placed by the algorithm.'''
+
+    def Initialize(self):
+        self.SetStartDate(2013, 10, 7)
+        self.SetEndDate(2013, 10, 11)
+        self.SetCash(100000)
+
+        self.SetBrokerageMessageHandler(CustomBrokerageMessageHandler(self))
+
+        self._spy = Symbol.Create("SPY", SecurityType.Equity, Market.USA)
+
+    def OnEndOfAlgorithm(self):
+        # The security should have been added
+        if not self.Securities.ContainsKey(self._spy):
+            raise Exception("Expected security to have been added")
+
+        if self.Transactions.OrdersCount == 0:
+            raise Exception("Expected orders to be added from brokerage side")
+
+        if len(list(self.Portfolio.Positions.Groups)) != 1:
+            raise Exception("Expected only one position")
+
+class CustomBrokerageMessageHandler(IBrokerageMessageHandler):
+    __namespace__ = "CustomBrokerageSideOrderHandlingRegressionAlgorithm"
+
+    def __init__(self, algorithm):
+        self._algorithm = algorithm
+
+    def HandleMessage(self, message):
+        self._algorithm.Debug(f"{self._algorithm.Time} Event: {message.Message}")
+
+    def HandleOrder(self, eventArgs):
+        order = eventArgs.Order
+        if order.Tag is None or not order.Tag.isdigit():
+            raise Exception("Expected all new brokerage-side orders to have a valid tag")
+
+        # We will only process orders with even tags
+        return int(order.Tag) % 2 == 0

Common/Brokerages/DefaultBrokerageMessageHandler.cs

@@ -68,7 +68,7 @@ public DefaultBrokerageMessageHandler(IAlgorithm algorithm, AlgorithmNodePacket
         /// Handles the message
         /// </summary>
         /// <param name="message">The message to be handled</param>
-        public void Handle(BrokerageMessageEvent message)
+        public void HandleMessage(BrokerageMessageEvent message)
         {
             // based on message type dispatch to result handler
             switch (message.Type)
@@ -159,6 +159,16 @@ where exchange.IsOpenDuringBar(
             }
         }
 
+        /// <summary>
+        /// Handles a new order placed manually in the brokerage side
+        /// </summary>
+        /// <param name="eventArgs">The new order event</param>
+        /// <returns>Whether the order should be added to the transaction handler</returns>
+        public bool HandleOrder(NewBrokerageOrderNotificationEventArgs eventArgs)
+        {
+            return true;
+        }
+
         private void StartCheckReconnected(TimeSpan delay, BrokerageMessageEvent message)
         {
             _cancellationTokenSource.DisposeSafely();

Common/Brokerages/DowngradeErrorCodeToWarningBrokerageMessageHandler.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -41,15 +41,25 @@ public DowngradeErrorCodeToWarningBrokerageMessageHandler(IBrokerageMessageHandl
         /// Handles the message
         /// </summary>
         /// <param name="message">The message to be handled</param>
-        public void Handle(BrokerageMessageEvent message)
+        public void HandleMessage(BrokerageMessageEvent message)
         {
             if (message.Type == BrokerageMessageType.Error && _errorCodesToIgnore.Contains(message.Code))
             {
                 // rewrite the ignored message as a warning message
                 message = new BrokerageMessageEvent(BrokerageMessageType.Warning, message.Code, message.Message);
             }
 
-            _brokerageMessageHandler.Handle(message);
+            _brokerageMessageHandler.HandleMessage(message);
+        }
+
+        /// <summary>
+        /// Handles a new order placed manually in the brokerage side
+        /// </summary>
+        /// <param name="eventArgs">The new order event</param>
+        /// <returns>Whether the order should be added to the transaction handler</returns>
+        public bool HandleOrder(NewBrokerageOrderNotificationEventArgs eventArgs)
+        {
+            return _brokerageMessageHandler.HandleOrder(eventArgs);
         }
     }
-}
+}

Common/Brokerages/IBrokerageMessageHandler.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -25,6 +25,13 @@ public interface IBrokerageMessageHandler
         /// Handles the message
         /// </summary>
         /// <param name="message">The message to be handled</param>
-        void Handle(BrokerageMessageEvent message);
+        void HandleMessage(BrokerageMessageEvent message);
+
+        /// <summary>
+        /// Handles a new order placed manually in the brokerage side
+        /// </summary>
+        /// <param name="eventArgs">The new order event</param>
+        /// <returns>Whether the order should be added to the transaction handler</returns>
+        bool HandleOrder(NewBrokerageOrderNotificationEventArgs eventArgs);
     }
 }

Common/Extensions.cs

@@ -3630,6 +3630,82 @@ private static IEnumerable<Universe> CreateFutureChain(this IAlgorithm algorithm
             return algorithm.UniverseManager.Values.Where(universe => universe.Configuration.Symbol == symbol.Canonical || ContinuousContractUniverse.CreateSymbol(symbol.Canonical) == universe.Configuration.Symbol);
         }
 
+        private static bool _notifiedUniverseSettingsUsed;
+        private static readonly HashSet<SecurityType> _supportedSecurityTypes = new()
+        {
+            SecurityType.Equity,
+            SecurityType.Forex,
+            SecurityType.Cfd,
+            SecurityType.Option,
+            SecurityType.Future,
+            SecurityType.FutureOption,
+            SecurityType.IndexOption,
+            SecurityType.Crypto,
+            SecurityType.CryptoFuture
+        };
+
+        /// <summary>
+        /// Gets the security for the specified symbol from the algorithm's securities collection.
+        /// In case the security is not found, it will be created using the <see cref="IAlgorithm.UniverseSettings"/>
+        /// and a best effort configuration setup.
+        /// </summary>
+        /// <param name="algorithm">The algorithm instance</param>
+        /// <param name="symbol">The symbol which security is being looked up</param>
+        /// <param name="security">The found or added security instance</param>
+        /// <param name="onError">Callback to invoke in case of unsupported security type</param>
+        /// <returns>True if the security was found or added</returns>
+        public static bool GetOrAddUnrequestedSecurity(this IAlgorithm algorithm, Symbol symbol, out Security security,
+            Action<IReadOnlyCollection<SecurityType>> onError = null)
+        {
+            if (!algorithm.Securities.TryGetValue(symbol, out security))
+            {
+                if (!_supportedSecurityTypes.Contains(symbol.SecurityType))
+                {
+                    Log.Error("GetOrAddUnrequestedSecurity(): Unsupported security type: " + symbol.SecurityType + "-" + symbol.Value);
+                    onError?.Invoke(_supportedSecurityTypes);
+                    return false;
+                }
+
+                var resolution = algorithm.UniverseSettings.Resolution;
+                var fillForward = algorithm.UniverseSettings.FillForward;
+                var leverage = algorithm.UniverseSettings.Leverage;
+                var extendedHours = algorithm.UniverseSettings.ExtendedMarketHours;
+
+                if (!_notifiedUniverseSettingsUsed)
+                {
+                    // let's just send the message once
+                    _notifiedUniverseSettingsUsed = true;
+
+                    var leverageMsg = $" Leverage = {leverage};";
+                    if (leverage == Security.NullLeverage)
+                    {
+                        leverageMsg = $" Leverage = default;";
+                    }
+                    algorithm.Debug($"Will use UniverseSettings for automatically added securities for open orders and holdings. UniverseSettings:" +
+                        $" Resolution = {resolution};{leverageMsg} FillForward = {fillForward}; ExtendedHours = {extendedHours}");
+                }
+
+                Log.Trace("GetOrAddUnrequestedSecurity(): Adding unrequested security: " + symbol.Value);
+
+                if (symbol.SecurityType.IsOption())
+                {
+                    // add current option contract to the system
+                    security = algorithm.AddOptionContract(symbol, resolution, fillForward, leverage, extendedHours);
+                }
+                else if (symbol.SecurityType == SecurityType.Future)
+                {
+                    // add current future contract to the system
+                    security = algorithm.AddFutureContract(symbol, resolution, fillForward, leverage, extendedHours);
+                }
+                else
+                {
+                    // for items not directly requested set leverage to 1 and at the min resolution
+                    security = algorithm.AddSecurity(symbol.SecurityType, symbol.Value, resolution, symbol.ID.Market, fillForward, leverage, extendedHours);
+                }
+            }
+            return true;
+        }
+
         /// <summary>
         /// Inverts the specified <paramref name="right"/>
         /// </summary>

Common/Python/BrokerageMessageHandlerPythonWrapper.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -38,11 +38,24 @@ public BrokerageMessageHandlerPythonWrapper(PyObject model)
         /// Handles the message
         /// </summary>
         /// <param name="message">The message to be handled</param>
-        public void Handle(BrokerageMessageEvent message)
+        public void HandleMessage(BrokerageMessageEvent message)
         {
             using (Py.GIL())
             {
-                _model.Handle(message);
+                _model.HandleMessage(message);
+            }
+        }
+
+        /// <summary>
+        /// Handles a new order placed manually in the brokerage side
+        /// </summary>
+        /// <param name="eventArgs">The new order event</param>
+        /// <returns>Whether the order should be added to the transaction handler</returns>
+        public bool HandleOrder(NewBrokerageOrderNotificationEventArgs eventArgs)
+        {
+            using (Py.GIL())
+            {
+                return _model.HandleOrder(eventArgs);
             }
         }
     }

Engine/Engine.cs

@@ -300,7 +300,7 @@ public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemb
                     // wire up the brokerage message handler
                     brokerage.Message += (sender, message) =>
                     {
-                        algorithm.BrokerageMessageHandler.Handle(message);
+                        algorithm.BrokerageMessageHandler.HandleMessage(message);
 
                         // fire brokerage message events
                         algorithm.OnBrokerageMessage(message);