Refactor algo mgr time loop isolator limit

Extracting this behavior into it's own class. We'll later extend
the functionality of the implementation to enable a training event
a mechanism for extending the current time loop maximum and/or for
flat out disabling it while the training is runnig and the leaky
bucket has capacity.

Common/IIsolatorLimitResultProvider.cs

@@ -0,0 +1,30 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+namespace QuantConnect
+{
+    /// <summary>
+    /// Provides an abstraction for managing isolator limit results.
+    /// This is originally intended to be used by the training feature to permit a single
+    /// algorithm time loop to extend past the default of ten minutes
+    /// </summary>
+    public interface IIsolatorLimitResultProvider
+    {
+        /// <summary>
+        /// Determines whether or not a custom isolator limit has be reached.
+        /// </summary>
+        IsolatorLimitResult IsWithinLimit();
+    }
+}

Common/QuantConnect.csproj

@@ -290,6 +290,7 @@
     <Compile Include="Data\SubscriptionDataConfigExtensions.cs" />
     <Compile Include="Expiry.cs" />
     <Compile Include="HistoryProviderEvents.cs" />
+    <Compile Include="IIsolatorLimitResultProvider.cs" />
     <Compile Include="Indicators\IIndicatorWarmUpPeriodProvider.cs" />
     <Compile Include="Interfaces\IAccountCurrencyProvider.cs" />
     <Compile Include="Interfaces\IAlgorithmSubscriptionManager.cs" />

Engine/AlgorithmManager.cs

@@ -50,8 +50,8 @@ public class AlgorithmManager
         private decimal _dailyPortfolioValue;
         private IAlgorithm _algorithm;
         private readonly object _lock;
-        private DateTime _currentTimeStepTime;
         private readonly bool _liveMode;
+        private readonly AlgorithmTimeLimitManager _timeLimitManager;
 
         /// <summary>
         /// Publicly accessible algorithm status
@@ -64,27 +64,10 @@ public class AlgorithmManager
         public string AlgorithmId { get; private set; }
 
         /// <summary>
-        /// Gets the amount of time spent on the current time step
+        /// Provides the isolator with a function for verifying that we're not spending too much time in each
+        /// algorithm manager time loop
         /// </summary>
-        public TimeSpan CurrentTimeStepElapsed
-        {
-            get
-            {
-                if (_currentTimeStepTime == DateTime.MinValue)
-                {
-                    _currentTimeStepTime = DateTime.UtcNow;
-                    return TimeSpan.Zero;
-                }
-
-                return DateTime.UtcNow - _currentTimeStepTime;
-            }
-        }
-
-        /// <summary>
-        /// Gets a function used with the Isolator for verifying we're not spending too much time in each
-        /// algo manager timer loop
-        /// </summary>
-        public readonly Func<IsolatorLimitResult> TimeLoopWithinLimits;
+        public IIsolatorLimitResultProvider TimeLimit => _timeLimitManager;
 
         /// <summary>
         /// Quit state flag for the running algorithm. When true the user has requested the backtest stops through a Quit() method.
@@ -107,16 +90,7 @@ public AlgorithmManager(bool liveMode)
             var timeLoopMaximum
                 = TimeSpan.FromMinutes(Config.GetDouble("algorithm-manager-time-loop-maximum", 20));
             AlgorithmId = "";
-            TimeLoopWithinLimits = () =>
-            {
-                var message = string.Empty;
-                if (CurrentTimeStepElapsed > timeLoopMaximum)
-                {
-                    message = $"Algorithm took longer than {timeLoopMaximum.TotalMinutes} minutes on a single time loop.";
-                }
-
-                return new IsolatorLimitResult(CurrentTimeStepElapsed, message);
-            };
+            _timeLimitManager = new AlgorithmTimeLimitManager(timeLoopMaximum);
             _liveMode = liveMode;
         }
 
@@ -197,7 +171,7 @@ public void Run(AlgorithmNodePacket job, IAlgorithm algorithm, ISynchronizer syn
             foreach (var timeSlice in Stream(algorithm, synchronizer, results, token))
             {
                 // reset our timer on each loop
-                _currentTimeStepTime = DateTime.MinValue;
+                _timeLimitManager.StartNewTimeStep();
 
                 //Check this backtest is still running:
                 if (_algorithm.Status != AlgorithmStatus.Running)
@@ -704,7 +678,7 @@ public void Run(AlgorithmNodePacket job, IAlgorithm algorithm, ISynchronizer syn
             } // End of ForEach feed.Bridge.GetConsumingEnumerable
 
             // stop timing the loops
-            _currentTimeStepTime = DateTime.MinValue;
+            _timeLimitManager.StopEnforcingTimeLimit();
 
             //Stream over:: Send the final packet and fire final events:
             Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");

Engine/AlgorithmTimeLimitManager.cs

@@ -0,0 +1,108 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+
+namespace QuantConnect.Lean.Engine
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="IIsolatorLimitResultProvider"/> that tracks the algorithm
+    /// manager's time loops and enforces a maximum amount of time that each time loop may take to execute.
+    /// The isolator uses the result provided by <see cref="IsWithinLimit"/> to determine if it should
+    /// terminate the algorithm for violation of the imposed limits.
+    /// </summary>
+    public class AlgorithmTimeLimitManager : IIsolatorLimitResultProvider
+    {
+        private volatile bool _stopped;
+        private DateTime _currentTimeStepTime;
+        private readonly TimeSpan _timeLoopMaximum;
+
+        /// <summary>
+        /// Initializes a new instance of <see cref="AlgorithmTimeLimitManager"/> to manage the
+        /// creation of <see cref="IsolatorLimitResult"/> instances as it pertains to the
+        /// algorithm manager's time loop
+        /// </summary>
+        /// <param name="timeLoopMaximum">Specifies the maximum amount of time the algorithm is permitted to
+        /// spend in a single time loop. This value can be overriden if certain actions are taken by the
+        /// algorithm, such as invoking the training methods.</param>
+        public AlgorithmTimeLimitManager(TimeSpan timeLoopMaximum)
+        {
+            _timeLoopMaximum = timeLoopMaximum;
+        }
+
+        /// <summary>
+        /// Gets the current amount of time that has elapsed since the beginning of the
+        /// most recent algorithm manager time loop
+        /// </summary>
+        public TimeSpan CurrentTimeStepElapsed
+        {
+            get
+            {
+                if (_currentTimeStepTime == DateTime.MinValue)
+                {
+                    _currentTimeStepTime = DateTime.UtcNow;
+                    return TimeSpan.Zero;
+                }
+
+                return DateTime.UtcNow - _currentTimeStepTime;
+            }
+        }
+
+        /// <summary>
+        /// Invoked by the algorithm at the start of each time loop. This resets the current time step
+        /// elapsed time.
+        /// </summary>
+        /// <remarks>
+        /// This class is the result of a mechanical refactor with the intention of preserving all existing
+        /// behavior, including setting the <code>_currentTimeStepTime</code> to <see cref="DateTime.MinValue"/>
+        /// </remarks>
+        public void StartNewTimeStep()
+        {
+            // maintains existing implementation behavior to reset the time to min value and then
+            // when the isolator pings IsWithinLimit, invocation of CurrentTimeStepElapsed will cause
+            // it to update to the current time. IIRC, this was done to avoid a potential race
+            _currentTimeStepTime = DateTime.MinValue;
+        }
+
+        /// <summary>
+        /// Stops this instance from tracking the algorithm manager's time loop elapsed time.
+        /// This is invoked at the end of the algorithm to prevent the isolator from terminating
+        /// the algorithm during final clean up and shutdown.
+        /// </summary>
+        public void StopEnforcingTimeLimit()
+        {
+            _stopped = true;
+        }
+
+        /// <summary>
+        /// Determines whether or not the algorithm time loop is considered within the limits
+        /// </summary>
+        public IsolatorLimitResult IsWithinLimit()
+        {
+            if (_stopped)
+            {
+                return new IsolatorLimitResult(TimeSpan.Zero, string.Empty);
+            }
+
+            var message = string.Empty;
+            if (CurrentTimeStepElapsed > _timeLoopMaximum)
+            {
+                message = $"Algorithm took longer than {_timeLoopMaximum.TotalMinutes} minutes on a single time loop.";
+            }
+
+            return new IsolatorLimitResult(CurrentTimeStepElapsed, message);
+        }
+    }
+}

Engine/Engine.cs

@@ -320,7 +320,7 @@ public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemb
                         var isolator = new Isolator();
 
                         // Execute the Algorithm Code:
-                        var complete = isolator.ExecuteWithTimeLimit(AlgorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () =>
+                        var complete = isolator.ExecuteWithTimeLimit(AlgorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLimit.IsWithinLimit, () =>
                         {
                             try
                             {

Engine/QuantConnect.Lean.Engine.csproj

@@ -162,6 +162,7 @@
       <Link>Properties\SharedAssemblyInfo.cs</Link>
     </Compile>
     <Compile Include="AlgorithmManager.cs" />
+    <Compile Include="AlgorithmTimeLimitManager.cs" />
     <Compile Include="Alphas\ChartingInsightManagerExtension.cs" />
     <Compile Include="Alphas\StatisticsInsightManagerExtension.cs" />
     <Compile Include="DataFeeds\ApiDataProvider.cs" />