Making WorkerThread static

- Making WorkerThread instance static
- Adding PythonInitialize.AddPythonPaths(), useful to add paths to
python after initialization

Author: Martin-Molinero

Common/Python/PythonInitializer.cs

@@ -14,6 +14,8 @@
  *
 */
 
+using System.Collections.Generic;
+using System.Linq;
 using Python.Runtime;
 using QuantConnect.Logging;
 
@@ -44,5 +46,20 @@ public static void Initialize()
                 Log.Trace("PythonInitializer.Initialize(): ended");
             }
         }
+
+        /// <summary>
+        /// Adds directories to the python path at runtime
+        /// </summary>
+        public static void AddPythonPaths(IEnumerable<string> paths)
+        {
+            if (_isBeginAllowThreadsCalled)
+            {
+                using (Py.GIL())
+                {
+                    var code = string.Join(";", paths.Select(s => $"sys.path.append('{s}')"));
+                    PythonEngine.Exec($"import sys;{code}");
+                }
+            }
+        }
     }
 }

Common/Util/WorkerThread.cs

@@ -31,6 +31,11 @@ public class WorkerThread : IDisposable
         private readonly CancellationTokenSource _threadCancellationTokenSource;
         private readonly Thread _workerThread;
 
+        /// <summary>
+        /// The worker thread instance
+        /// </summary>
+        public static WorkerThread Instance = new WorkerThread();
+
         /// <summary>
         /// Will be set when the worker thread finishes a work item
         /// </summary>
@@ -40,7 +45,7 @@ public class WorkerThread : IDisposable
         /// Creates a new instance, which internally launches a new worker thread
         /// </summary>
         /// <remarks><see cref="Dispose"/></remarks>
-        public WorkerThread()
+        protected WorkerThread()
         {
             _threadCancellationTokenSource = new CancellationTokenSource();
             FinishedWorkItem = new AutoResetEvent(false);

Engine/Engine.cs

@@ -79,9 +79,10 @@ public Engine(LeanEngineSystemHandlers systemHandlers, LeanEngineAlgorithmHandle
         /// Runs a single backtest/live job from the job queue
         /// </summary>
         /// <param name="job">The algorithm job to be processed</param>
-        /// <param name="manager"></param>
+        /// <param name="manager">The algorithm manager instance</param>
         /// <param name="assemblyPath">The path to the algorithm's assembly</param>
-        public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath)
+        /// <param name="workerThread">The worker thread instance</param>
+        public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath, WorkerThread workerThread)
         {
             var marketHoursDatabaseTask = Task.Run(() => StaticInitializations());
 
@@ -96,7 +97,6 @@ public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemb
                 Thread threadResults = null;
                 Thread threadRealTime = null;
                 Thread threadAlphas = null;
-                WorkerThread workerThread = null;
 
                 //-> Initialize messaging system
                 SystemHandlers.Notify.SetAuthentication(job);
@@ -116,8 +116,6 @@ public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemb
                     // since the algorithm constructor will use it
                     var marketHoursDatabase = marketHoursDatabaseTask.Result;
 
-                    // start worker thread
-                    workerThread = new WorkerThread();
                     AlgorithmHandlers.Setup.WorkerThread = workerThread;
 
                     // Save algorithm to cache, load algorithm instance:

Launcher/Program.cs

@@ -127,7 +127,7 @@ static void Main(string[] args)
                 leanEngineSystemHandlers.LeanManager.Initialize(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
 
                 var engine = new Engine.Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode);
-                engine.Run(job, algorithmManager, assemblyPath);
+                engine.Run(job, algorithmManager, assemblyPath, WorkerThread.Instance);
             }
             finally
             {

Tests/AlgorithmFactory/LoaderTests.cs

@@ -30,7 +30,7 @@ public class LoaderTests
         [SetUp]
         public void SetUp()
         {
-            _workerThread = new WorkerThread();
+            _workerThread = new TestWorkerThread();
         }
 
         [TearDown]
@@ -108,5 +108,10 @@ public void LoadsSepereateAlgorithm_UsingSingleOrAlgorithmTypeName_ExtensionMeth
             Assert.IsTrue(two);
             Assert.AreNotEqual(algorithm1.ToString(), algorithm2.ToString());
         }
+
+        private class TestWorkerThread : WorkerThread
+        {
+
+        }
     }
 }

Tests/AlgorithmRunner.cs

@@ -120,7 +120,7 @@ public static AlgorithmManager RunLocalBacktest(
 
                             systemHandlers.LeanManager.Initialize(systemHandlers, algorithmHandlers, job, algorithmManager);
 
-                            engine.Run(job, algorithmManager, algorithmPath);
+                            engine.Run(job, algorithmManager, algorithmPath, new TestWorkerThread());
                             ordersLogFile = ((RegressionResultHandler)algorithmHandlers.Results).LogFilePath;
                         }
                         catch (Exception e)
@@ -237,5 +237,9 @@ public override IEnumerable<Slice> GetHistory(IEnumerable<HistoryRequest> reques
                 return base.GetHistory(requests, sliceTimeZone);
             }
         }
+
+        class TestWorkerThread : WorkerThread
+        {
+        }
     }
 }

Tests/Common/IsolatorTests.cs

@@ -27,7 +27,7 @@ public class IsolatorTests
         [Test]
         public void WorksCorrectlyUsingWorker()
         {
-            using (var worker = new WorkerThread())
+            using (var worker = new TestWorkerThread())
             {
                 var isolator = new Isolator();
                 var executed = false;
@@ -51,7 +51,7 @@ public void WorksCorrectlyUsingWorker()
         [TestCase(Language.CSharp, false)]
         public void TimeOutWorkCorrectly(Language language, bool useWorker)
         {
-            var worker = useWorker ? new WorkerThread() : null;
+            var worker = useWorker ? new TestWorkerThread() : null;
             using (worker)
             {
                 var isolator = new Isolator();
@@ -92,5 +92,10 @@ public void TimeOutWorkCorrectly(Language language, bool useWorker)
                 }
             }
         }
+
+        private class TestWorkerThread : WorkerThread
+        {
+
+        }
     }
 }