drop GIL before Run to enable multiple python threads to execute in p…

…arallel (microsoft#1042)

* drop GIL before Run to allow multiple python threads to execute in parallel.

* revert setup.py change

onnxruntime/python/onnxruntime_pybind_state.cc

@@ -361,8 +361,6 @@ void addOpSchemaSubmodule(py::module& m){
 #endif //onnxruntime_PYBIND_EXPORT_OPSCHEMA
 
 void addObjectMethods(py::module& m) {
-  // allow unit tests to redirect std::cout and std::cerr to sys.stdout and sys.stderr
-  py::add_ostream_redirect(m, "onnxruntime_ostream_redirect");
   py::class_<SessionOptions>(m, "SessionOptions", R"pbdoc(Configuration information for a session.)pbdoc")
       .def(py::init())
       .def_readwrite("enable_cpu_mem_arena", &SessionOptions::enable_cpu_mem_arena,
@@ -512,10 +510,14 @@ including arg name, arg type (contains both type and shape).)pbdoc")
         std::vector<MLValue> fetches;
         common::Status status;
 
-        if (run_options != nullptr) {
-          status = sess->Run(*run_options, feeds, output_names, &fetches);
-        } else {
-          status = sess->Run(feeds, output_names, &fetches);
+        {
+          // release GIL to allow multiple python threads to invoke Run() in parallel.
+          py::gil_scoped_release release;
+          if (run_options != nullptr) {
+            status = sess->Run(*run_options, feeds, output_names, &fetches);
+          } else {
+            status = sess->Run(feeds, output_names, &fetches);
+          }
         }
 
         if (!status.IsOK()) {

onnxruntime/test/python/onnxruntime_test_python.py

@@ -4,10 +4,9 @@
 # -*- coding: UTF-8 -*-
 import unittest
 import os
-import sys
 import numpy as np
 import onnxruntime as onnxrt
-from onnxruntime.capi._pybind_state import onnxruntime_ostream_redirect
+import threading
 
 
 class TestInferenceSession(unittest.TestCase):
@@ -25,6 +24,13 @@ def get_name(self, name):
             return res
         raise FileNotFoundError("Unable to find '{0}' or '{1}' or '{2}'".format(name, rel, res))
 
+    def run_model(self, session_object, run_options):
+        x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32)
+        input_name = session_object.get_inputs()[0].name
+        res = session_object.run([], {input_name: x}, run_options=run_options)
+        output_expected = np.array([[1.0, 4.0], [9.0, 16.0], [25.0, 36.0]], dtype=np.float32)
+        np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08)
+
     def testRunModel(self):
         sess = onnxrt.InferenceSession(self.get_name("mul_1.pb"))
         x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32)
@@ -72,6 +78,22 @@ def testRunModel2(self):
         output_expected = np.array([[5.0], [11.0], [17.0]], dtype=np.float32)
         np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08)
 
+    def testRunModelMultipleThreads(self):
+        so = onnxrt.SessionOptions()
+        so.session_log_verbosity_level = 1
+        so.session_logid = "MultiThreadsTest"
+        sess = onnxrt.InferenceSession(self.get_name("mul_1.pb"), sess_options=so)
+        ro1 = onnxrt.RunOptions()
+        ro1.run_tag =  "thread1"
+        t1 = threading.Thread(target=self.run_model, args = (sess, ro1))
+        ro2 = onnxrt.RunOptions()
+        ro2.run_tag = "thread2"
+        t2 = threading.Thread(target=self.run_model, args = (sess, ro2))
+        t1.start()
+        t2.start()
+        t1.join()
+        t2.join()
+
     def testRunDevice(self):
         device = onnxrt.get_device()
         self.assertTrue('CPU' in device or 'GPU' in device)
@@ -315,29 +337,6 @@ def testModelMeta(self):
         self.assertEqual('', modelmeta.domain)
         self.assertEqual('', modelmeta.description)
 
-    def testConfigureSessionVerbosityLevel(self):
-        so = onnxrt.SessionOptions()
-        so.session_log_verbosity_level = 1
-
-        # use onnxruntime_ostream_redirect to redirect c++ stdout/stderr to python sys.stdout and sys.stderr
-        with onnxruntime_ostream_redirect(stdout=True, stderr=True):
-          sess = onnxrt.InferenceSession(self.get_name("matmul_1.pb"), sess_options=so)
-          output = sys.stderr.getvalue()
-          self.assertTrue('[I:onnxruntime:InferenceSession, inference_session' in output)
-
-    def testConfigureRunVerbosityLevel(self):
-        ro = onnxrt.RunOptions()
-        ro.run_log_verbosity_level = 1
-        ro.run_tag = "testtag123"
-
-        # use onnxruntime_ostream_redirect to redirect c++ stdout/stderr to python sys.stdout and sys.stderr
-        with onnxruntime_ostream_redirect(stdout=True, stderr=True):
-            sess = onnxrt.InferenceSession(self.get_name("mul_1.pb"))
-            x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32)
-            sess.run([], {'X': x}, run_options=ro)
-            output = sys.stderr.getvalue()
-            self.assertTrue('[I:onnxruntime:testtag123,' in output)
-
     def testProfilerWithSessionOptions(self):
         so = onnxrt.SessionOptions()
         so.enable_profiling = True
@@ -459,4 +458,4 @@ def test_run_model_mlnet(self):
 
 
 if __name__ == '__main__':
-    unittest.main(module=__name__, buffer=True)
+    unittest.main()