Add wsi/model endpoint tests (Project-MONAI#1275)

* Add wsi/model endpoint tests Signed-off-by: Sachidanand Alle <[email protected]> * Fix object instantiate Signed-off-by: Sachidanand Alle <[email protected]> --------- Signed-off-by: Sachidanand Alle <[email protected]>
jose-rfj · Feb 6, 2023 · 2a9d727 · 2a9d727
1 parent 4a96ae7
commit 2a9d727
Show file tree

Hide file tree

Showing 13 changed files with 149 additions and 40 deletions.
diff --git a/monailabel/interfaces/utils/app.py b/monailabel/interfaces/utils/app.py
@@ -13,7 +13,6 @@
 import json
 import logging
 import os
-import sys
 from typing import Any, Dict
 
 from monailabel.config import settings
@@ -97,9 +96,6 @@ def run_main():
     logger.debug(f"++ APP_DIR: {app_dir}")
     logger.debug(f"++ STUDIES: {studies}")
 
-    sys.path.append(app_dir)
-    sys.path.append(os.path.join(app_dir, "lib"))
-
     logging.basicConfig(
         level=(logging.DEBUG if args.debug else logging.INFO),
         format="[%(asctime)s] [%(threadName)s] [%(levelname)s] (%(name)s:%(lineno)d) - %(message)s",

diff --git a/monailabel/main.py b/monailabel/main.py
@@ -295,8 +295,6 @@ def start_server_init_settings(self, args):
             if not os.path.exists(d):
                 os.makedirs(d)
 
-        sys.path.append(args.app)
-        sys.path.append(os.path.join(args.app, "lib"))
         os.environ["PATH"] += os.pathsep + os.path.join(args.app, "bin")
 
         if args.dryrun:

diff --git a/monailabel/tasks/infer/bundle.py b/monailabel/tasks/infer/bundle.py
@@ -12,6 +12,7 @@
 import json
 import logging
 import os
+import sys
 from typing import Any, Callable, Dict, Optional, Sequence, Union
 
 from monai.bundle import ConfigItem, ConfigParser
@@ -22,6 +23,7 @@
 from monailabel.tasks.infer.basic_infer import BasicInferTask
 from monailabel.transform.post import Restored
 from monailabel.transform.pre import LoadImageTensord
+from monailabel.utils.others.class_utils import unload_module
 from monailabel.utils.others.generic import strtobool
 
 logger = logging.getLogger(__name__)
@@ -96,6 +98,10 @@ def __init__(
             logger.warning(f"Ignore {path} as there is no infer config {self.const.configs()} exists")
             return
 
+        self.bundle_path = path
+        sys.path.insert(0, self.bundle_path)
+        unload_module("scripts")
+
         self.bundle_config = ConfigParser()
         self.bundle_config.read_config(os.path.join(path, "configs", config_paths[0]))
         self.bundle_config.config.update({self.const.key_bundle_root(): path})  # type: ignore
@@ -112,6 +118,7 @@ def __init__(
                 logger.warning(
                     f"Ignore {path} as neither {self.const.model_pytorch()} nor {self.const.model_torchscript()} exists"
                 )
+                sys.path.remove(self.bundle_path)
                 return
 
         # https://docs.monai.io/en/latest/mb_specification.html#metadata-json-file
@@ -142,6 +149,7 @@ def __init__(
         )
         self.valid = True
         self.version = metadata.get("version")
+        sys.path.remove(self.bundle_path)
 
     def is_valid(self) -> bool:
         return self.valid
@@ -152,6 +160,8 @@ def info(self) -> Dict[str, Any]:
         return i
 
     def pre_transforms(self, data=None) -> Sequence[Callable]:
+        sys.path.insert(0, self.bundle_path)
+        unload_module("scripts")
         self._update_device(data)
 
         pre = []
@@ -169,30 +179,47 @@ def pre_transforms(self, data=None) -> Sequence[Callable]:
                 else:
                     res.append(t)
             pre = res
+
+        sys.path.remove(self.bundle_path)
         return pre
 
     def inferer(self, data=None) -> Inferer:
+        sys.path.insert(0, self.bundle_path)
+        unload_module("scripts")
         self._update_device(data)
 
+        i = None
         for k in self.const.key_inferer():
             if self.bundle_config.get(k):
-                return self.bundle_config.get_parsed_content(k, instantiate=True)  # type: ignore
-        return SimpleInferer()
+                i = self.bundle_config.get_parsed_content(k, instantiate=True)  # type: ignore
+                break
+
+        sys.path.remove(self.bundle_path)
+        return i if i is not None else SimpleInferer()
 
     def detector(self, data=None) -> Optional[Callable]:
+        sys.path.insert(0, self.bundle_path)
+        unload_module("scripts")
         self._update_device(data)
 
+        d = None
         for k in self.const.key_detector():
             if self.bundle_config.get(k):
                 detector = self.bundle_config.get_parsed_content(k, instantiate=True)  # type: ignore
                 for k in self.const.key_detector_ops():
                     self.bundle_config.get_parsed_content(k, instantiate=True)
+
                 if detector is None or callable(detector):
-                    return detector  # type: ignore
+                    d = detector  # type: ignore
+                    break
                 raise ValueError("Invalid Detector type;  It's not callable")
-        return None
+
+        sys.path.remove(self.bundle_path)
+        return d
 
     def post_transforms(self, data=None) -> Sequence[Callable]:
+        sys.path.insert(0, self.bundle_path)
+        unload_module("scripts")
         self._update_device(data)
 
         post = []
@@ -204,6 +231,8 @@ def post_transforms(self, data=None) -> Sequence[Callable]:
 
         if self.add_post_restore:
             post.append(Restored(keys=self.key_pred, ref_image=self.key_image))
+
+        sys.path.remove(self.bundle_path)
         return post
 
     def _get_type(self, name, type):

diff --git a/monailabel/tasks/train/bundle.py b/monailabel/tasks/train/bundle.py
@@ -13,6 +13,7 @@
 import logging
 import os
 import subprocess
+import sys
 from typing import Dict, Optional, Sequence
 
 import monai.bundle
@@ -24,6 +25,7 @@
 from monailabel.config import settings
 from monailabel.interfaces.datastore import Datastore
 from monailabel.interfaces.tasks.train import TrainTask
+from monailabel.utils.others.class_utils import unload_module
 
 logger = logging.getLogger(__name__)
 
@@ -266,6 +268,9 @@ def __call__(self, request, datastore: Datastore):
                 self.bundle_config.set(v, k)
             ConfigParser.export_config_file(self.bundle_config.config, train_path, indent=2)  # type: ignore
 
+            sys.path.insert(0, self.bundle_path)
+            unload_module("scripts")
+
             env = os.environ.copy()
             env["CUDA_VISIBLE_DEVICES"] = ",".join([str(g) for g in gpus])
             logger.info(f"Using CUDA_VISIBLE_DEVICES: {env['CUDA_VISIBLE_DEVICES']}")
@@ -293,8 +298,13 @@ def __call__(self, request, datastore: Datastore):
 
             self.run_multi_gpu(request, cmd, env)
         else:
+            sys.path.insert(0, self.bundle_path)
+            unload_module("scripts")
+
             self.run_single_gpu(request, overrides)
 
+        sys.path.remove(self.bundle_path)
+
         logger.info("Training Finished....")
         return {}
 

diff --git a/monailabel/utils/others/class_utils.py b/monailabel/utils/others/class_utils.py
@@ -14,17 +14,37 @@
 import inspect
 import logging
 import os
+import sys
+from distutils.util import strtobool
 from typing import List
 
 from monailabel.interfaces.exception import MONAILabelError, MONAILabelException
 
 logger = logging.getLogger(__name__)
 
 
+def unload_module(name):
+    modules = []
+    for m in sorted(sys.modules):
+        if m == name or m.startswith(f"{name}."):
+            modules.append(m)
+
+    if modules and strtobool(os.environ.get("MONAI_LABEL_RELOAD_APP_LIB", "true")):
+        logger.info(f"Remove/Reload previous Modules: {modules}")
+        for m in modules:
+            del sys.modules[m]
+
+
 def module_from_file(module_name, file_path):
+    app_dir = os.path.dirname(file_path)
+    sys.path.insert(0, app_dir)
+    unload_module("lib")
+
     spec = importlib.util.spec_from_file_location(module_name, file_path)
     module = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(module)
+
+    sys.path.remove(app_dir)
     logger.debug(f"module: {module}")
     return module
 

diff --git a/monailabel/utils/others/generic.py b/monailabel/utils/others/generic.py
@@ -18,7 +18,6 @@
 import pathlib
 import shutil
 import subprocess
-import sys
 import time
 from typing import Dict
 
@@ -308,7 +307,6 @@ def get_bundle_models(app_dir, conf, conf_key="models"):
                 download(name=name, version=version, bundle_dir=model_dir, source=zoo_source, repo=zoo_repo)
                 if version:
                     shutil.move(os.path.join(model_dir, name), p)
-        sys.path.append(p)
 
         bundles[k] = p
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -46,7 +46,7 @@ exclude = '''
 
 [tool.pytest.ini_options]
 log_cli = true
-log_cli_level = "ERROR"
+log_cli_level = "INFO"
 log_cli_format = "[%(threadName)s] [%(levelname)s] (%(name)s:%(lineno)d) %(message)s"
 
 #log_file = "pytest.log"

diff --git a/sample-apps/pathology/lib/activelearning/__init__.py b/sample-apps/pathology/lib/activelearning/__init__.py
@@ -9,4 +9,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .random import Random
+from .random import WSIRandom
diff --git a/tests/unit/endpoints/context.py b/tests/unit/endpoints/context.py
@@ -12,7 +12,7 @@
 import logging
 import os
 import random
-import sys
+import time
 import unittest
 
 from fastapi.testclient import TestClient
@@ -54,7 +54,9 @@ def create_client(app_dir, studies, data_dir, conf=None):
     from monailabel.interfaces.utils.app import clear_cache
 
     clear_cache()
-    return TestClient(app)
+    c = TestClient(app)
+    time.sleep(1)
+    return c
 
 
 class BasicEndpointTestSuite(unittest.TestCase):
@@ -68,12 +70,11 @@ class BasicEndpointTestSuite(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
-        sys.path.append(cls.app_dir)
         cls.client = create_client(cls.app_dir, cls.studies, cls.data_dir)
 
     @classmethod
     def tearDownClass(cls) -> None:
-        sys.path.remove(cls.app_dir)
+        pass
 
 
 class DICOMWebEndpointTestSuite(unittest.TestCase):
@@ -88,12 +89,11 @@ class DICOMWebEndpointTestSuite(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
-        sys.path.append(cls.app_dir)
         cls.client = create_client(cls.app_dir, cls.studies, cls.data_dir)
 
     @classmethod
     def tearDownClass(cls) -> None:
-        sys.path.remove(cls.app_dir)
+        pass
 
 
 class BasicEndpointV2TestSuite(unittest.TestCase):
@@ -107,14 +107,13 @@ class BasicEndpointV2TestSuite(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
-        sys.path.append(cls.app_dir)
         cls.client = create_client(
             cls.app_dir, cls.studies, cls.data_dir, {"models": "deepgrow_2d,deepgrow_3d,segmentation_spleen,deepedit"}
         )
 
     @classmethod
     def tearDownClass(cls) -> None:
-        sys.path.remove(cls.app_dir)
+        pass
 
 
 class BasicEndpointV3TestSuite(unittest.TestCase):
@@ -135,12 +134,11 @@ def setUpClass(cls) -> None:
             "skip_scoring": "false",
             "models": "segmentation_spleen",
         }
-        sys.path.append(cls.app_dir)
         cls.client = create_client(cls.app_dir, cls.studies, cls.data_dir, conf=conf)
 
     @classmethod
     def tearDownClass(cls) -> None:
-        sys.path.remove(cls.app_dir)
+        pass
 
 
 class BasicEndpointV4TestSuite(unittest.TestCase):
@@ -153,21 +151,15 @@ class BasicEndpointV4TestSuite(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
-        sys.path.append(cls.app_dir)
-        # sys.path.append(os.path.join(cls.app_dir, "lib", "activelearning", "random.py"))
-        # sys.path.append(os.path.join(cls.app_dir, "lib", "infers", "nuclick.py"))
-
-        cls.client = create_client(
-            cls.app_dir, cls.studies, cls.data_dir, {"models": "segmentation_nuclei,nuclick,classification_nuclei"}
-        )
+        cls.client = create_client(cls.app_dir, cls.studies, cls.data_dir, {"models": "segmentation_nuclei"})
         response = cls.client.get("/info/")
         # check if following fields exist in the response
         res = response.json()
         print(res)
 
     @classmethod
     def tearDownClass(cls) -> None:
-        sys.path.remove(cls.app_dir)
+        pass
 
 
 class BasicDetectionBundleTestSuite(unittest.TestCase):
@@ -181,14 +173,13 @@ class BasicDetectionBundleTestSuite(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
-        sys.path.append(cls.app_dir)
         cls.client = create_client(
             cls.app_dir, cls.studies, cls.data_dir, {"models": "lung_nodule_ct_detection", "tracking": False}
         )
 
     @classmethod
     def tearDownClass(cls) -> None:
-        sys.path.remove(cls.app_dir)
+        pass
 
 
 class BasicBundleTestSuite(unittest.TestCase):
@@ -202,12 +193,11 @@ class BasicBundleTestSuite(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
-        sys.path.append(cls.app_dir)
         cls.client = create_client(cls.app_dir, cls.studies, cls.data_dir, {"models": "spleen_ct_segmentation"})
 
     @classmethod
     def tearDownClass(cls) -> None:
-        sys.path.remove(cls.app_dir)
+        pass
 
 
 class BasicBundleV2TestSuite(unittest.TestCase):
@@ -228,9 +218,8 @@ def setUpClass(cls) -> None:
             "epistemic_dropout": 0.2,
             "models": "spleen_ct_segmentation",
         }
-        sys.path.append(cls.app_dir)
         cls.client = create_client(cls.app_dir, cls.studies, cls.data_dir, conf=conf)
 
     @classmethod
     def tearDownClass(cls) -> None:
-        sys.path.remove(cls.app_dir)
+        pass