Merge pull request Ericsson#4076 from bruntib/multiprocess_lib

[cfg] Using multiprocess lib instead of multiprocessing

CONTRIBUTING.md

@@ -31,7 +31,7 @@ rules while writing your code:
 Order your `import` commands according to as follows:
 
   1. **System-wide** imports come first and foremost, e.g.
-    `import multiprocessing`.
+    `import subprocess`.
   2. _(Empty line for readability)_
   3. External module/library imports that are not system-wide but related to
      CodeChecker's **dependencies**, e.g. `from thrift import Thrift`.
@@ -62,7 +62,7 @@ Documentation about the module.
 
 # -- 1. System specific imports
 import atexit
-from multiprocessing import Pool
+from collections import defaultdict
 import os
 import tempfile
 

analyzer/codechecker_analyzer/analysis_manager.py

@@ -10,7 +10,6 @@
 
 
 import glob
-import multiprocessing
 import os
 import shlex
 import shutil
@@ -22,6 +21,7 @@
 
 from threading import Timer
 
+import multiprocess
 import psutil
 
 from codechecker_common.logger import get_logger
@@ -740,12 +740,11 @@ def signal_handler(signum, frame):
 
     actions, skipped_actions = skip_cpp(actions, skip_handlers)
     # Start checking parallel.
-    checked_var = multiprocessing.Value('i', 1)
-    actions_num = multiprocessing.Value('i', len(actions))
-    pool = multiprocessing.Pool(jobs,
-                                initializer=init_worker,
-                                initargs=(checked_var,
-                                          actions_num))
+    checked_var = multiprocess.Value('i', 1)
+    actions_num = multiprocess.Value('i', len(actions))
+    pool = multiprocess.Pool(jobs,
+                             initializer=init_worker,
+                             initargs=(checked_var, actions_num))
     signal.signal(signal.SIGINT, signal_handler)
 
     # If the analysis has failed, we help debugging.

analyzer/codechecker_analyzer/analyzer.py

@@ -11,13 +11,14 @@
 
 
 from collections import defaultdict
-from multiprocessing.managers import SyncManager
 import os
 import shutil
 import signal
 import sys
 import time
 
+from multiprocess.managers import SyncManager
+
 from codechecker_common.logger import get_logger
 
 from . import analyzer_context, analysis_manager, pre_analysis_manager, \

analyzer/codechecker_analyzer/cmd/analyze.py

@@ -13,11 +13,11 @@
 import argparse
 import collections
 import json
-import multiprocessing
 import os
 import shutil
 import sys
 
+import multiprocess
 from typing import List
 
 from tu_collector import tu_collector
@@ -169,7 +169,7 @@ def add_arguments_to_parser(parser):
                         type=int,
                         dest="jobs",
                         required=False,
-                        default=multiprocessing.cpu_count(),
+                        default=multiprocess.cpu_count(),
                         help="Number of threads to use in analysis. More "
                              "threads mean faster analysis at the cost of "
                              "using more memory.")

analyzer/codechecker_analyzer/cmd/check.py

@@ -12,12 +12,13 @@
 """
 
 import argparse
-import multiprocessing
 import os
 import shutil
 import sys
 import tempfile
 
+import multiprocess
+
 from codechecker_analyzer.analyzers import analyzer_types
 from codechecker_analyzer.arg import \
     OrderedCheckersAction, OrderedConfigAction, \
@@ -182,7 +183,7 @@ def add_arguments_to_parser(parser):
                                type=int,
                                dest="jobs",
                                required=False,
-                               default=multiprocessing.cpu_count(),
+                               default=multiprocess.cpu_count(),
                                help="Number of threads to use in analysis. "
                                     "More threads mean faster analysis at "
                                     "the cost of using more memory.")

analyzer/codechecker_analyzer/pre_analysis_manager.py

@@ -9,7 +9,6 @@
 Run pre analysis, collect statistics or CTU data.
 """
 
-import multiprocessing
 import os
 import shlex
 import shutil
@@ -18,6 +17,8 @@
 import traceback
 import uuid
 
+import multiprocess
+
 from codechecker_common.logger import get_logger
 
 from codechecker_statistics_collector import post_process_stats
@@ -164,13 +165,12 @@ def signal_handler(signum, frame):
 
     signal.signal(signal.SIGINT, signal_handler)
 
-    processed_var = multiprocessing.Value('i', 0)
-    actions_num = multiprocessing.Value('i', len(actions))
+    processed_var = multiprocess.Value('i', 0)
+    actions_num = multiprocess.Value('i', len(actions))
 
-    pool = multiprocessing.Pool(jobs,
-                                initializer=init_worker,
-                                initargs=(processed_var,
-                                          actions_num))
+    pool = multiprocess.Pool(jobs,
+                             initializer=init_worker,
+                             initargs=(processed_var, actions_num))
 
     if statistics_data:
         # Statistics collection is enabled setup temporary

analyzer/requirements.txt

@@ -4,3 +4,4 @@ psutil==5.8.0
 PyYAML==6.0.1
 sarif-tools==1.0.0
 mypy_extensions==0.4.3
+multiprocess==0.70.15

web/requirements.txt

@@ -4,6 +4,7 @@ alembic==1.5.5
 portalocker==2.2.1
 psutil==5.8.0
 mypy_extensions==0.4.3
+multiprocess==0.70.15
 thrift==0.13.0
 gitpython==3.1.37
 PyYAML==6.0.1

web/server/codechecker_server/server.py

@@ -14,7 +14,6 @@
 import atexit
 import datetime
 from hashlib import sha256
-from multiprocessing import Process
 import os
 import posixpath
 from random import sample
@@ -28,6 +27,7 @@
 
 from http.server import HTTPServer, BaseHTTPRequestHandler, \
     SimpleHTTPRequestHandler
+from multiprocess import Process
 
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.sql.expression import func

web/tests/functional/authentication/__init__.py

@@ -10,15 +10,15 @@
 """Setup for the package tests."""
 
 
-import multiprocessing
 import os
 import shutil
 
 from libtest import codechecker
 from libtest import env
+import multiprocess
 
 # Stopping event for CodeChecker server.
-__STOP_SERVER = multiprocessing.Event()
+__STOP_SERVER = multiprocess.Event()
 
 # Test workspace initialized at setup for authentication tests.
 TEST_WORKSPACE = None

web/tests/functional/cli_config/test_server_config.py

@@ -13,12 +13,12 @@
 
 
 import json
-import multiprocessing
 import os
 import unittest
 
 from libtest import codechecker
 from libtest import env
+import multiprocess
 
 from . import setup_class_common, teardown_class_common
 
@@ -56,7 +56,7 @@ def test_valid_json_config(self):
             json.dump({
                 'server': ['--skip-db-cleanup']}, config_f)
 
-        event = multiprocessing.Event()
+        event = multiprocess.Event()
         event.clear()
 
         self.codechecker_cfg['viewer_port'] = env.get_free_port()
@@ -80,7 +80,7 @@ def test_valid_yaml_config(self):
 server:
   - --skip-db-cleanup""")
 
-        event = multiprocessing.Event()
+        event = multiprocess.Event()
         event.clear()
 
         self.codechecker_cfg['viewer_port'] = env.get_free_port()
@@ -102,7 +102,7 @@ def test_invalid_config(self):
             json.dump({
                 'server': ['--dummy-option']}, config_f)
 
-        event = multiprocessing.Event()
+        event = multiprocess.Event()
         event.clear()
 
         self.codechecker_cfg['viewer_port'] = env.get_free_port()
@@ -123,7 +123,7 @@ def test_empty_config(self):
                   encoding="utf-8", errors="ignore") as config_f:
             config_f.write("")
 
-        event = multiprocessing.Event()
+        event = multiprocess.Event()
         event.clear()
 
         self.codechecker_cfg['viewer_port'] = env.get_free_port()

web/tests/functional/db_cleanup/test_db_cleanup.py

@@ -12,7 +12,6 @@
 
 
 import json
-import multiprocessing
 import os
 import shutil
 import unittest
@@ -24,6 +23,7 @@
 
 from libtest import codechecker
 from libtest import env
+import multiprocess
 
 
 class TestDbCleanup(unittest.TestCase):
@@ -187,7 +187,7 @@ def __check_serverity_of_reports(self, run_name):
                              severity_id)
 
     def test_garbage_file_collection(self):
-        event = multiprocessing.Event()
+        event = multiprocess.Event()
         event.clear()
 
         self.codechecker_cfg['viewer_port'] = env.get_free_port()

web/tests/functional/instance_manager/test_instances.py

@@ -11,7 +11,6 @@
 """
 
 
-import multiprocessing
 import os
 import shutil
 import subprocess
@@ -22,10 +21,11 @@
 
 from libtest import env
 from libtest.codechecker import start_server
+import multiprocess
 
 # Stopping events for CodeChecker servers.
-EVENT_1 = multiprocessing.Event()
-EVENT_2 = multiprocessing.Event()
+EVENT_1 = multiprocess.Event()
+EVENT_2 = multiprocess.Event()
 
 
 class TestInstances(unittest.TestCase):

web/tests/functional/products/test_config_db_share.py

@@ -16,7 +16,6 @@
 from . import setup_class_common, teardown_class_common
 
 from copy import deepcopy
-import multiprocessing
 import os
 import shutil
 import unittest
@@ -30,9 +29,10 @@
 
 from libtest import codechecker
 from libtest import env
+import multiprocess
 
 # Stopping events for CodeChecker server.
-EVENT = multiprocessing.Event()
+EVENT = multiprocess.Event()
 
 
 class TestProductConfigShare(unittest.TestCase):

web/tests/functional/ssl/test_ssl.py

@@ -10,7 +10,6 @@
 SSL test.
 """
 
-import multiprocessing
 import os
 import shutil
 import subprocess
@@ -20,6 +19,7 @@
 
 from libtest import codechecker
 from libtest import env
+import multiprocess
 
 
 class TestSSL(unittest.TestCase):
@@ -32,7 +32,7 @@ def setup_class(self):
 
         # Stopping event for CodeChecker server.
         global __STOP_SERVER
-        __STOP_SERVER = multiprocessing.Event()
+        __STOP_SERVER = multiprocess.Event()
 
         global TEST_WORKSPACE
         TEST_WORKSPACE = env.get_workspace('ssl')

web/tests/functional/storage_of_analysis_statistics/test_storage_of_analysis_statistics.py

@@ -11,7 +11,6 @@
 """
 
 
-import multiprocessing
 import json
 import os
 import shutil
@@ -25,6 +24,7 @@
 
 from libtest import codechecker
 from libtest import env
+import multiprocess
 
 
 def extract(zip_file, output_dir):
@@ -48,7 +48,7 @@ def setup_class(self):
 
         # Stopping event for CodeChecker server.
         global EVENT_1
-        EVENT_1 = multiprocessing.Event()
+        EVENT_1 = multiprocess.Event()
 
         global TEST_WORKSPACE
         TEST_WORKSPACE = env.get_workspace('storage_of_analysis_statistics')

web/tests/libtest/codechecker.py

@@ -11,14 +11,15 @@
 
 
 import json
-import multiprocessing
 import os
 import shlex
 import stat
 import subprocess
 from subprocess import CalledProcessError
 import time
 
+import multiprocess
+
 from codechecker_api_shared.ttypes import Permission
 
 from codechecker_client.product import create_product_url
@@ -699,7 +700,7 @@ def start_server_proc(event, server_cmd, checking_env):
                           pg_config,
                           server_args or [])
 
-    server_proc = multiprocessing.Process(
+    server_proc = multiprocess.Process(
         name='server',
         target=start_server_proc,
         args=(event, server_cmd, codechecker_cfg['check_env']))