Skip to content

Commit

Permalink
Merge pull request Ericsson#3807 from bruntib/singleton_context
Browse files Browse the repository at this point in the history 
[refactor] Analyzer context as singleton
  • Loading branch information
@vodorok
vodorok authored Jan 20, 2023
2 parents a195af7 + dcc47a5 commit 5f4d503
Show file tree
Hide file tree
Showing 34 changed files with 261 additions and 445 deletions.
45 changes: 10 additions & 35 deletions analyzer/codechecker_analyzer/analysis_manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import glob
import multiprocessing
import os
import pickle
import shlex
import shutil
import signal
Expand All @@ -25,7 +24,6 @@

import psutil

from codechecker_analyzer import env
from codechecker_common.logger import get_logger

from codechecker_statistics_collector.collectors.special_return_value import \
Expand Down Expand Up @@ -60,8 +58,7 @@ def print_analyzer_statistic_summary(metadata_analyzers, status, msg=None):
LOG.info(" %s: %s", analyzer_type, res)


def worker_result_handler(results, metadata_tool, output_path,
analyzer_binaries):
def worker_result_handler(results, metadata_tool, output_path):
""" Print the analysis summary. """
skipped_num = 0
reanalyzed_num = 0
Expand Down Expand Up @@ -169,7 +166,7 @@ def is_ctu_active(source_analyzer):
source_analyzer.is_ctu_enabled()


def prepare_check(action, analyzer_config, output_dir, checker_labels,
def prepare_check(action, analyzer_config, output_dir,
skip_handlers, statistics_data, disable_ctu=False):
""" Construct the source analyzer and result handler. """
# Create a source analyzer.
Expand Down Expand Up @@ -209,7 +206,6 @@ def prepare_check(action, analyzer_config, output_dir, checker_labels,
# which only returns metadata, but can't process the results.
rh = source_analyzer.construct_result_handler(action,
output_dir,
checker_labels,
skip_handlers)

# NOTICE!
Expand Down Expand Up @@ -488,10 +484,10 @@ def check(check_data):
skiplist handler is None if no skip file was configured.
"""
actions_map, action, context, analyzer_config, \
actions_map, action, analyzer_config, \
output_dir, skip_handlers, quiet_output_on_stdout, \
capture_analysis_output, generate_reproducer, analysis_timeout, \
analyzer_environment, ctu_reanalyze_on_failure, \
ctu_reanalyze_on_failure, \
output_dirs, statistics_data = check_data

failed_dir = output_dirs["failed"]
Expand All @@ -509,7 +505,7 @@ def check(check_data):
raise Exception("Analyzer configuration is missing.")

source_analyzer, rh = prepare_check(action, analyzer_config,
output_dir, context.checker_labels,
output_dir,
skip_handlers, statistics_data)

reanalyzed = os.path.exists(rh.analyzer_result_file)
Expand Down Expand Up @@ -545,20 +541,8 @@ def __create_timeout(analyzer_process):

result_file_exists = os.path.exists(rh.analyzer_result_file)

# FIXME: cppcheck (and other analyzers)
# need the original env when invoked
# we should handle this in a generic way.
if "cppcheck" in os.path.basename(analyzer_cmd[0]):
original_env_file = os.environ.get(
'CODECHECKER_ORIGINAL_BUILD_ENV')
if original_env_file:
with open(original_env_file, 'rb') as env_file:
analyzer_environment = \
pickle.load(env_file, encoding='utf-8')

# Fills up the result handler with the analyzer information.
source_analyzer.analyze(analyzer_cmd, rh, analyzer_environment,
__create_timeout)
source_analyzer.analyze(analyzer_cmd, rh, __create_timeout)

# If execution reaches this line, the analyzer process has quit.
if timeout_cleanup[0]():
Expand Down Expand Up @@ -652,7 +636,7 @@ def handle_analysis_result(success, zip_file=zip_file):
# Try to reanalyze with CTU disabled.
source_analyzer, rh = \
prepare_check(action, analyzer_config,
output_dir, context.checker_labels,
output_dir,
skip_handlers, statistics_data,
True)
reanalyzed = os.path.exists(rh.analyzer_result_file)
Expand All @@ -662,9 +646,7 @@ def handle_analysis_result(success, zip_file=zip_file):

# Fills up the result handler with
# the analyzer information.
source_analyzer.analyze(analyzer_cmd,
rh,
analyzer_environment)
source_analyzer.analyze(analyzer_cmd, rh)

return_codes = rh.analyzer_returncode
if rh.analyzer_returncode == 0:
Expand Down Expand Up @@ -733,7 +715,7 @@ def skip_cpp(compile_actions, skip_handlers):
return analyze, skip


def start_workers(actions_map, actions, context, analyzer_config_map,
def start_workers(actions_map, actions, analyzer_config_map,
jobs, output_path, skip_handlers, metadata_tool,
quiet_analyze, capture_analysis_output, generate_reproducer,
timeout, ctu_reanalyze_on_failure, statistics_data, manager,
Expand Down Expand Up @@ -791,21 +773,15 @@ def signal_handler(signum, frame):
'reproducer': reproducer_dir,
'ctu_connections': ctu_connections_dir}

# Construct analyzer env.
analyzer_environment = env.extend(context.path_env_extra,
context.ld_lib_path_extra)

analyzed_actions = [(actions_map,
build_action,
context,
analyzer_config_map.get(build_action.analyzer_type),
output_path,
skip_handlers,
quiet_analyze,
capture_analysis_output,
generate_reproducer,
timeout,
analyzer_environment,
ctu_reanalyze_on_failure,
output_dirs,
statistics_data)
Expand All @@ -829,8 +805,7 @@ def signal_handler(signum, frame):
analyzed_actions,
1,
callback=lambda results: worker_result_handler(
results, metadata_tool, output_path,
context.analyzer_binaries)
results, metadata_tool, output_path)
).get(timeout)

pool.close()
Expand Down
60 changes: 11 additions & 49 deletions analyzer/codechecker_analyzer/analyzer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,8 @@
from collections import defaultdict
from multiprocessing.managers import SyncManager
import os
import shlex
import shutil
import signal
import subprocess
import time

from codechecker_common.logger import get_logger
Expand All @@ -26,7 +24,8 @@
from codechecker_statistics_collector.collectors.return_value import \
ReturnValueCollector

from . import analysis_manager, pre_analysis_manager, env, checkers
from . import analyzer_context, analysis_manager, pre_analysis_manager, \
checkers
from .analyzers import analyzer_types
from .analyzers.config_handler import CheckerState
from .analyzers.clangsa.analyzer import ClangSA
Expand Down Expand Up @@ -69,41 +68,6 @@ def create_actions_map(actions, manager):
return result


def __get_analyzer_version(context, analyzer_config_map):
"""
Get the path and the version of the analyzer binaries.
"""
check_env = env.extend(context.path_env_extra,
context.ld_lib_path_extra)

# Get the analyzer binaries from the config_map which
# contains only the checked and available analyzers.
versions = {}
for _, analyzer_cfg in analyzer_config_map.items():
analyzer_bin = analyzer_cfg.analyzer_binary
version = [analyzer_bin, ' --version']
try:
output = subprocess.check_output(
shlex.split(
' '.join(version)),
env=check_env,
universal_newlines=True,
encoding="utf-8",
errors="ignore")
versions[analyzer_bin] = output
except subprocess.CalledProcessError as oerr:
LOG.warning("Failed to get analyzer version: %s",
' '.join(version))
LOG.warning(oerr.output)
LOG.warning(oerr.stderr)
except OSError as oerr:
LOG.warning("Failed to get analyzer version: %s",
' '.join(version))
LOG.warning(oerr.strerror)

return versions


def __mgr_init():
"""
This function is set for the SyncManager object which handles shared data
Expand Down Expand Up @@ -159,14 +123,16 @@ def __get_ctu_data(config_map, ctu_dir):
'ctu_temp_fnmap_folder': 'tmpExternalFnMaps'}


def perform_analysis(args, skip_handlers, context, actions, metadata_tool,
def perform_analysis(args, skip_handlers, actions, metadata_tool,
compile_cmd_count):
"""
Perform static analysis via the given (or if not, all) analyzers,
in the given analysis context for the supplied build actions.
Additionally, insert statistical information into the metadata dict.
"""

context = analyzer_context.get_context()

ctu_reanalyze_on_failure = 'ctu_reanalyze_on_failure' in args and \
args.ctu_reanalyze_on_failure
if ctu_reanalyze_on_failure:
Expand All @@ -176,8 +142,7 @@ def perform_analysis(args, skip_handlers, context, actions, metadata_tool,

analyzers = args.analyzers if 'analyzers' in args \
else analyzer_types.supported_analyzers
analyzers, errored = analyzer_types.check_supported_analyzers(
analyzers, context)
analyzers, errored = analyzer_types.check_supported_analyzers(analyzers)
analyzer_types.check_available_analyzers(analyzers, errored)

ctu_collect = False
Expand All @@ -199,7 +164,7 @@ def perform_analysis(args, skip_handlers, context, actions, metadata_tool,
return

actions = prepare_actions(actions, analyzers)
config_map = analyzer_types.build_config_handlers(args, context, analyzers)
config_map = analyzer_types.build_config_handlers(args, analyzers)

available_checkers = set()
# Add profile names to the checkers list so we will not warn
Expand Down Expand Up @@ -241,9 +206,6 @@ def perform_analysis(args, skip_handlers, context, actions, metadata_tool,
config_map[ClangSA.ANALYZER_NAME].set_checker_enabled(
ReturnValueCollector.checker_collect, False)

check_env = env.extend(context.path_env_extra,
context.ld_lib_path_extra)

enabled_checkers = defaultdict(list)

# Save some metadata information.
Expand All @@ -264,7 +226,8 @@ def perform_analysis(args, skip_handlers, context, actions, metadata_tool,
if state == CheckerState.enabled:
enabled_checkers[analyzer].append(check)

version = config_map[analyzer].get_version(check_env)
# TODO: cppcheck may require a different environment than clang.
version = config_map[analyzer].get_version(context.analyzer_env)
metadata_info['analyzer_statistics']['version'] = version

metadata_tool['analyzers'][analyzer] = metadata_info
Expand All @@ -280,7 +243,7 @@ def perform_analysis(args, skip_handlers, context, actions, metadata_tool,
ctu_data = __get_ctu_data(config_map, ctu_dir)

makefile_creator = MakeFileCreator(analyzers, args.output_path,
config_map, context, skip_handlers,
config_map, skip_handlers,
ctu_collect, statistics_data,
ctu_data)
makefile_creator.create(actions)
Expand Down Expand Up @@ -326,7 +289,6 @@ def perform_analysis(args, skip_handlers, context, actions, metadata_tool,

if clangsa_config is not None:
pre_analysis_manager.run_pre_analysis(pre_analyze,
context,
clangsa_config,
args.jobs,
pre_anal_skip_handlers,
Expand All @@ -346,7 +308,7 @@ def perform_analysis(args, skip_handlers, context, actions, metadata_tool,
if ctu_analyze or statistics_data or (not ctu_analyze and not ctu_collect):

LOG.info("Starting static analysis ...")
analysis_manager.start_workers(actions_map, actions, context,
analysis_manager.start_workers(actions_map, actions,
config_map, args.jobs,
args.output_path,
skip_handlers,
Expand Down
20 changes: 17 additions & 3 deletions analyzer/codechecker_analyzer/analyzer_context.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,14 @@

# -----------------------------------------------------------------------------
class Context(metaclass=Singleton):
""" Generic package specific context. """
"""
Generic package specific context.
This class is to query all information that might be important about the
running environment around CodeChecker analysis. This is a singleton
object, so it is cheap to construct and can be used as a read-only
dictionary of the data on its interface.
"""

def __init__(self):
""" Initialize analyzer context. """
Expand All @@ -54,6 +61,7 @@ def __init__(self):
self.__package_build_date = None
self.__package_git_hash = None
self.__analyzers = {}
self.__analyzer_env = None

self.logger_lib_dir_path = os.path.join(
self._data_files_dir_path, 'ld_logger', 'lib')
Expand Down Expand Up @@ -146,8 +154,7 @@ def __populate_analyzers(self):
analyzer_env = None
analyzer_from_path = env.is_analyzer_from_path()
if not analyzer_from_path:
analyzer_env = env.extend(self.path_env_extra,
self.ld_lib_path_extra)
analyzer_env = self.analyzer_env

compiler_binaries = self.pckg_layout.get('analyzers')
for name, value in compiler_binaries.items():
Expand Down Expand Up @@ -257,6 +264,13 @@ def ld_lib_path_extra(self):
ld_paths.append(os.path.join(self._data_files_dir_path, path))
return ld_paths

@property
def analyzer_env(self):
if not self.__analyzer_env:
self.__analyzer_env = \
env.extend(self.path_env_extra, self.ld_lib_path_extra)
return self.__analyzer_env

@property
def analyzer_binaries(self):
return self.__analyzers
Expand Down
Loading

0 comments on commit 5f4d503

Please sign in to comment.