diff --git a/bin/CodeChecker.py b/bin/CodeChecker.py index 72800ce666..d7a177e5ae 100755 --- a/bin/CodeChecker.py +++ b/bin/CodeChecker.py @@ -31,31 +31,6 @@ analyzers = ' '.join(list(analyzer_types.supported_analyzers)) -class OrderedCheckersAction(argparse.Action): - """ - Action to store enabled and disabled checkers - and keep ordering from command line. - - Create separate lists based on the checker names for - each analyzer. - """ - - def __init__(self, option_strings, dest, nargs=None, **kwargs): - if nargs is not None: - raise ValueError("nargs not allowed") - super(OrderedCheckersAction, self).__init__(option_strings, dest, - **kwargs) - - def __call__(self, parser, namespace, value, option_string=None): - - if 'ordered_checkers' not in namespace: - namespace.ordered_checkers = [] - ordered_checkers = namespace.ordered_checkers - ordered_checkers.append((value, self.dest == 'enable')) - - namespace.ordered_checkers = ordered_checkers - - # ----------------------------------------------------------------------------- class DeprecatedOptionAction(argparse.Action): """ @@ -112,45 +87,6 @@ def add_database_arguments(parser): help='Database user name.') -# TODO: Superseded by libcodechecker/analyze.py -def add_analyzer_arguments(parser): - """ - Analyzer related arguments. - """ - parser.add_argument('-e', '--enable', - default=argparse.SUPPRESS, - action=OrderedCheckersAction, - help='Enable checker.') - parser.add_argument('-d', '--disable', - default=argparse.SUPPRESS, - action=OrderedCheckersAction, - help='Disable checker.') - parser.add_argument('--keep-tmp', action="store_true", - dest="keep_tmp", required=False, - help="Keep temporary report files " - "generated during the analysis.") - - parser.add_argument('--analyzers', nargs='+', - dest="analyzers", required=False, - default=[analyzer_types.CLANG_SA, - analyzer_types.CLANG_TIDY], - help="Select which analyzer should be enabled.\n" - "Currently supported analyzers are: " + - analyzers + "\ne.g. '--analyzers " + analyzers + "'") - - parser.add_argument('--saargs', dest="clangsa_args_cfg_file", - required=False, default=argparse.SUPPRESS, - help="File with arguments which will be forwarded " - "directly to the Clang static analyzer " - "without modification.") - - parser.add_argument('--tidyargs', dest="tidy_args_cfg_file", - required=False, default=argparse.SUPPRESS, - help="File with arguments which will be forwarded " - "directly to the Clang tidy analyzer " - "without modification.") - - def main(subcommands=None): """ CodeChecker main command line. @@ -216,11 +152,6 @@ def _warn_deprecated_command(cmd_name): workspace_help_msg = 'Directory where the CodeChecker can' \ ' store analysis related data.' - name_help_msg = 'Name of the analysis.' - - jobs_help_msg = 'Number of jobs. ' \ - 'Start multiple processes for faster analysis.' - # -------------------------------------- # Checkers parser. checker_p = subparsers.add_parser('checkers', @@ -336,55 +267,6 @@ def _warn_deprecated_command(cmd_name): logger.add_verbose_arguments(debug_parser) debug_parser.set_defaults(func=arg_handler.handle_debug) - # -------------------------------------- - # Plist parser. - plist_parser = subparsers.add_parser('plist', - formatter_class=ADHF, - help='Parse plist files in ' - 'the given directory and ' - 'store them to the database ' - 'or print to the standard ' - 'output.') - old_subcommands.append('plist') - - plist_parser.add_argument('-w', '--workspace', type=str, - dest="workspace", - default=util.get_default_workspace(), - help=workspace_help_msg) - - plist_parser.add_argument('-n', '--name', type=str, - dest="name", required=True, - default=argparse.SUPPRESS, - help=name_help_msg) - - plist_parser.add_argument('-d', '--directory', type=str, - dest="directory", required=True, - help='Path of a directory containing plist ' - 'files to parse.') - - plist_parser.add_argument('-j', '--jobs', type=int, dest="jobs", - default=1, required=False, - help=jobs_help_msg) - - plist_parser.add_argument('-s', '--steps', action="store_true", - dest="print_steps", help='Print steps.') - - plist_parser.add_argument('--stdout', action="store_true", - dest="stdout", - required=False, default=False, - help='Print results to stdout instead of ' - 'storing to the database.') - - plist_parser.add_argument('--force', action="store_true", - dest="force", default=False, required=False, - help='Delete analysis results form the ' - 'database if a run with the given ' - 'name already exists.') - - add_database_arguments(plist_parser) - logger.add_verbose_arguments(plist_parser) - plist_parser.set_defaults(func=arg_handler.handle_plist) - # -------------------------------------- # Package version info. version_parser = subparsers.add_parser('version', diff --git a/bin/codechecker-plist b/bin/codechecker-plist new file mode 100644 index 0000000000..c3b71b24ee --- /dev/null +++ b/bin/codechecker-plist @@ -0,0 +1,3 @@ +# DO_NOT_INSTALL_TO_PATH +This file marks 'CodeChecker plist' to be a valid command, but prohibits +use as 'codechecker-plist'. diff --git a/docs/user_guide.md b/docs/user_guide.md index fdb73b2683..74cf50bfc8 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -953,16 +953,6 @@ positional arguments: optional arguments: -h, --help show this help message and exit ~~~~~~~~~~~~~~~~~~~~~ -## 6. plist mode: -Clang Static Analyzer's scan-build script can generate analyis output into plist xml files. -In this You can import these files into the database. -You will need to specify containing the plist files afther the -d option. - -Example: -~~~~~~~~~~~~~~~~~~~~~ -CodeChecker plist -d ./results_plist -n myresults -~~~~~~~~~~~~~~~~~~~~~ - ## 7. debug mode: diff --git a/libcodechecker/analyze/analyzer.py b/libcodechecker/analyze/analyzer.py index 82230f1280..60f032332f 100644 --- a/libcodechecker/analyze/analyzer.py +++ b/libcodechecker/analyze/analyzer.py @@ -4,17 +4,13 @@ # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- """ -Prepare and start different analisys types +Prepare and start different analysis types """ import copy -import json -import os import shlex import subprocess -import sys import time -from libcodechecker import client from libcodechecker.logger import LoggerFactory from libcodechecker.analyze import analysis_manager from libcodechecker.analyze import analyzer_env diff --git a/libcodechecker/arg_handler.py b/libcodechecker/arg_handler.py index 31f2b55bb7..d1d3f45f12 100644 --- a/libcodechecker/arg_handler.py +++ b/libcodechecker/arg_handler.py @@ -4,18 +4,14 @@ # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- """ -Handle command line arguments. +Handle old-style subcommand invocation. """ import errno import json import multiprocessing import os -import psutil -import socket -import shutil import socket import sys -import tempfile from libcodechecker import client from libcodechecker import debug_reporter @@ -24,13 +20,10 @@ from libcodechecker import host_check from libcodechecker import session_manager from libcodechecker import util -from libcodechecker.analyze import analyzer from libcodechecker.analyze import analyzer_env -from libcodechecker.analyze import log_parser from libcodechecker.analyze.analyzers import analyzer_types from libcodechecker.database_handler import SQLServer from libcodechecker.log import build_action -from libcodechecker.log import build_manager from libcodechecker.logger import LoggerFactory from libcodechecker.server import client_db_access_server from libcodechecker.server import instance_manager @@ -244,81 +237,6 @@ def handle_debug(args): args.force) -def consume_plist(item): - plist, args, context = item - LOG.info('Consuming ' + plist) - - action = build_action.BuildAction() - action.analyzer_type = analyzer_types.CLANG_SA - action.original_command = 'Imported from PList directly' - - rh = analyzer_types.construct_result_handler(args, - action, - context.run_id, - args.directory, - context.severity_map, - None, - None, - not args.stdout) - - rh.analyzer_returncode = 0 - rh.buildaction.analyzer_type = 'Build action from plist' - rh.buildaction.original_command = plist - rh.analyzer_cmd = '' - rh.analyzed_source_file = '' # TODO: fill from plist. - rh.result_file = os.path.join(args.directory, plist) - rh.handle_results() - - -def handle_plist(args): - context = generic_package_context.get_context() - context.codechecker_workspace = args.workspace - context.db_username = args.dbusername - - if not args.stdout: - args.workspace = os.path.realpath(args.workspace) - if not os.path.isdir(args.workspace): - os.mkdir(args.workspace) - - check_env = analyzer_env.get_check_env(context.path_env_extra, - context.ld_lib_path_extra) - - sql_server = SQLServer.from_cmdline_args(args, - context.migration_root, - check_env) - - conn_mgr = client.ConnectionManager(sql_server, - 'localhost', - util.get_free_port()) - - sql_server.start(context.db_version_info, wait_for_start=True, - init=True) - - conn_mgr.start_report_server() - - with client.get_connection() as connection: - context.run_id = connection.add_checker_run(' '.join(sys.argv), - args.name, - context.version, - args.force) - - pool = multiprocessing.Pool(args.jobs) - - try: - items = [(plist, args, context) - for plist in os.listdir(args.directory)] - pool.map_async(consume_plist, items, 1).get(float('inf')) - pool.close() - except Exception: - pool.terminate() - raise - finally: - pool.join() - - if not args.stdout: - log_startserver_hint(args) - - def handle_version_info(args): """ Get and print the version information from the diff --git a/libcodechecker/plist.py b/libcodechecker/plist.py new file mode 100644 index 0000000000..c6d003fa9b --- /dev/null +++ b/libcodechecker/plist.py @@ -0,0 +1,336 @@ +# ------------------------------------------------------------------------- +# The CodeChecker Infrastructure +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# ------------------------------------------------------------------------- +""" +Implements the old command "CodeChecker plist", a branching wrapper over the +functionality of 'store' and 'parse'. +""" + +import argparse +import imp +import os +import shutil + +from libcodechecker import util +from libcodechecker.analyze.analyzers import analyzer_types +from libcodechecker.logger import add_verbose_arguments +from libcodechecker.logger import LoggerFactory + +LOG = LoggerFactory.get_new_logger('PLIST') + + +class OrderedCheckersAction(argparse.Action): + """ + Action to store enabled and disabled checkers + and keep ordering from command line. + + Create separate lists based on the checker names for + each analyzer. + """ + + # Users can supply invocation to 'codechecker-analyze' as follows: + # -e core -d core.uninitialized -e core.uninitialized.Assign + # We must support having multiple '-e' and '-d' options and the order + # specified must be kept when the list of checkers are assembled for Clang. + + def __init__(self, option_strings, dest, nargs=None, **kwargs): + if nargs is not None: + raise ValueError("nargs not allowed") + super(OrderedCheckersAction, self).__init__(option_strings, dest, + **kwargs) + + def __call__(self, parser, namespace, value, option_string=None): + + if 'ordered_checkers' not in namespace: + namespace.ordered_checkers = [] + ordered_checkers = namespace.ordered_checkers + ordered_checkers.append((value, self.dest == 'enable')) + + namespace.ordered_checkers = ordered_checkers + + +class DeprecatedOptionAction(argparse.Action): + """ + Deprecated argument action. + """ + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + super(DeprecatedOptionAction, self). \ + __init__(option_strings, + dest, + const='deprecated_option', + default=argparse.SUPPRESS, + type=None, + choices=None, + required=False, + help="(Usage of this argument is DEPRECATED and has no " + "effect!)", + metavar='') + + def __call__(self, parser, namespace, value=None, option_string=None): + LOG.warning("Deprecated command line option used: '" + + option_string + "'") + + +def get_argparser_ctor_args(): + """ + This method returns a dict containing the kwargs for constructing an + argparse.ArgumentParser (either directly or as a subparser). + """ + + return { + 'prog': 'CodeChecker plist', + 'formatter_class': argparse.ArgumentDefaultsHelpFormatter, + + # Description is shown when the command's help is queried directly + 'description': "Parse plist files in the given directory and " + "store the defects found therein to the database " + "or print to the standard output.", + + # Help is shown when the "parent" CodeChecker command lists the + # individual subcommands. + 'help': "Use plist files in a given directory to pretty-print or " + "store the results." + } + + +def add_arguments_to_parser(parser): + """ + Add the subcommand's arguments to the given argparse.ArgumentParser. + """ + + # TODO: --name does not exist in 'parse'. + # In 'store', --name is not a required argument by argparse, as 'analyze' + # can prepare a name, which is read after 'store' is started. + # If the name is missing, the user is explicitly warned. + # TODO: This should be an optional argument here too. Also it doesn't make + # sense to require --name if --stdout ('parse'-mode) is set. + parser.add_argument('-n', '--name', + type=str, + dest="name", + required=True, + default=argparse.SUPPRESS, + help="The name of the analysis run to use in storing " + "the reports to the database. If not specified, " + "the '--name' parameter given to 'codechecker-" + "analyze' will be used, if exists.") + + # TODO: This argument is without an opt-string in 'store' and 'parse'. + parser.add_argument('-d', '--directory', + type=str, + dest="directory", + required=True, + help="Path of the directory where the plist files " + "to be used are found.") + + # TODO: Workspace is no longer a concept in the new subcommands. + parser.add_argument('-w', '--workspace', + type=str, + default=util.get_default_workspace(), + dest="workspace", + help="Directory where CodeChecker can store analysis " + "related data, such as the database.") + + parser.add_argument('-f', '--force', + dest="force", + default=False, + action='store_true', + required=False, + help="Delete analysis results stored in the database " + "for the current analysis run's name and store " + "only the results reported in the 'input' files. " + "(By default, CodeChecker would keep reports that " + "were coming from files not affected by the " + "analysis, and only incrementally update defect " + "reports for source files that were analysed.)") + + parser.add_argument('-j', '--jobs', + type=int, + dest="jobs", + required=False, + default=1, + help="Number of threads to use in parsing and storing " + "of results. More threads mean faster analysis " + "at the cost of using more memory.") + + # TODO: Parse does not know '-s' or '--steps' for this. + parser.add_argument('-s', '--steps', '--print-steps', + dest="print_steps", + action="store_true", + required=False, + help="Print the steps the analyzers took in finding " + "the reported defect.") + + parser.add_argument('--stdout', + dest="stdout", + action='store_true', + required=False, + default=False, + help="Print the analysis results to the standard " + "output instead of storing to the database.") + + dbmodes = parser.add_argument_group("database arguments") + + dbmodes = dbmodes.add_mutually_exclusive_group(required=False) + + # SQLite is the default, and for 'check', it was deprecated. + # TODO: In 'store', --sqlite has been replaced as an option to specify the + # .sqlite file, essentially replacing the concept of 'workspace'. + dbmodes.add_argument('--sqlite', + action=DeprecatedOptionAction) + + dbmodes.add_argument('--postgresql', + dest="postgresql", + action='store_true', + required=False, + default=argparse.SUPPRESS, + help="Specifies that a PostgreSQL database is to be " + "used instead of SQLite. See the \"PostgreSQL " + "arguments\" section on how to configure the " + "database connection.") + + pgsql = parser.add_argument_group("PostgreSQL arguments", + "Values of these arguments are ignored, " + "unless '--postgresql' is specified!") + + # WARNING: '--dbaddress' default value influences workspace creation + # in SQLite. + # TODO: These are '--db-something' in 'store', not '--dbsomething'. + pgsql.add_argument('--dbaddress', + type=str, + dest="dbaddress", + default="localhost", + required=False, + help="Database server address.") + + pgsql.add_argument('--dbport', + type=int, + dest="dbport", + default=5432, + required=False, + help="Database server port.") + + pgsql.add_argument('--dbusername', + type=str, + dest="dbusername", + default='codechecker', + required=False, + help="Username to use for connection.") + + pgsql.add_argument('--dbname', + type=str, + dest="dbname", + default="codechecker", + required=False, + help="Name of the database to use.") + + add_verbose_arguments(parser) + parser.set_defaults(func=main) + + +def main(args): + """ + Execute a wrapper over 'parse' or 'store'. + """ + + # Load the 'libcodechecker' module and acquire its path. + file, path, descr = imp.find_module("libcodechecker") + libcc_path = imp.load_module("libcodechecker", + file, path, descr).__path__[0] + + def __load_module(name): + """Loads the given subcommand's definition from the libs.""" + module_file = os.path.join(libcc_path, name.replace('-', '_') + ".py") + try: + module = imp.load_source(name, module_file) + except ImportError: + LOG.error("Couldn't import subcommand '" + name + "'.") + raise + + return module + + def __update_if_key_exists(source, target, key): + """Append the source Namespace's element with 'key' to target with + the same key, but only if it exists.""" + if key in source: + setattr(target, key, getattr(source, key)) + + try: + if args.stdout: + # --- Parse mode --- + parse_args = argparse.Namespace( + input=[args.directory], + print_steps=args.print_steps + ) + + parse_module = __load_module("parse") + __update_if_key_exists(args, parse_args, "verbose") + + LOG.debug("Calling PARSE with args:") + LOG.debug(parse_args) + parse_module.main(parse_args) + else: + # --- Store mode --- + workspace = os.path.abspath(args.workspace) + if not os.path.isdir(workspace): + os.makedirs(workspace) + + store_args = argparse.Namespace( + input=[args.directory], + input_format='plist', + jobs=args.jobs, + force=args.force, + dbaddress=args.dbaddress, + dbport=args.dbport, + dbusername=args.dbusername, + dbname=args.dbname + ) + # Some arguments don't have default values. + # We can't set these keys to None because it would result in an + # error after the call. + if 'postgresql' in args: + __update_if_key_exists(args, store_args, 'postgresql') + else: + # If we are saving to a SQLite database, the wrapped 'check' + # command used to do it in the workspace folder. + setattr(store_args, 'sqlite', os.path.join(workspace, + 'codechecker.sqlite')) + args_to_update = ['suppress', + 'name' + ] + for key in args_to_update: + __update_if_key_exists(args, store_args, key) + + store_module = __load_module("store") + __update_if_key_exists(args, store_args, "verbose") + + LOG.debug("Calling STORE with args:") + LOG.debug(store_args) + store_module.main(store_args) + + # Show a hint for server start. + db_data = "" + if 'postgresql' in args: + db_data += " --postgresql" \ + + " --dbname " + args.dbname \ + + " --dbport " + str(args.dbport) \ + + " --dbusername " + args.dbusername + + LOG.info("To view results run:\nCodeChecker server -w " + + args.workspace + db_data) + except ImportError: + LOG.error("Check failed: couldn't import a library.") + except Exception as ex: + LOG.error("Running check failed. " + ex.message)