diff --git a/bin/CodeChecker.py b/bin/CodeChecker.py index 5797542466..f7413fbf63 100755 --- a/bin/CodeChecker.py +++ b/bin/CodeChecker.py @@ -18,10 +18,6 @@ from shared.ttypes import RequestFailed from libcodechecker import libhandlers -from libcodechecker.logger import LoggerFactory - - -LOG = LoggerFactory.get_new_logger('MAIN') def main(subcommands=None): @@ -67,7 +63,7 @@ def signal_handler(sig, frame): In this case, no database is used, and the results are printed on the standard output. - CodeChecker check -b "cd ~/myproject && make\"""") + CodeChecker check -b "cd ~/myproject && make\" """) subparsers = parser.add_subparsers(help='commands') @@ -78,37 +74,33 @@ def signal_handler(sig, frame): if len(sys.argv) > 1: first_command = sys.argv[1] if first_command in subcommands: - LOG.debug("Supplied an existing, valid subcommand: " + - first_command) # Consider only the given command as an available one. subcommands = [first_command] for subcommand in subcommands: - LOG.debug("Creating arg parser for subcommand " + subcommand) try: libhandlers.add_subcommand(subparsers, str(subcommand)) except (IOError, ImportError): - LOG.warning("Couldn't import module for subcommand '" + - subcommand + "'... ignoring.") + print("Couldn't import module for subcommand '" + + subcommand + "'... ignoring.") import traceback traceback.print_exc(file=sys.stdout) args = parser.parse_args() - if 'verbose' in args: - LoggerFactory.set_log_level(args.verbose) + args.func(args) except KeyboardInterrupt as kb_err: - LOG.info(str(kb_err)) - LOG.info("Interrupted by user...") + print(str(kb_err)) + print("Interrupted by user...") sys.exit(1) except RequestFailed as thrift_ex: - LOG.info("Server error.") - LOG.info("Error code: " + str(thrift_ex.errorCode)) - LOG.info("Error message: " + str(thrift_ex.message)) + print("Server error.") + print("Error code: " + str(thrift_ex.errorCode)) + print("Error message: " + str(thrift_ex.message)) sys.exit(1) # Handle all exception, but print stacktrace. It is needed for atexit. @@ -122,10 +114,6 @@ def signal_handler(sig, frame): # ----------------------------------------------------------------------------- if __name__ == "__main__": - LOG.debug(sys.path) - LOG.debug(sys.version) - LOG.debug(sys.executable) - LOG.debug(os.environ.get('LD_LIBRARY_PATH')) # Load the available CodeChecker subcommands. # This list is generated dynamically by scripts/build_package.py, and is @@ -135,7 +123,4 @@ def signal_handler(sig, frame): with open(commands_cfg) as cfg_file: commands = json.load(cfg_file) - LOG.debug("Available CodeChecker subcommands: ") - LOG.debug(commands) - main(commands) diff --git a/config/logger.conf b/config/logger.conf new file mode 100644 index 0000000000..366156e1f2 --- /dev/null +++ b/config/logger.conf @@ -0,0 +1,56 @@ +{ + "version": 1, + "disable_existing_loggers": true, + "formatters": { + "brief": { + "format": "[%(asctime)s] - %(message)s", + "datefmt": "%Y-%m-%d %H:%M" + }, + "precise": { + "format": "[%(levelname)s][%(asctime)s] {%(name)s} [%(process)d] <%(thread)d> - %(filename)s:%(lineno)d %(funcName)s() - %(message)s", + "datefmt": "%Y-%m-%d %H:%M:%S" + } + }, + "loggers": { + "analyzer": { + "level": "INFO", + "handlers": ["console"] + }, + "analyzer.clangsa": { + "level": "INFO", + "handlers": ["console"] + }, + "analyzer.tidy": { + "level": "INFO", + "handlers": ["console"] + }, + "buildlogger": { + "level": "INFO", + "handlers": ["console"] + }, + "profiler": { + "level": "INFO", + "handlers": ["console"] + }, + "report": { + "level": "INFO", + "handlers": ["console"] + }, + "server": { + "level": "INFO", + "handlers": ["console"] + }, + "system": { + "level": "INFO", + "handlers": ["console"] + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "brief", + "stream": "ext://sys.stdout" + } + } +} diff --git a/docs/logging.md b/docs/logging.md new file mode 100644 index 0000000000..49a5b5f174 --- /dev/null +++ b/docs/logging.md @@ -0,0 +1,138 @@ +# Log configuration + +## Command line + +Command line flag can be used to turn in CodeChecker debug mode. The different +subcommands can be given a `-v` flag which needs a parameter. Possible values +are `debug`, `debug_analyzer` and `info`. Default is `info`. + +`debug_analyzer` switches analyzer related logs on: + +~~~~~~~~~~~~~~~~~~~~~ +CodeChecker check -b --verbose debug_analyzer +~~~~~~~~~~~~~~~~~~~~~ + +Turning on CodeChecker debug level logging is possible for the most +subcommands: + +~~~~~~~~~~~~~~~~~~~~~ +CodeChecker check -b --verbose debug +CodeChecker server -v --verbose debug +~~~~~~~~~~~~~~~~~~~~~ + +## Change log configuration for the server + +The log level for a server can be changed in two ways. + +### 1. At the server start + +Debug log can be enabled like this at the server start. +``` +Codechecker server ... --verbose debug +``` + +The log levels can not be changed after the server was started. + +### 2. While the server is running + +The log configuration can be changed for a running server if it was started +in an environment where the `CC_LOG_CONFIG_PORT` environment variable is set. + +```sh +CC_LOG_CONFIG_PORT='8888' Codechecker server ... +``` +The running server will accept log configurations on the configured `8888` port. +This way the log configuration can be changed without a server restart. + +#### Sending new configuration to a running server + +Sending a new configuration to the server can be done easily with +`scripts/send_log_config.py` like this: +``` +./send_log_config.py -c new_log_cfg.conf -p 8888 +``` + +Sending a configuration can be done only on the same machine where the server was started. See further +details [here](https://docs.python.org/2/library/logging.config.html#logging.config.listen). + +## Example debug log configuration with multiple handlers + +For more information see: [logging-cookbook](https://docs.python.org/2/howto/logging-cookbook.html) or +[logging.config](https://docs.python.org/2/library/logging.config.html) + +```json +{ + "version": 1, + "disable_existing_loggers": true, + "formatters": { + "brief": { + "format": "[%(asctime)s] - %(message)s", + "datefmt": "%Y-%m-%d %H:%M" + }, + "precise": { + "format": "[%(asctime)s] {%(name)s} [%(process)d] <%(thread)d> - %(filename)s:%(lineno)d %(funcName)s() - %(message)s", + "datefmt": "%Y-%m-%d %H:%M:%S" + } + }, + "loggers": { + "analyzer": { + "level": "DEBUG", + "handlers": [ "console", "file" ] + }, + "analyzer.tidy": { + "level": "DEBUG", + "handlers": [ "console", "file" ] + }, + "analyzer.clangsa": { + "level": "DEBUG", + "handlers": [ "console", "file" ] + }, + "server": { + "level": "DEBUG", + "handlers": [ "console", "file" , "daily_log"] + }, + "buildlogger": { + "level": "DEBUG", + "handlers": [ "console", "file" ] + }, + "report": { + "level": "DEBUG", + "handlers": [ "console", "file" ] + }, + "system": { + "level": "DEBUG", + "handlers": [ "console", "file", "daily_log" ] + }, + "profiler": { + "level": "DEBUG", + "handlers": [ "console", "file" ] + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "brief", + "stream": "ext://sys.stdout" + }, + "file": { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "precise", + "level": "DEBUG", + "filename": "file.log", + "encoding": "utf-8", + "maxBytes": 1000000, + "backupCount": 3 + }, + "daily_log": { + "class": "logging.handlers.TimedRotatingFileHandler", + "formatter": "precise", + "level": "DEBUG", + "filename": "daily.log", + "encoding": "utf-8", + "when": "D", + "interval": 1 + } + } +} +``` diff --git a/docs/user_guide.md b/docs/user_guide.md index 41d10d2335..01a9e96c8a 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -1513,23 +1513,4 @@ http://www.postgresql.org/docs/current/static/libpq-pgpass.html # Debugging CodeChecker -Command line flag can be used to turn in CodeChecker debug mode. The different -subcommands can be given a `-v` flag which needs a parameter. Possible values -are `debug`, `debug_analyzer` and `info`. Default is `info`. - -`debug_analyzer` switches analyzer related logs on: - -~~~~~~~~~~~~~~~~~~~~~ -CodeChecker check -b --verbose debug_analyzer -~~~~~~~~~~~~~~~~~~~~~ - -Turning on CodeChecker debug level logging is possible for the most -subcommands: - -~~~~~~~~~~~~~~~~~~~~~ -CodeChecker check -b --verbose debug -CodeChecker server -v --verbose debug -~~~~~~~~~~~~~~~~~~~~~ - -If debug logging is enabled and PostgreSQL database is used, PostgreSQL logs -are written to `postgresql.log` in the workspace directory. +To change the log levels check out the [logging](logging.md) documentation. diff --git a/libcodechecker/analyze/analysis_manager.py b/libcodechecker/analyze/analysis_manager.py index 3eabbb9ce6..009e83405a 100644 --- a/libcodechecker/analyze/analysis_manager.py +++ b/libcodechecker/analyze/analysis_manager.py @@ -20,9 +20,9 @@ from libcodechecker.analyze import analyzer_env from libcodechecker.analyze import plist_parser from libcodechecker.analyze.analyzers import analyzer_types -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('ANALYSIS MANAGER') +LOG = get_logger('analyzer') def worker_result_handler(results, metadata, output_path): diff --git a/libcodechecker/analyze/analyzer.py b/libcodechecker/analyze/analyzer.py index 736bacc32b..1d1ac61608 100644 --- a/libcodechecker/analyze/analyzer.py +++ b/libcodechecker/analyze/analyzer.py @@ -15,14 +15,14 @@ import subprocess import time -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger from libcodechecker.analyze import analysis_manager from libcodechecker.analyze import analyzer_env from libcodechecker.analyze import ctu_manager from libcodechecker.analyze import skiplist_handler from libcodechecker.analyze.analyzers import analyzer_types -LOG = LoggerFactory.get_new_logger('ANALYZER') +LOG = get_logger('analyzer') def prepare_actions(actions, enabled_analyzers): diff --git a/libcodechecker/analyze/analyzer_env.py b/libcodechecker/analyze/analyzer_env.py index ddd2146b69..8d11a47a81 100644 --- a/libcodechecker/analyze/analyzer_env.py +++ b/libcodechecker/analyze/analyzer_env.py @@ -7,9 +7,9 @@ import os -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('ENV') +LOG = get_logger('system') def get_log_env(logfile, context, original_env): diff --git a/libcodechecker/analyze/analyzers/analyzer_base.py b/libcodechecker/analyze/analyzers/analyzer_base.py index 087be4be58..3866f79f74 100644 --- a/libcodechecker/analyze/analyzers/analyzer_base.py +++ b/libcodechecker/analyze/analyzers/analyzer_base.py @@ -11,9 +11,9 @@ import subprocess import sys -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('ANALYZER (BASE)') +LOG = get_logger('analyzer') class SourceAnalyzer(object): diff --git a/libcodechecker/analyze/analyzers/analyzer_clang_tidy.py b/libcodechecker/analyze/analyzers/analyzer_clang_tidy.py index 3f13bf1592..02ecd84190 100644 --- a/libcodechecker/analyze/analyzers/analyzer_clang_tidy.py +++ b/libcodechecker/analyze/analyzers/analyzer_clang_tidy.py @@ -12,12 +12,12 @@ import subprocess from libcodechecker.analyze.analyzers import analyzer_base -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger from libcodechecker.util import get_binary_in_path from libcodechecker.analyze.analyzer_env import\ extend_analyzer_cmd_with_resource_dir -LOG = LoggerFactory.get_new_logger('CLANG TIDY') +LOG = get_logger('analyzer') class ClangTidy(analyzer_base.SourceAnalyzer): diff --git a/libcodechecker/analyze/analyzers/analyzer_clangsa.py b/libcodechecker/analyze/analyzers/analyzer_clangsa.py index 385db65939..0be4d98dd5 100644 --- a/libcodechecker/analyze/analyzers/analyzer_clangsa.py +++ b/libcodechecker/analyze/analyzers/analyzer_clangsa.py @@ -12,12 +12,12 @@ from libcodechecker.analyze.analyzers import analyzer_base from libcodechecker.analyze.analyzers import ctu_triple_arch from libcodechecker.analyze import analyzer_env -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger from libcodechecker.util import get_binary_in_path from libcodechecker.analyze.analyzer_env import\ extend_analyzer_cmd_with_resource_dir -LOG = LoggerFactory.get_new_logger('CLANGSA') +LOG = get_logger('analyzer') class ClangSA(analyzer_base.SourceAnalyzer): diff --git a/libcodechecker/analyze/analyzers/analyzer_types.py b/libcodechecker/analyze/analyzers/analyzer_types.py index 451492cbf1..8e60b68182 100644 --- a/libcodechecker/analyze/analyzers/analyzer_types.py +++ b/libcodechecker/analyze/analyzers/analyzer_types.py @@ -21,9 +21,9 @@ from libcodechecker.analyze.analyzers import result_handler_base from libcodechecker.analyze.analyzers import result_handler_clang_tidy from libcodechecker.analyze.analyzers import result_handler_plist_to_stdout -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('ANALYZER TYPES') +LOG = get_logger('analyzer') CLANG_SA = 'clangsa' CLANG_TIDY = 'clang-tidy' diff --git a/libcodechecker/analyze/analyzers/config_handler.py b/libcodechecker/analyze/analyzers/config_handler.py index 1f0d84a360..032341891c 100644 --- a/libcodechecker/analyze/analyzers/config_handler.py +++ b/libcodechecker/analyze/analyzers/config_handler.py @@ -8,9 +8,9 @@ import collections import os -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('CONFIG HANDLER') +LOG = get_logger('system') class AnalyzerConfigHandler(object): diff --git a/libcodechecker/analyze/analyzers/config_handler_clang_tidy.py b/libcodechecker/analyze/analyzers/config_handler_clang_tidy.py index 14f5bfc06e..6dfd4cd845 100644 --- a/libcodechecker/analyze/analyzers/config_handler_clang_tidy.py +++ b/libcodechecker/analyze/analyzers/config_handler_clang_tidy.py @@ -10,9 +10,9 @@ import shlex from libcodechecker.analyze.analyzers import config_handler -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('CLANG TIDY CONFIG') +LOG = get_logger('analyzer.tidy') class ClangTidyConfigHandler(config_handler.AnalyzerConfigHandler): diff --git a/libcodechecker/analyze/analyzers/config_handler_clangsa.py b/libcodechecker/analyze/analyzers/config_handler_clangsa.py index 6d09cbf778..b106755813 100644 --- a/libcodechecker/analyze/analyzers/config_handler_clangsa.py +++ b/libcodechecker/analyze/analyzers/config_handler_clangsa.py @@ -7,9 +7,9 @@ import re from libcodechecker.analyze.analyzers import config_handler -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('CLANGSA CONFIG HANDLER') +LOG = get_logger('analyzer.clangsa') class ClangSAConfigHandler(config_handler.AnalyzerConfigHandler): diff --git a/libcodechecker/analyze/analyzers/result_handler_base.py b/libcodechecker/analyze/analyzers/result_handler_base.py index 6c57539add..f6fd558c82 100644 --- a/libcodechecker/analyze/analyzers/result_handler_base.py +++ b/libcodechecker/analyze/analyzers/result_handler_base.py @@ -8,9 +8,9 @@ import hashlib import os -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('RESULT HANDLER (BASE)') +LOG = get_logger('analyzer') class ResultHandler(object): diff --git a/libcodechecker/analyze/analyzers/result_handler_clang_tidy.py b/libcodechecker/analyze/analyzers/result_handler_clang_tidy.py index 861809bb83..723775867d 100644 --- a/libcodechecker/analyze/analyzers/result_handler_clang_tidy.py +++ b/libcodechecker/analyze/analyzers/result_handler_clang_tidy.py @@ -9,9 +9,9 @@ import ResultHandler from libcodechecker.analyze.analyzers.result_handler_plist_to_stdout \ import PlistToStdout -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('CLANG-TIDY RESULT HANDLER') +LOG = get_logger('report') def generate_plist_from_tidy_result(output_file, tidy_stdout): diff --git a/libcodechecker/analyze/analyzers/result_handler_plist_to_stdout.py b/libcodechecker/analyze/analyzers/result_handler_plist_to_stdout.py index effadc42ac..aadcce285d 100644 --- a/libcodechecker/analyze/analyzers/result_handler_plist_to_stdout.py +++ b/libcodechecker/analyze/analyzers/result_handler_plist_to_stdout.py @@ -14,9 +14,9 @@ from libcodechecker import util from libcodechecker.analyze import plist_parser from libcodechecker.analyze.analyzers.result_handler_base import ResultHandler -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('PLIST TO STDOUT') +LOG = get_logger('report') class PlistToStdout(ResultHandler): diff --git a/libcodechecker/analyze/ctu_manager.py b/libcodechecker/analyze/ctu_manager.py index b642db8df3..95a30090d2 100644 --- a/libcodechecker/analyze/ctu_manager.py +++ b/libcodechecker/analyze/ctu_manager.py @@ -20,9 +20,9 @@ from libcodechecker.analyze.analyzers import analyzer_types from libcodechecker.analyze.analyzers import ctu_triple_arch from libcodechecker.analyze import analyzer_env -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('CTU MANAGER') +LOG = get_logger('analyzer') def generate_func_map_lines(fnmap_dir): diff --git a/libcodechecker/analyze/host_check.py b/libcodechecker/analyze/host_check.py index 7b5f021e40..90fecb636d 100644 --- a/libcodechecker/analyze/host_check.py +++ b/libcodechecker/analyze/host_check.py @@ -8,9 +8,9 @@ import subprocess import tempfile -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('HOST CHECK') +LOG = get_logger('analyze') def check_clang(compiler_bin, env): diff --git a/libcodechecker/analyze/log_parser.py b/libcodechecker/analyze/log_parser.py index cfc9e12eae..1c113cccc0 100644 --- a/libcodechecker/analyze/log_parser.py +++ b/libcodechecker/analyze/log_parser.py @@ -15,9 +15,9 @@ # TODO: This is a cross-subpackage import! from libcodechecker.log import build_action from libcodechecker.log import option_parser -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('LOG PARSER') +LOG = get_logger('buildlogger') # If these options are present in the original build command, they must diff --git a/libcodechecker/analyze/plist_parser.py b/libcodechecker/analyze/plist_parser.py index 23fa3a72f8..0bfa45271b 100644 --- a/libcodechecker/analyze/plist_parser.py +++ b/libcodechecker/analyze/plist_parser.py @@ -36,12 +36,12 @@ import traceback from xml.parsers.expat import ExpatError -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger from libcodechecker.report import Report from libcodechecker.report import generate_report_hash -LOG = LoggerFactory.get_new_logger('PLIST_PARSER') +LOG = get_logger('report') def get_checker_name(diagnostic, path=""): diff --git a/libcodechecker/analyze/skiplist_handler.py b/libcodechecker/analyze/skiplist_handler.py index 776a18eaee..288dbfe3b5 100644 --- a/libcodechecker/analyze/skiplist_handler.py +++ b/libcodechecker/analyze/skiplist_handler.py @@ -9,9 +9,9 @@ import fnmatch import re -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('SKIPLIST HANDLER') +LOG = get_logger('system') class SkipListHandler(object): diff --git a/libcodechecker/analyze/tidy_output_converter.py b/libcodechecker/analyze/tidy_output_converter.py index 1d6fd33b94..f2899020a6 100644 --- a/libcodechecker/analyze/tidy_output_converter.py +++ b/libcodechecker/analyze/tidy_output_converter.py @@ -14,10 +14,10 @@ import plistlib import re -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger from libcodechecker.report import generate_report_hash -LOG = LoggerFactory.get_new_logger('TIDY OUTPUT CONVERTER') +LOG = get_logger('analyzer.tidy') class Note(object): diff --git a/libcodechecker/cmd/cmd_line_client.py b/libcodechecker/cmd/cmd_line_client.py index 8a73bf0757..c229a0eaea 100644 --- a/libcodechecker/cmd/cmd_line_client.py +++ b/libcodechecker/cmd/cmd_line_client.py @@ -19,18 +19,18 @@ from codeCheckerDBAccess_v6 import constants, ttypes from libcodechecker import generic_package_context +from libcodechecker import logger from libcodechecker import suppress_handler from libcodechecker import suppress_file_handler from libcodechecker.analyze import plist_parser from libcodechecker.libclient.client import handle_auth from libcodechecker.libclient.client import setup_client -from libcodechecker.logger import LoggerFactory from libcodechecker.output_formatters import twodim_to_str from libcodechecker.report import Report from libcodechecker.util import split_server_url -LOG = LoggerFactory.get_new_logger('CMD') +LOG = logger.get_logger('system') class CmdLineOutputEncoder(json.JSONEncoder): @@ -124,6 +124,8 @@ def handle_list_runs(args): def handle_list_results(args): + logger.setup_logger(args.verbose) + client = setup_client(args.product_url) run_info = check_run_names(client, [args.name]) @@ -173,6 +175,9 @@ def handle_list_results(args): def handle_diff_results(args): + + logger.setup_logger(args.verbose) + context = generic_package_context.get_context() def get_diff_results(client, baseids, cmp_data): @@ -671,6 +676,9 @@ def checker_count(checker_dict, key): def handle_remove_run_results(args): + + logger.setup_logger(args.verbose) + client = setup_client(args.product_url) def is_later(d1, d2): @@ -718,6 +726,9 @@ def condition(name, runid, date): def handle_suppress(args): + + logger.setup_logger(args.verbose) + def bug_hash_filter(bug_id, filepath): filepath = '%' + filepath return [ @@ -746,6 +757,9 @@ def bug_hash_filter(bug_id, filepath): def handle_login(args): + + logger.setup_logger(args.verbose) + protocol, host, port = split_server_url(args.server_url) handle_auth(protocol, host, port, args.username, login='logout' not in args) diff --git a/libcodechecker/cmd/product_client.py b/libcodechecker/cmd/product_client.py index 9bf93cd11f..557f744eba 100644 --- a/libcodechecker/cmd/product_client.py +++ b/libcodechecker/cmd/product_client.py @@ -13,7 +13,7 @@ from ProductManagement_v6.ttypes import * from libcodechecker.libclient.client import setup_product_client -from libcodechecker.logger import LoggerFactory +from libcodechecker import logger from libcodechecker.output_formatters import twodim_to_str from libcodechecker.server.database import database_status from libcodechecker.util import split_server_url @@ -21,10 +21,12 @@ from cmd_line_client import CmdLineOutputEncoder -LOG = LoggerFactory.get_new_logger('CMD') +LOG = logger.get_logger('system') def handle_list_products(args): + logger.setup_logger(args.verbose) + protocol, host, port = split_server_url(args.server_url) client = setup_product_client(protocol, host, port) products = client.getProducts(None, None) @@ -57,6 +59,8 @@ def handle_list_products(args): def handle_add_product(args): + logger.setup_logger(args.verbose) + protocol, host, port = split_server_url(args.server_url) client = setup_product_client(protocol, host, port) @@ -106,6 +110,8 @@ def handle_add_product(args): def handle_del_product(args): + logger.setup_logger(args.verbose) + protocol, host, port = split_server_url(args.server_url) client = setup_product_client(protocol, host, port) diff --git a/libcodechecker/generic_package_context.py b/libcodechecker/generic_package_context.py index c28d10d545..a8599050ef 100644 --- a/libcodechecker/generic_package_context.py +++ b/libcodechecker/generic_package_context.py @@ -13,7 +13,7 @@ # TODO: Refers subpackage library from libcodechecker.analyze.analyzers import analyzer_types -LOG = logger.LoggerFactory.get_new_logger('CONTEXT') +LOG = logger.get_logger('system') # ----------------------------------------------------------------------------- @@ -103,8 +103,8 @@ def __set_version(self): self.__package_git_hash = package_git_hash self.__package_git_tag = package_git_tag - if (logger.LoggerFactory.get_log_level() == logger.DEBUG or - logger.LoggerFactory.get_log_level() == + if (LOG.getEffectiveLevel() == logger.DEBUG or + LOG.getEffectiveLevel() == logger.DEBUG_ANALYZER): self.__package_git_tag = package_git_dirtytag diff --git a/libcodechecker/generic_package_suppress_handler.py b/libcodechecker/generic_package_suppress_handler.py index a6154b65b1..1e07e1c54f 100644 --- a/libcodechecker/generic_package_suppress_handler.py +++ b/libcodechecker/generic_package_suppress_handler.py @@ -11,10 +11,10 @@ from libcodechecker import suppress_file_handler from libcodechecker import suppress_handler -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger # Warning! this logger should only be used in this module. -LOG = LoggerFactory.get_new_logger('SUPPRESS') +LOG = get_logger('system') class GenericSuppressHandler(suppress_handler.SuppressHandler): diff --git a/libcodechecker/host_check.py b/libcodechecker/host_check.py index 2dcd51b35a..8ad4f8758d 100644 --- a/libcodechecker/host_check.py +++ b/libcodechecker/host_check.py @@ -8,9 +8,9 @@ import subprocess from libcodechecker import generic_package_context -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('HOST CHECK') +LOG = get_logger('system') def is_ctu_capable(): diff --git a/libcodechecker/libauth/cc_ldap.py b/libcodechecker/libauth/cc_ldap.py index fb752b9f52..540ddccd9a 100644 --- a/libcodechecker/libauth/cc_ldap.py +++ b/libcodechecker/libauth/cc_ldap.py @@ -90,9 +90,9 @@ import ldap -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger("LDAP") +LOG = get_logger('server') def log_ldap_error(ldap_error): diff --git a/libcodechecker/libauth/cc_pam.py b/libcodechecker/libauth/cc_pam.py index bee597003c..35c43f2987 100644 --- a/libcodechecker/libauth/cc_pam.py +++ b/libcodechecker/libauth/cc_pam.py @@ -25,9 +25,9 @@ import grp import pwd -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger("PAM") +LOG = get_logger('server') def auth_user(pam_config, username, password): diff --git a/libcodechecker/libclient/client.py b/libcodechecker/libclient/client.py index 95c39057fd..fc70b469dd 100644 --- a/libcodechecker/libclient/client.py +++ b/libcodechecker/libclient/client.py @@ -14,7 +14,7 @@ from Authentication_v6 import ttypes as AuthTypes from libcodechecker import session_manager -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger from libcodechecker.util import split_product_url from libcodechecker.version import CLIENT_API @@ -22,7 +22,7 @@ from . import thrift_helper from . import product_helper -LOG = LoggerFactory.get_new_logger('CLIENT') +LOG = get_logger('system') def check_api_version(auth_client): diff --git a/libcodechecker/libclient/thrift_helper.py b/libcodechecker/libclient/thrift_helper.py index dc6c932931..59fdd0c4bc 100644 --- a/libcodechecker/libclient/thrift_helper.py +++ b/libcodechecker/libclient/thrift_helper.py @@ -18,10 +18,10 @@ from libcodechecker import session_manager from libcodechecker import util -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('THRIFT HELPER') +LOG = get_logger('system') class ThriftClientHelper(object): diff --git a/libcodechecker/libhandlers/analyze.py b/libcodechecker/libhandlers/analyze.py index 3f3bd36a44..aaf4a0ad03 100644 --- a/libcodechecker/libhandlers/analyze.py +++ b/libcodechecker/libhandlers/analyze.py @@ -13,15 +13,14 @@ import shutil import sys +from libcodechecker import logger from libcodechecker import generic_package_context from libcodechecker import host_check from libcodechecker.analyze import log_parser from libcodechecker.analyze import analyzer from libcodechecker.analyze.analyzers import analyzer_types -from libcodechecker.logger import add_verbose_arguments -from libcodechecker.logger import LoggerFactory -LOG = LoggerFactory.get_new_logger('ANALYZE') +LOG = logger.get_logger('system') class OrderedCheckersAction(argparse.Action): @@ -322,7 +321,7 @@ def add_arguments_to_parser(parser): "the analysis. USE WISELY AND AT YOUR " "OWN RISK!") - add_verbose_arguments(parser) + logger.add_verbose_arguments(parser) parser.set_defaults(func=main) @@ -347,6 +346,7 @@ def main(args): Perform analysis on the given logfiles and store the results in a machine- readable format. """ + logger.setup_logger(args.verbose) if len(args.logfile) != 1: LOG.warning("Only one log file can be processed right now!") diff --git a/libcodechecker/libhandlers/analyzers.py b/libcodechecker/libhandlers/analyzers.py index 7ba47b3f33..f832f34261 100644 --- a/libcodechecker/libhandlers/analyzers.py +++ b/libcodechecker/libhandlers/analyzers.py @@ -11,13 +11,12 @@ import argparse import subprocess +from libcodechecker import logger from libcodechecker import generic_package_context from libcodechecker import output_formatters from libcodechecker.analyze.analyzers import analyzer_types -from libcodechecker.logger import add_verbose_arguments -from libcodechecker.logger import LoggerFactory -LOG = LoggerFactory.get_new_logger('ANALYZERS') +LOG = logger.get_logger('analyze') def get_argparser_ctor_args(): @@ -68,7 +67,7 @@ def add_arguments_to_parser(parser): choices=output_formatters.USER_FORMATS, help="Specify the format of the output list.") - add_verbose_arguments(parser) + logger.add_verbose_arguments(parser) parser.set_defaults(func=main) @@ -76,6 +75,7 @@ def main(args): """ List the analyzers' basic information supported by CodeChecker. """ + logger.setup_logger(args.verbose) context = generic_package_context.get_context() working, errored = \ diff --git a/libcodechecker/libhandlers/check.py b/libcodechecker/libhandlers/check.py index 5c5ce80e3c..3391fe1e90 100644 --- a/libcodechecker/libhandlers/check.py +++ b/libcodechecker/libhandlers/check.py @@ -14,13 +14,12 @@ import shutil import tempfile +from libcodechecker import logger from libcodechecker import host_check from libcodechecker import libhandlers from libcodechecker.analyze.analyzers import analyzer_types -from libcodechecker.logger import add_verbose_arguments -from libcodechecker.logger import LoggerFactory -LOG = LoggerFactory.get_new_logger('CHECK') +LOG = logger.get_logger('system') class OrderedCheckersAction(argparse.Action): @@ -352,7 +351,7 @@ def add_arguments_to_parser(parser): help="Print the steps the analyzers took in " "finding the reported defect.") - add_verbose_arguments(parser) + logger.add_verbose_arguments(parser) parser.set_defaults(func=main) @@ -361,6 +360,8 @@ def main(args): Execute a wrapper over log-analyze-parse, aka 'check'. """ + logger.setup_logger(args.verbose) + def __load_module(name): """Loads the given subcommand's definition from the libs.""" try: diff --git a/libcodechecker/libhandlers/checkers.py b/libcodechecker/libhandlers/checkers.py index f10f1a9a8f..b3c3a67d57 100644 --- a/libcodechecker/libhandlers/checkers.py +++ b/libcodechecker/libhandlers/checkers.py @@ -11,13 +11,12 @@ import os from libcodechecker import generic_package_context +from libcodechecker import logger from libcodechecker import output_formatters from libcodechecker.analyze import analyzer_env from libcodechecker.analyze.analyzers import analyzer_types -from libcodechecker.logger import add_verbose_arguments -from libcodechecker.logger import LoggerFactory -LOG = LoggerFactory.get_new_logger('CHECKERS') +LOG = logger.get_logger('system') def get_argparser_ctor_args(): @@ -103,7 +102,7 @@ def add_arguments_to_parser(parser): choices=output_formatters.USER_FORMATS, help="The format to list the applicable checkers as.") - add_verbose_arguments(parser) + logger.add_verbose_arguments(parser) parser.set_defaults(func=main) @@ -113,6 +112,8 @@ def main(args): alongside with their description or enabled status in various formats. """ + logger.setup_logger(args.verbose) + # If nothing is set, list checkers for all supported analyzers. analyzers = args.analyzers \ if 'analyzers' in args \ diff --git a/libcodechecker/libhandlers/cmd.py b/libcodechecker/libhandlers/cmd.py index 084f2ad426..ce1605ac54 100644 --- a/libcodechecker/libhandlers/cmd.py +++ b/libcodechecker/libhandlers/cmd.py @@ -13,14 +13,11 @@ import datetime import sys +from libcodechecker import logger from libcodechecker import output_formatters from libcodechecker import util from libcodechecker.cmd import cmd_line_client from libcodechecker.cmd import product_client -from libcodechecker.logger import add_verbose_arguments -from libcodechecker.logger import LoggerFactory - -LOG = LoggerFactory.get_new_logger('CMD') def valid_time(t): @@ -127,7 +124,7 @@ def __add_common_arguments(parser, "overwrites only those that contain " "any reports).") - add_verbose_arguments(common_group) + logger.add_verbose_arguments(common_group) def __add_filtering_arguments(parser): diff --git a/libcodechecker/libhandlers/log.py b/libcodechecker/libhandlers/log.py index 5c1bfd4d09..bc1fa1ceb9 100644 --- a/libcodechecker/libhandlers/log.py +++ b/libcodechecker/libhandlers/log.py @@ -15,9 +15,9 @@ import os from libcodechecker import generic_package_context +from libcodechecker import logger from libcodechecker.log import build_manager from libcodechecker.log.host_check import check_intercept -from libcodechecker.logger import add_verbose_arguments def get_argparser_ctor_args(): @@ -77,7 +77,7 @@ def add_arguments_to_parser(parser): help="Do not print the output of the build tool into " "the output of this command.") - add_verbose_arguments(parser) + logger.add_verbose_arguments(parser) parser.set_defaults(func=main) @@ -86,6 +86,8 @@ def main(args): Generates a build log by running the original build command. No analysis is done. """ + logger.setup_logger(args.verbose) + args.logfile = os.path.realpath(args.logfile) if os.path.exists(args.logfile): os.remove(args.logfile) diff --git a/libcodechecker/libhandlers/parse.py b/libcodechecker/libhandlers/parse.py index 74744e297b..17ca50142e 100644 --- a/libcodechecker/libhandlers/parse.py +++ b/libcodechecker/libhandlers/parse.py @@ -18,15 +18,14 @@ from libcodechecker import generic_package_context from libcodechecker import generic_package_suppress_handler +from libcodechecker import logger from libcodechecker import util from libcodechecker.analyze.analyzers import analyzer_types # TODO: This is a cross-subpackage reference... from libcodechecker.log import build_action -from libcodechecker.logger import add_verbose_arguments -from libcodechecker.logger import LoggerFactory from libcodechecker.output_formatters import twodim_to_str -LOG = LoggerFactory.get_new_logger('PARSE') +LOG = logger.get_logger('system') def get_argparser_ctor_args(): @@ -130,7 +129,7 @@ def add_arguments_to_parser(parser): help="Print the steps the analyzers took in finding " "the reported defect.") - add_verbose_arguments(parser) + logger.add_verbose_arguments(parser) def __handle(args): """Custom handler for 'parser' so custom error messages can be @@ -195,6 +194,8 @@ def main(args): stdout in a human-readable format. """ + logger.setup_logger(args.verbose) + context = generic_package_context.get_context() # To ensure the help message prints the default folder properly, diff --git a/libcodechecker/libhandlers/server.py b/libcodechecker/libhandlers/server.py index bda4667254..0983086530 100644 --- a/libcodechecker/libhandlers/server.py +++ b/libcodechecker/libhandlers/server.py @@ -24,12 +24,11 @@ from libcodechecker import generic_package_context from libcodechecker import generic_package_suppress_handler from libcodechecker import host_check +from libcodechecker import logger from libcodechecker import output_formatters from libcodechecker import session_manager from libcodechecker import util from libcodechecker.analyze import analyzer_env -from libcodechecker.logger import LoggerFactory -from libcodechecker.logger import add_verbose_arguments from libcodechecker.server import server from libcodechecker.server import instance_manager from libcodechecker.server.database import database @@ -42,7 +41,7 @@ import IDENTIFIER as RUN_META -LOG = LoggerFactory.get_new_logger('SERVER') +LOG = logger.get_logger('server') def get_argparser_ctor_args(): @@ -297,7 +296,7 @@ def add_arguments_to_parser(parser): " to create a full backup of " "the product databases.") - add_verbose_arguments(parser) + logger.add_verbose_arguments(parser) def __handle(args): """Custom handler for 'server' so custom error messages can be @@ -640,7 +639,7 @@ def __instance_management(args): raise -def main(args): +def server_init_start(args): """ Start or manage a CodeChecker report server. """ @@ -857,3 +856,12 @@ def main(args): sys.exit(1) else: raise + + +def main(args): + """ + Setup a logger server based on the configuration and + manage the Codechecker server. + """ + with logger.LOG_CFG_SERVER(args.verbose): + server_init_start(args) diff --git a/libcodechecker/libhandlers/store.py b/libcodechecker/libhandlers/store.py index b7ed2b044c..2af94741ea 100644 --- a/libcodechecker/libhandlers/store.py +++ b/libcodechecker/libhandlers/store.py @@ -23,19 +23,19 @@ from shared.ttypes import Permission +from libcodechecker import logger from libcodechecker import generic_package_context from libcodechecker import host_check from libcodechecker import util from libcodechecker.analyze import plist_parser from libcodechecker.libclient import client as libclient -from libcodechecker.logger import add_verbose_arguments -from libcodechecker.logger import LoggerFactory from libcodechecker.util import sizeof_fmt from libcodechecker.util import split_product_url -LOG = LoggerFactory.get_new_logger('STORE') -MAX_UPLOAD_SIZE = 1 * 1024 * 1024 * 1024 # 1 GiB. +LOG = logger.get_logger('system') + +MAX_UPLOAD_SIZE = 1 * 1024 * 1024 * 1024 # 1GiB def full_traceback(func): @@ -145,7 +145,7 @@ def add_arguments_to_parser(parser): "results for, in the format of " "'[http[s]://]host:port/Endpoint'.") - add_verbose_arguments(parser) + logger.add_verbose_arguments(parser) parser.set_defaults(func=main) @@ -265,6 +265,7 @@ def main(args): Store the defect results in the specified input list as bug reports in the database. """ + logger.setup_logger(args.verbose) if not host_check.check_zlib(): raise Exception("zlib is not available on the system!") diff --git a/libcodechecker/libhandlers/version.py b/libcodechecker/libhandlers/version.py index c651c87762..4458a9971a 100644 --- a/libcodechecker/libhandlers/version.py +++ b/libcodechecker/libhandlers/version.py @@ -10,10 +10,10 @@ import argparse import json +from libcodechecker import logger from libcodechecker import generic_package_context from libcodechecker import output_formatters from libcodechecker import version -from libcodechecker.logger import add_verbose_arguments def get_argparser_ctor_args(): @@ -48,7 +48,7 @@ def add_arguments_to_parser(parser): choices=output_formatters.USER_FORMATS, help="The format to use when printing the version.") - add_verbose_arguments(parser) + logger.add_verbose_arguments(parser) parser.set_defaults(func=main) @@ -57,6 +57,7 @@ def main(args): Get and print the version information from the version config file and Thrift API definition. """ + logger.setup_logger(args.verbose) context = generic_package_context.get_context() diff --git a/libcodechecker/log/build_manager.py b/libcodechecker/log/build_manager.py index 79ab936b8c..d3dd533341 100644 --- a/libcodechecker/log/build_manager.py +++ b/libcodechecker/log/build_manager.py @@ -14,13 +14,13 @@ import sys from uuid import uuid4 -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger # TODO: Cross-reference between subpacakges... from libcodechecker.analyze import analyzer_env from . import host_check -LOG = LoggerFactory.get_new_logger('BUILD MANAGER') +LOG = get_logger('buildlogger') def execute_buildcmd(command, silent=False, env=None, cwd=None): diff --git a/libcodechecker/log/host_check.py b/libcodechecker/log/host_check.py index 33e1867777..5b8df1b762 100644 --- a/libcodechecker/log/host_check.py +++ b/libcodechecker/log/host_check.py @@ -8,9 +8,9 @@ import os import subprocess -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('HOST CHECK') +LOG = get_logger('system') def check_intercept(env): diff --git a/libcodechecker/log/option_parser.py b/libcodechecker/log/option_parser.py index d1452bfb67..d928313ec5 100644 --- a/libcodechecker/log/option_parser.py +++ b/libcodechecker/log/option_parser.py @@ -20,9 +20,9 @@ import re import shlex -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('OPTION PARSER') +LOG = get_logger('buildlogger') # Compiler options in the following format # argument_name: number of parameters sepearated by space. diff --git a/libcodechecker/logger.py b/libcodechecker/logger.py index 8b407a8d85..4d38a06c47 100644 --- a/libcodechecker/logger.py +++ b/libcodechecker/logger.py @@ -6,9 +6,12 @@ """ """ +import json import logging -import sys -import time +from logging import config +import os +import string + # The logging leaves can be accesses without # importing the logging module in other modules. @@ -19,24 +22,7 @@ CRITICAL = logging.CRITICAL NOTSET = logging.NOTSET - -class BColors(object): - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - - -def add_verbose_arguments(parser): - """ - Verbosity level arguments. - """ - parser.add_argument('--verbose', type=str, dest='verbose', - choices=['info', 'debug', 'debug_analyzer'], - default='info', - help='Set verbosity level.') +CMDLINE_LOG_LEVELS = ['info', 'debug', 'debug_analyzer'] DEBUG_ANALYZER = logging.DEBUG_ANALYZER = 15 @@ -54,137 +40,127 @@ def debug_analyzer(self, msg, *args, **kwargs): logging.setLoggerClass(CCLogger) - -class CustomFormatter(logging.Formatter): - """ - Custom formatter to print log level in case of ERROR, WARNING - or CRITICAL. - """ - - # Short format strings where only the time and the message is printed. - format_string_short = { - 'info_fmt': '[%(asctime)s] - %(message)s', - 'error_fmt': '[%(levelname)s] [%(asctime)s] - %(message)s', - 'debug_fmt': '[%(asctime)s] {%(name)s} [%(process)d] <%(thread)d> - ' - '%(filename)s:%(lineno)d %(funcName)s() - %(message)s' +package_root = os.environ.get('CC_PACKAGE_ROOT', '') +DEFAULT_LOG_CFG_FILE = os.path.join(package_root, 'config', 'logger.conf') + + +# Default config wich can be used if reading log config from a +# file fails. +DEFAULT_LOG_CONFIG = '''{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "brief": { + "format": "[%(asctime)s][%(levelname)s] - %(message)s", + "datefmt": "%Y-%m-%d %H:%M" + }, + "precise": { + "format": "[%(levelname)s] [%(asctime)s] {%(name)s} [%(process)d] \ +<%(thread)d> - %(filename)s:%(lineno)d %(funcName)s() - %(message)s", + "datefmt": "%Y-%m-%d %H:%M" } - - # In a more verbose mode (such as --verbose debug), the error messages - # should also be a bit more verbose. - format_strings_verbose = { - 'info_fmt': '[%(asctime)s] {%(name)s} - ' - '%(filename)s:%(lineno)d %(funcName)s() - %(message)s', - 'error_fmt': '[%(levelname)s] [%(asctime)s] {%(name)s} - ' - '%(filename)s:%(lineno)d %(funcName)s() - %(message)s', - 'debug_fmt': '[%(asctime)s] {%(name)s} [%(process)d] <%(thread)d> - ' - '%(filename)s:%(lineno)d %(funcName)s() - %(message)s' + }, + "handlers": { + "default": { + "level": "INFO", + "formatter": "brief", + "class": "logging.StreamHandler" } - - # This sets to which --verbose (loglevel), what verbosity strings should - # be used by the loggers. - formatter_map = { - logging.DEBUG: format_strings_verbose, - logging.DEBUG_ANALYZER: format_strings_verbose, - logging.INFO: format_string_short, - logging.ERROR: format_string_short, - logging.WARNING: format_string_short, - logging.CRITICAL: format_string_short + }, + "loggers": { + "": { + "handlers": ["default"], + "level": "INFO", + "propagate": true } + } +}''' - def __init__(self): - self.level = logging.INFO - super(CustomFormatter, self).__init__() - - def set_level(self, new_level): - self.level = new_level - def formatTime(self, record, datefmt=None): - if LoggerFactory.log_level == logging.DEBUG or \ - LoggerFactory.log_level == logging.DEBUG_ANALYZER: - return time.strftime('%Y-%m-%d %H:%M:%S') - else: - return time.strftime('%Y-%m-%d %H:%M') +try: + with open(DEFAULT_LOG_CFG_FILE, 'r') as dlc: + DEFAULT_LOG_CONFIG = dlc.read() +except IOError as ex: + print(ex) + print("Failed to load logger configuration. Using built-in config.") - def format(self, record): - # Save the original format - format_orig = self._fmt - - # Get a format based on the most verbose loglevel set by the user. - format_strings = CustomFormatter.formatter_map[self.level] - - # Replace the original format. - if record.levelno == logging.DEBUG: - self._fmt = format_strings['debug_fmt'] - if record.levelno == logging.DEBUG_ANALYZER: - self._fmt = format_strings['debug_fmt'] - elif record.levelno == logging.INFO: - self._fmt = format_strings['info_fmt'] - elif record.levelno == logging.ERROR: - self._fmt = format_strings['error_fmt'] - elif record.levelno == logging.WARNING: - self._fmt = format_strings['error_fmt'] - elif record.levelno == logging.CRITICAL: - self._fmt = format_strings['error_fmt'] +def add_verbose_arguments(parser): + """ + Verbosity level arguments. + """ + parser.add_argument('--verbose', type=str, dest='verbose', + choices=CMDLINE_LOG_LEVELS, + default='info', + help='Set verbosity level.') - result = logging.Formatter.format(self, record) - self._fmt = format_orig +def get_logger(name): + """ + Return a logger instance if already exists with the given name. + """ + return logging.getLogger(name) - return result +def validate_loglvl(log_level): + """ + Should return a valid log level name + """ + log_level = log_level.upper() + if log_level not in map(string.upper, CMDLINE_LOG_LEVELS): + return "INFO" + else: + return log_level -class LoggerFactory(object): - log_level = logging.INFO - loggers = [] - short_format_handler = logging.StreamHandler(stream=sys.stdout) - long_format_handler = logging.StreamHandler(stream=sys.stdout) +class LOG_CFG_SERVER(object): + """ + Initialize a log configuration server for dynamic log configuration. + The log config server will only be started if the + 'CC_LOG_CONFIG_PORT' environment variable is set. + """ - custom_formatter = CustomFormatter() + def __init__(self, log_level='INFO'): - short_format_handler.setFormatter(custom_formatter) - long_format_handler.setFormatter(custom_formatter) + # Configure the logging with the default config. + setup_logger(log_level) - handlers = { - logging.INFO: short_format_handler, - logging.DEBUG: long_format_handler, - logging.DEBUG_ANALYZER: short_format_handler} + self.log_server = None - @classmethod - def get_log_level(cls): - return cls.log_level + log_cfg_port = os.environ.get('CC_LOG_CONFIG_PORT') - @classmethod - def set_log_level(cls, level): - for logger in cls.loggers: - logger.removeHandler(cls.handlers[LoggerFactory.log_level]) + if log_cfg_port: + self.log_server = config.listen(int(log_cfg_port)) + self.log_server.start() - if level == 'debug': - cls.log_level = logging.DEBUG - elif level == 'debug_analyzer': - cls.log_level = logging.DEBUG_ANALYZER - elif level == 'info': - cls.log_level = logging.INFO - else: - cls.log_level = logging.INFO + def __enter__(self, *args): + return self - cls.custom_formatter.set_level(cls.log_level) + def __exit__(self, *args): + if self.log_server: + config.stopListening() + self.log_server.join() - cls.short_format_handler.setLevel(cls.log_level) - cls.long_format_handler.setLevel(cls.log_level) - for logger in cls.loggers: - logger.setLevel(cls.log_level) - logger.addHandler(cls.handlers[cls.log_level]) +def setup_logger(log_level=None): + """ + Modifies the log configuration. + Overwrites the log levels for the loggers and handlers in the + configuration. + """ - @classmethod - def get_new_logger(cls, logger_name): - logger = logging.getLogger(logger_name) + LOG_CONFIG = json.loads(DEFAULT_LOG_CONFIG) + if log_level: + log_level = validate_loglvl(log_level) - logger.setLevel(cls.log_level) - logger.addHandler(cls.handlers[cls.log_level]) + loggers = LOG_CONFIG.get("loggers", {}) + for k in loggers.keys(): + LOG_CONFIG['loggers'][k]['level'] = log_level - cls.loggers.append(logger) + handlers = LOG_CONFIG.get("handlers", {}) + for k in handlers.keys(): + LOG_CONFIG['handlers'][k]['level'] = log_level + if log_level == 'DEBUG' or log_level == 'DEBUG_ANALYZER': + LOG_CONFIG['handlers'][k]['formatter'] = 'precise' - return logger + config.dictConfig(LOG_CONFIG) diff --git a/libcodechecker/profiler.py b/libcodechecker/profiler.py index 544cb452e0..2cbe8513bc 100644 --- a/libcodechecker/profiler.py +++ b/libcodechecker/profiler.py @@ -16,9 +16,9 @@ from io import BytesIO as StringIO from libcodechecker import logger -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('PROFILER') +LOG = get_logger('profiler') class Timer(object): @@ -65,7 +65,7 @@ def release_wrapper(*args, **kwargs): @wraps(function) def wrapper(*args, **kwargs): - if LoggerFactory.get_log_level() == logger.DEBUG: + if LOG.getEffectiveLevel() == logger.DEBUG: return debug_wrapper(*args, **kwargs) else: return release_wrapper(*args, **kwargs) diff --git a/libcodechecker/report.py b/libcodechecker/report.py index 8107268a05..ee04dd4ecd 100644 --- a/libcodechecker/report.py +++ b/libcodechecker/report.py @@ -15,10 +15,10 @@ import json import os -from libcodechecker.logger import LoggerFactory import libcodechecker.util as util +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('REPORT') +LOG = get_logger('report') def generate_report_hash(path, source_file, check_name): diff --git a/libcodechecker/server/api/authentication.py b/libcodechecker/server/api/authentication.py index ac194d97e7..9c8985c669 100644 --- a/libcodechecker/server/api/authentication.py +++ b/libcodechecker/server/api/authentication.py @@ -14,7 +14,7 @@ import shared from Authentication_v6.ttypes import * -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger from libcodechecker.profiler import timeit from libcodechecker.server import permissions from libcodechecker.server.permissions \ @@ -22,7 +22,7 @@ from libcodechecker.server.permissions \ import require_manager, require_permission -LOG = LoggerFactory.get_new_logger('AUTH HANDLER') +LOG = get_logger('server') class ThriftAuthHandler(object): diff --git a/libcodechecker/server/api/bad_api_version.py b/libcodechecker/server/api/bad_api_version.py index c895f3b33a..d63d0ab7be 100644 --- a/libcodechecker/server/api/bad_api_version.py +++ b/libcodechecker/server/api/bad_api_version.py @@ -9,10 +9,10 @@ import shared -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger from libcodechecker.profiler import timeit -LOG = LoggerFactory.get_new_logger('BAD API VERSION HANDLER') +LOG = get_logger('system') class ThriftAPIMismatchHandler(object): diff --git a/libcodechecker/server/api/product_server.py b/libcodechecker/server/api/product_server.py index 8069e4e29a..fcf6f543c1 100644 --- a/libcodechecker/server/api/product_server.py +++ b/libcodechecker/server/api/product_server.py @@ -17,14 +17,14 @@ from ProductManagement_v6 import ttypes from libcodechecker import util -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger from libcodechecker.profiler import timeit from libcodechecker.server import permissions from libcodechecker.server.database.config_db_model import * from libcodechecker.server.database.database import SQLServer from libcodechecker.server.routing import is_valid_product_endpoint -LOG = LoggerFactory.get_new_logger('PRODUCT HANDLER') +LOG = get_logger('server') class ThriftProductHandler(object): diff --git a/libcodechecker/server/api/report_server.py b/libcodechecker/server/api/report_server.py index ad17847461..0a266bc84a 100644 --- a/libcodechecker/server/api/report_server.py +++ b/libcodechecker/server/api/report_server.py @@ -31,13 +31,13 @@ from libcodechecker import util # TODO: Cross-subpackage import here. from libcodechecker.analyze import plist_parser -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger from libcodechecker.profiler import timeit from libcodechecker.server import permissions from libcodechecker.server.database import db_cleanup from libcodechecker.server.database.run_db_model import * -LOG = LoggerFactory.get_new_logger('RUN ACCESS HANDLER') +LOG = get_logger('server') class CountFilter: diff --git a/libcodechecker/server/api/store_handler.py b/libcodechecker/server/api/store_handler.py index c2ec3911db..235578d8ef 100644 --- a/libcodechecker/server/api/store_handler.py +++ b/libcodechecker/server/api/store_handler.py @@ -16,11 +16,11 @@ from codeCheckerDBAccess_v6 import ttypes from libcodechecker import generic_package_context -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger # TODO: This is a cross-subpackage import. from libcodechecker.server.database.run_db_model import * -LOG = LoggerFactory.get_new_logger('STORE HANDLER') +LOG = get_logger('system') def metadata_info(metadata_file): diff --git a/libcodechecker/server/database/database.py b/libcodechecker/server/database/database.py index 3a1856fb78..0c27008ebb 100644 --- a/libcodechecker/server/database/database.py +++ b/libcodechecker/server/database/database.py @@ -23,11 +23,11 @@ from libcodechecker import host_check from libcodechecker import pgpass from libcodechecker import util -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger from shared.ttypes import DBStatus -LOG = LoggerFactory.get_new_logger('DATABASE HANDLER') +LOG = get_logger('system') class DBContext(): diff --git a/libcodechecker/server/database/db_cleanup.py b/libcodechecker/server/database/db_cleanup.py index a2ba1675f4..6d3ef6d5c6 100644 --- a/libcodechecker/server/database/db_cleanup.py +++ b/libcodechecker/server/database/db_cleanup.py @@ -1,9 +1,9 @@ from codeCheckerDBAccess_v6.ttypes import* -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger from libcodechecker.server.database.run_db_model import * -LOG = LoggerFactory.get_new_logger('DB CLEANUP') +LOG = get_logger('server') def remove_unused_files(session): diff --git a/libcodechecker/server/permissions.py b/libcodechecker/server/permissions.py index 44f08c9b2c..4a5b9c1611 100644 --- a/libcodechecker/server/permissions.py +++ b/libcodechecker/server/permissions.py @@ -16,9 +16,9 @@ from shared.ttypes import Permission as PermissionEnum -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('PERMISSIONS') +LOG = get_logger('server') config_db_model = None # Module will be loaded later... diff --git a/libcodechecker/server/server.py b/libcodechecker/server/server.py index e6363e10a0..a6aa00b982 100644 --- a/libcodechecker/server/server.py +++ b/libcodechecker/server/server.py @@ -41,7 +41,7 @@ from ProductManagement_v6 import codeCheckerProductService as ProductAPI_v6 from libcodechecker import session_manager -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger from libcodechecker.util import get_tmp_dir_hash from . import instance_manager @@ -56,7 +56,7 @@ from database.config_db_model import Product as ORMProduct from database.run_db_model import IDENTIFIER as RUN_META -LOG = LoggerFactory.get_new_logger('SERVER') +LOG = get_logger('server') class RequestHandler(SimpleHTTPRequestHandler): diff --git a/libcodechecker/session_manager.py b/libcodechecker/session_manager.py index 95580992ad..cda794e8c1 100644 --- a/libcodechecker/session_manager.py +++ b/libcodechecker/session_manager.py @@ -19,7 +19,7 @@ import portalocker -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger unsupported_methods = [] @@ -33,7 +33,7 @@ except ImportError: unsupported_methods.append("pam") -LOG = LoggerFactory.get_new_logger("SESSION MANAGER") +LOG = get_logger("server") SESSION_COOKIE_NAME = "__ccPrivilegedAccessToken" session_lifetimes = {} diff --git a/libcodechecker/suppress_file_handler.py b/libcodechecker/suppress_file_handler.py index c453aa4ce6..fb3dfd53d4 100644 --- a/libcodechecker/suppress_file_handler.py +++ b/libcodechecker/suppress_file_handler.py @@ -21,9 +21,9 @@ import os import re -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('SUPPRESS FILE HANDLER') +LOG = get_logger('system') COMMENT_SEPARATOR = '||' HASH_TYPE_SEPARATOR = '#' diff --git a/libcodechecker/suppress_handler.py b/libcodechecker/suppress_handler.py index 99f17618e7..1da39cf154 100644 --- a/libcodechecker/suppress_handler.py +++ b/libcodechecker/suppress_handler.py @@ -12,9 +12,9 @@ import re from libcodechecker import util -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -LOG = LoggerFactory.get_new_logger('SUPPRESS HANDLER') +LOG = get_logger('system') class SuppressHandler(object): diff --git a/libcodechecker/util.py b/libcodechecker/util.py index 51f7256056..6d58c470ce 100644 --- a/libcodechecker/util.py +++ b/libcodechecker/util.py @@ -18,10 +18,9 @@ import psutil -from libcodechecker.logger import LoggerFactory +from libcodechecker.logger import get_logger -# WARNING! LOG should be only used in this module. -LOG = LoggerFactory.get_new_logger('UTIL') +LOG = get_logger('system') def get_free_port(): diff --git a/scripts/send_log_config.py b/scripts/send_log_config.py new file mode 100755 index 0000000000..ebbfc1f90d --- /dev/null +++ b/scripts/send_log_config.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# ------------------------------------------------------------------------- +# The CodeChecker Infrastructure +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# ------------------------------------------------------------------------- +import argparse +import json +import socket +import struct + + +parser = argparse.ArgumentParser( + description=""" +Send a python log config (json) to a port where a logging process listens. + +Further details about the log configuration format and usage can be found here: +https://docs.python.org/2/library/logging.config.html """, + formatter_class=argparse.RawTextHelpFormatter) + +parser.add_argument('-c', action="store", required="True", + dest="config_file", + help="Log configuration in json format.") + +parser.add_argument('-p', action="store", required="True", + dest="port", type=int, + help="Port number of the logger server.") + +args = parser.parse_args() + +try: + with open(args.config_file, 'rb') as cf: + log_config = cf.read() + + # Just a simple check for valid json before sending + json.loads(log_config) + + data_to_send = log_config + + host = 'localhost' + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + print('Connecting to {0}:{1} ...'.format(host, args.port)) + s.connect((host, args.port)) + print('Connection done.') + + print('Sending config ...') + s.send(struct.pack('>L', len(data_to_send))) + s.send(data_to_send) + s.close() + print('Sending config done.') + +except IOError as ex: + print("Failed to read config file" + args.config_file) + print(ex) + print(ex.message) + +except ValueError as ex: + print("Wrong config file format.") + print(ex) + print(ex.message)