From 05ad34dc4778eec7560433d8cd2f49b76054fe60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20G=C3=A1bor=20Isp=C3=A1novics?= Date: Fri, 20 Nov 2015 16:58:52 +0100 Subject: [PATCH 1/3] SQLite support - Move all database server specific code to database_handler.py - Change check constraint naming convensions to be compatible with SQLite. This is backward compatible because currently there is no check constraint in the database schema. - Fix Run.mark_finished() to store integer - Fix suppress queries to use True and False (boolean) instead of strings. - Fix supress -> suppress typos - Add sqlite support to package_test - Add sqlite test to Travis --- .travis.yml | 2 + codechecker/CodeChecker.py | 58 +-- codechecker_lib/arg_handler.py | 66 +-- codechecker_lib/client.py | 202 ++------- codechecker_lib/database_handler.py | 401 +++++++++++++++--- codechecker_lib/debug_reporter.py | 5 +- codechecker_lib/host_check.py | 14 + codechecker_lib/util.py | 16 - db_model/orm_model.py | 7 +- .../build-logger/src/ldlogger-tool-javac.c | 2 +- storage_server/report_server.py | 54 +-- test_package.py | 91 ++-- viewer_server/client_db_access_handler.py | 10 +- viewer_server/client_db_access_server.py | 2 +- 14 files changed, 504 insertions(+), 426 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2fd1b419ba..e16b917db4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,4 +34,6 @@ script: - ./test_package.py -p ./codechecker_package/CodeChecker -v stable --dbport 5432 --dbusername postgres - export CODECHECKER_DB_DRIVER=pg8000 - ./test_package.py -p ./codechecker_package/CodeChecker -v stable --dbport 5432 --dbusername postgres + - unset CODECHECKER_DB_DRIVER + - ./test_package.py -p ./codechecker_package/CodeChecker -v stable --sqlite - ./test_quickcheck.py -p ./codechecker_package/CodeChecker diff --git a/codechecker/CodeChecker.py b/codechecker/CodeChecker.py index f4639d5c45..d059407fed 100755 --- a/codechecker/CodeChecker.py +++ b/codechecker/CodeChecker.py @@ -75,6 +75,25 @@ def __call__(self, parser, namespace, value=None, option_string=None): LOG.warning("Deprecated command line option in use: '" + option_string + "'") +def add_database_arguments(parser): + '''Helper method for adding database arguments to an argument parser.''' + + parser.add_argument('--sqlite', dest="sqlite", + action='store_true', required=False, + help='Use sqlite database.') + parser.add_argument('--dbport', type=int, dest="dbport", + default=8764, required=False, + help='Postgres server port.') + parser.add_argument('--dbaddress', type=str, dest="dbaddress", + default="localhost", required=False, + help='Postgres database server address') + parser.add_argument('--dbname', type=str, dest="dbname", + default="codechecker", required=False, + help='Name of the database.') + parser.add_argument('--dbusername', type=str, dest="dbusername", + default='codechecker', required=False, + help='Database user name.') + # ------------------------------------------------------------------------------ def main(): ''' @@ -137,22 +156,11 @@ def signal_handler(sig, frame): dest="keep_tmp", required=False, help='Keep temporary report files \ after sending data to database storage server.') - check_parser.add_argument('--dbaddress', type=str, dest="dbaddress", - default="localhost", required=False, - help='Postgres database server address.') - check_parser.add_argument('--dbport', type=int, dest="dbport", - default=8764, required=False, - help='Postgres database server port.') - check_parser.add_argument('--dbname', type=str, dest="dbname", - default="codechecker", required=False, - help='Name of the database.') - check_parser.add_argument('--dbusername', type=str, dest="dbusername", - default='codechecker', required=False, - help='Database user name.') check_parser.add_argument('--update', action="store_true", dest="update", default=False, required=False, help='Incremental parsing, \ update the results of a previous run.') + add_database_arguments(check_parser) check_parser.set_defaults(func=arg_handler.handle_check) # -------------------------------------- @@ -204,18 +212,6 @@ def signal_handler(sig, frame): help='Directory where the codechecker \ stored analysis related data \ (automatically created posgtreSql database).') - server_parser.add_argument('--dbport', type=int, dest="dbport", - default=8764, required=False, - help='Postgres server port.') - server_parser.add_argument('--dbaddress', type=str, dest="dbaddress", - default="localhost", required=False, - help='Postgres database server address') - server_parser.add_argument('--dbname', type=str, dest="dbname", - default="codechecker", required=False, - help='Name of the database.') - server_parser.add_argument('--dbusername', type=str, dest="dbusername", - default='codechecker', required=False, - help='Database user name.') server_parser.add_argument('-v', '--view-port', type=int, dest="view_port", default=11444, required=False, help='Port used for viewing.') @@ -231,6 +227,7 @@ def signal_handler(sig, frame): server_parser.add_argument('--check-address', type=str, dest="check_address", default="localhost", required=False, help='Server address.') + add_database_arguments(server_parser) server_parser.set_defaults(func=arg_handler.handle_server) # -------------------------------------- @@ -248,21 +245,10 @@ def signal_handler(sig, frame): dest="workspace", required=False, help='Directory where the codechecker stores \ analysis related data.') - debug_parser.add_argument('--dbport', type=int, dest="dbport", - default=8764, required=False, - help='Postgres server port.') - debug_parser.add_argument('--dbaddress', type=str, dest="dbaddress", - default="localhost", required=False, - help='Postgres database server address') - debug_parser.add_argument('--dbname', type=str, dest="dbname", - default="codechecker", required=False, - help='Name of the database.') - debug_parser.add_argument('--dbusername', type=str, dest="dbusername", - default='codechecker', required=False, - help='Database user name.') debug_parser.add_argument('-f', '--force', action="store_true", dest="force", required=False, default=False, help='Generate dump for all failed action.') + add_database_arguments(debug_parser) debug_parser.set_defaults(func=arg_handler.handle_debug) # -------------------------------------- diff --git a/codechecker_lib/arg_handler.py b/codechecker_lib/arg_handler.py index bf6df986e2..eee6bf7503 100644 --- a/codechecker_lib/arg_handler.py +++ b/codechecker_lib/arg_handler.py @@ -31,6 +31,7 @@ from codechecker_lib import analyzer_env from codechecker_lib import host_check from codechecker_lib import generic_package_suppress_handler +from codechecker_lib.database_handler import SQLServer LOG = logger.get_new_logger('ARGHANDLER') @@ -136,11 +137,11 @@ def signal_handler(*arg, **kwarg): def check_options_validity(args): # Args must has workspace and dbaddress if args.workspace and not util.is_localhost(args.dbaddress): - LOG.info("Workspace is not required when postgreSql server run on remote host.") + LOG.info("Workspace is not required when database server run on remote host.") sys.exit(1) if not args.workspace and util.is_localhost(args.dbaddress): - LOG.info("Workspace is required when postgreSql server run on localhost.") + LOG.info("Workspace is required when database server run on localhost.") sys.exit(1) @@ -166,21 +167,12 @@ def handle_list_checkers(args): print('') -def setup_connection_manager_db(args): - client.ConnectionManager.database_host = args.dbaddress - client.ConnectionManager.database_port = args.dbport - - def handle_server(args): if not host_check.check_zlib(): LOG.error("zlib error") sys.exit(1) - if not host_check.check_postgresql_driver(): - LOG.error("postgresql driver error") - sys.exit(1) - check_options_validity(args) if args.suppress is None: LOG.warning('WARNING! No suppress file was given, suppressed results will be only stored in the database.') @@ -194,32 +186,21 @@ def handle_server(args): context.codechecker_workspace = args.workspace context.db_username = args.dbusername - setup_connection_manager_db(args) - check_env = analyzer_env.get_check_env(context.path_env_extra, context.ld_lib_path_extra) - client.ConnectionManager.run_env = check_env - + sql_server = SQLServer.from_cmdline_args(args, context, check_env) + conn_mgr = client.ConnectionManager(sql_server, args.check_address, args.check_port) if args.check_port: - - LOG.debug('Starting codechecker server and postgres.') - client.ConnectionManager.host = args.check_address - client.ConnectionManager.port = args.check_port - client.ConnectionManager.run_env = check_env - - # starts posgres - client.ConnectionManager.start_server(args.dbname, context) + LOG.debug('Starting codechecker server and database server.') + sql_server.start(wait_for_start=True, init=True) + conn_mgr.start_report_server(context.db_version_info) else: - LOG.debug('Starting postgres.') - client.ConnectionManager.start_postgres(context, init_db=False) - - client.ConnectionManager.block_until_db_start_proc_free(context) + LOG.debug('Starting database.') + sql_server.start(wait_for_start=True, init=False) # start database viewer - db_connection_string = util.create_postgresql_connection_string( - args.dbusername, args.dbaddress, args.dbport, args.dbname) - + db_connection_string = sql_server.get_connection_string() suppress_handler = generic_package_suppress_handler.GenericSuppressHandler() suppress_handler.suppress_file = args.suppress LOG.debug('Using suppress file: ' + str(suppress_handler.suppress_file)) @@ -260,8 +241,6 @@ def handle_log(args): def handle_debug(args): - setup_connection_manager_db(args) - context = generic_package_context.get_context() context.codechecker_workspace = args.workspace context.db_username = args.dbusername @@ -269,14 +248,10 @@ def handle_debug(args): check_env = analyzer_env.get_check_env(context.path_env_extra, context.ld_lib_path_extra) - client.ConnectionManager.run_env = check_env - - client.ConnectionManager.start_postgres(context) + sql_server = SQLServer.from_cmdline_args(args, context, check_env) + sql_server.start(wait_for_start=True, init=False) - client.ConnectionManager.block_until_db_start_proc_free(context) - - debug_reporter.debug(context, args.dbusername, args.dbaddress, - args.dbport, args.dbname, args.force) + debug_reporter.debug(context, sql_server.get_connection_string(), args.force) def _check_generate_log_file(args, context, silent=False): '''Returns a build command log file for check/quickcheck command.''' @@ -316,10 +291,6 @@ def handle_check(args): LOG.error("zlib error") sys.exit(1) - if not host_check.check_postgresql_driver(): - LOG.error("postgresql driver error") - sys.exit(1) - args.workspace = os.path.realpath(args.workspace) if not os.path.isdir(args.workspace): os.mkdir(args.workspace) @@ -353,8 +324,8 @@ def handle_check(args): LOG.warning('There are no build actions in the log file.') sys.exit(1) - setup_connection_manager_db(args) - client.ConnectionManager.port = util.get_free_port() + sql_server = SQLServer.from_cmdline_args(args, context, check_env) + conn_mgr = client.ConnectionManager(sql_server, 'localhost', util.get_free_port()) if args.jobs <= 0: args.jobs = 1 @@ -368,9 +339,8 @@ def handle_check(args): if os.path.exists(suppress_file): send_suppress = True - client.ConnectionManager.run_env = check_env - - client.ConnectionManager.start_server(args.dbname, context) + sql_server.start(wait_for_start=True, init=True) + conn_mgr.start_report_server(context.db_version_info) LOG.debug("Checker server started.") diff --git a/codechecker_lib/client.py b/codechecker_lib/client.py index 9016f9d986..14c0ad0313 100644 --- a/codechecker_lib/client.py +++ b/codechecker_lib/client.py @@ -181,7 +181,7 @@ def send_config(connection, file_name): def get_connection(): ''' Automatic Connection handler via ContextManager idiom. You can use this in with statement.''' - connection = Connection() + connection = ConnectionManager.instance.create_connection() try: yield connection @@ -197,13 +197,13 @@ class Connection(object): Information what this use come from ConnectionManager class. So, you should set it up before create a connection.''' - def __init__(self): + def __init__(self, host, port): ''' Establish the connection beetwen client and server. ''' + tries_count = 0 while True: try: - self._transport = TSocket.TSocket(ConnectionManager.host, - ConnectionManager.port) + self._transport = TSocket.TSocket(host, port) self._transport = TTransport.TBufferedTransport(self._transport) self._protocol = TBinaryProtocol.TBinaryProtocol(self._transport) self._client = CheckerReport.Client(self._protocol) @@ -272,10 +272,6 @@ def add_config_info(self, config_list): ''' bool addConfigInfo(1: i64 run_id, 2: list values) ''' return self._client.addConfigInfo(ConnectionManager.run_id, config_list) - # def add_skip_path(self, path): - # ''' bool addSkipPath(1: i64 run_id, 2: list paths) ''' - # return self._client.addSkipPath(ConnectionManager.run_id, path) - def add_build_action(self, build_cmd, check_cmd): ''' i64 addBuildAction(1: i64 run_id, 2: string build_cmd) ''' return self._client.addBuildAction(ConnectionManager.run_id, @@ -306,164 +302,58 @@ def add_file_content(self, file_id, file_content): # ----------------------------------------------------------------------------- class ConnectionManager(object): - ''' ContextManager class for handling connections. - Store common information for about connection. - Start and stop the server.''' - host = 'localhost' - port = None - database_host = 'localhost' - database_port = 8764 - run_id = None - _database = None - _server = None - run_env = None - - # ------------------------------------------------------------------------- - @classmethod - def start_postgres(cls, context, init_db=True): - ''' - init_db : Initialize database locally if possible - ''' - - dbusername = context.db_username - - LOG.info('Checking for database') - if not database_handler.is_database_running(cls.database_host, - cls.database_port, - dbusername, cls.run_env): - LOG.info('Database is not running yet') - # On remote host we cannot initialize a new database - if not util.is_localhost(cls.database_host): - sys.exit(1) + ''' + ContextManager class for handling connections. + Store common information for about connection. + Start and stop the server. + ''' - db_path = context.database_path - if init_db: - if not database_handler.is_database_exist(db_path) and \ - not database_handler.initialize_database(db_path, dbusername, - cls.run_env): - # The database does not exist and cannot create - LOG.error('Database is missing and the initialization ' - 'of a new failed!') - LOG.error('Please check your configuration!') - sys.exit(1) - else: - if not database_handler.is_database_exist(db_path): - # The database does not exists - LOG.error('Database is missing!') - LOG.error('Please check your configuration!') - sys.exit(1) + run_id = None - LOG.info('Starting database') - cls._database = database_handler.start_database(db_path, - cls.database_host, - cls.database_port, - cls.run_env) - atexit.register(cls._database.terminate) + def __init__(self, database_server, host, port): + self.database_server = database_server + self.host = host + self.port = port + ConnectionManager.instance = self - # ------------------------------------------------------------------------- - @classmethod - def block_until_db_starts(cls, context): - ''' Wait for database to start if the database was - started by this client and polling is possible. ''' + def create_connection(self): + return Connection(self.host, self.port) - tries_count = 0 + def start_report_server(self, db_version_info): - while not database_handler.is_database_running(cls.database_host, - cls.database_port, context.db_username, cls.run_env) and \ - tries_count < 5: - tries_count += 1 + is_server_started = multiprocessing.Event() + server = multiprocessing.Process(target=report_server.run_server, + args=( + self.port, + self.database_server.get_connection_string(), + db_version_info, + is_server_started)) + + server.daemon = True + server.start() + + # Wait a bit + counter = 0 + while not is_server_started.is_set() and counter < 4: + LOG.debug('Waiting for checker server to start.') time.sleep(3) + counter += 1 - if tries_count >= 5 or not cls._database.poll(): + if counter >= 4 or not server.is_alive(): # last chance to start - if cls._database.returncode is None: + if server.exitcode is None: # it is possible that the database starts really slow - time.sleep(20) - if not database_handler.is_database_running(cls.host, - cls.database_port, context.db_username, cls.run_env): - - LOG.error('Failed to start database.') - sys.exit(1) - - else: - LOG.error('Failed to start database server.') - LOG.error('Database server exit code: '+str(cls._server.returncode)) - sys.exit(1) - - # ------------------------------------------------------------------------- - # Server related methods - @classmethod - def start_server(cls, dbname, context, wait_for_start=True): - - cls.start_postgres(context) - - if wait_for_start: - cls.block_until_db_start_proc_free(context) - - is_server_started = multiprocessing.Event() - - cls._server = multiprocessing.Process(target=report_server.run_server, - args=( - context.db_username, - cls.port, dbname, - cls.database_host, - cls.database_port, - context.db_version_info, - context.migration_root, - is_server_started)) - - cls._server.daemon = True - cls._server.start() - - if wait_for_start: - # Wait a bit - counter = 0 - while not is_server_started.is_set() and counter < 4: - LOG.debug('Waiting for checker server to start.') - time.sleep(3) - counter += 1 - - if counter >= 4 or not cls._server.is_alive(): - # last chance to start - if cls._server.exitcode is None: - # it is possible that the database starts really slow - time.sleep(5) - if not is_server_started.is_set(): - LOG.error('Failed to start checker server.') - sys.exit(1) - else: + time.sleep(5) + if not is_server_started.is_set(): LOG.error('Failed to start checker server.') - LOG.error('Checker server exit code: ' + - str(cls._server.exitcode)) sys.exit(1) + else: + LOG.error('Failed to start checker server.') + LOG.error('Checker server exit code: ' + + str(server.exitcode)) + sys.exit(1) - atexit.register(cls._server.terminate) - LOG.debug('Checker server start sequence done.') - - # ------------------------------------------------------------------------- - @classmethod - def stop_server(cls): - # if ConnectionManager._database: - # ConnectionManager._database.terminate() - - if ConnectionManager._server: - ConnectionManager._server.terminate() - - # ------------------------------------------------------------------------- - @classmethod - def block_until_db_start_proc_free(cls, context): - ''' Wait for database if the database process was stared - with a different client. No polling is possible.''' + atexit.register(server.terminate) + self.server = server - tries_count = 0 - max_try = 20 - timeout = 5 - while not database_handler.is_database_running(cls.database_host, - cls.database_port, context.db_username, cls.run_env) and \ - tries_count < max_try: - tries_count += 1 - time.sleep(timeout) - - if tries_count >= max_try: - LOG.error('Failed to start database.') - sys.exit(1) + LOG.debug('Checker server start sequence done.') diff --git a/codechecker_lib/database_handler.py b/codechecker_lib/database_handler.py index 991b2b5e9f..f3a3887214 100644 --- a/codechecker_lib/database_handler.py +++ b/codechecker_lib/database_handler.py @@ -4,74 +4,379 @@ # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- ''' -handle postgres database initialzation and start if required +Database server handling. ''' + import os import subprocess +import atexit +import time +import sys + +from abc import ABCMeta, abstractmethod +from codechecker_lib import util from codechecker_lib import logger +from codechecker_lib import pgpass +from codechecker_lib import host_check + +from db_model.orm_model import * + +import sqlalchemy +from sqlalchemy.engine.url import URL + +from alembic import command, config +from alembic.migration import MigrationContext LOG = logger.get_new_logger('DB_HANDLER') +class SQLServer(object): + ''' + Abstract base class for database server handling. An SQLServer instance is + responsible for the initialization, starting, and stopping the database + server, and also for connection string management. + + SQLServer implementations are created via SQLServer.from_cmdline_args(). + + How to add a new database server implementation: + 1, Derive from SQLServer and implement the abstract methods + 2, Add/modify some command line options in CodeChecker.py + 3, Modify SQLServer.from_cmdline_args() in order to create an + instance of the new server type if needed + ''' + + __metaclass__ = ABCMeta + + + def __init__(self, migration_root): + ''' + Sets self.migration_root. migration_root should be the path to the + alembic migration scripts. + ''' + + self.migration_root = migration_root + + + def _create_or_update_schema(self, use_migration=True): + ''' + Creates or updates the database schema. The database server should be + started before this method is called. + + If use_migration is True, this method runs an alembic upgrade to HEAD. + + In the False case, there is no migration support and only SQLAlchemy + meta data is used for schema creation. + + On error sys.exit(1) is called. + ''' + + try: + db_uri = self.get_connection_string() + engine = sqlalchemy.create_engine(db_uri, strategy='threadlocal') + + LOG.debug('Creating new database schema') + if use_migration: + LOG.debug('Creating new database session') + session = CreateSession(engine) + connection = session.connection() + + cfg = config.Config() + cfg.set_main_option("script_location", self.migration_root) + cfg.attributes["connection"] = connection + command.upgrade(cfg, "head") + + session.commit() + else: + CC_META.create_all(engine) + + LOG.debug('Creating new database schema done') + return True + + except sqlalchemy.exc.SQLAlchemyError as alch_err: + LOG.error(str(alch_err)) + sys.exit(1) + + + @abstractmethod + def start(self, wait_for_start=True, init=False): + ''' + Starts the database server and initializes the database server. + + On wait_for_start == True, this method returns when the server is up + and ready for connections. Otherwise it only starts the server and + returns immediately. + + On init == True, this it also initializes the database data and schema + if needed. + + On error sys.exit(1) should be called. + ''' + pass + + + @abstractmethod + def stop(self): + ''' + Terminates the database server. + + On error sys.exit(1) should be called. + ''' + pass + + + @abstractmethod + def get_connection_string(self): + ''' + Returns the connection string for SQLAlchemy. + + DO NOT LOG THE CONNECTION STRING BECAUSE IT MAY CONTAIN THE PASSWORD + FOR THE DATABASE! + ''' + pass + + + @classmethod + def from_cmdline_args(cls, args, context, env=None): + ''' + Normally only this method is called form outside of this module in + order to instance the proper server implementation. + + Parameters: + args: the command line arguments from CodeChecker.py + context: a Context object + env: a run environment dictionary. + ''' + + if not host_check.check_sql_driver(args.sqlite): + LOG.error("SQL driver error") + sys.exit(1) + + if args.sqlite: + LOG.debug("Using SQLiteDatabase") + return SQLiteDatabase(context, run_env=env) + else: + LOG.debug("Using PostgreSQLServer") + return PostgreSQLServer(context, + args.dbaddress, + args.dbport, + args.dbusername, + args.dbname, + run_env=env) + +class PostgreSQLServer(SQLServer): + ''' + Handler for PostgreSQL. + ''' + + def __init__(self, context, host, port, user, database, password = None, run_env=None): + super(PostgreSQLServer, self).__init__(context.migration_root) + + self.path = os.path.join(context.codechecker_workspace, 'pgsql_data') + self.host = host + self.port = port + self.user = user + self.database = database + self.password = password + self.run_env = run_env + + self.proc = None + + def _call_command(self, command): + ''' Call an external command and return with (output, return_code).''' + + try: + proc = subprocess.Popen(command, + bufsize=-1, + env=self.run_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + + stdout = proc.communicate()[0] + LOG.debug(stdout) + return stdout, proc.returncode + except OSError as oex: + LOG.error('Running command "' + ' '.join(command) + '" Failed') + LOG.error(str(oex)) + return '', 1 + + + def _is_database_data_exist(self): + '''Check the PostgreSQL instance existence in a given path.''' + + LOG.debug('Checking for database at ' + self.path) + + return os.path.exists(self.path) and \ + os.path.exists(os.path.join(self.path, 'PG_VERSION')) and \ + os.path.exists(os.path.join(self.path, 'base')) + + + def _initialize_database_data(self): + '''Initialize a PostgreSQL instance with initdb. ''' + LOG.debug('Initializing database at ' + self.path) + + init_db = ['initdb', '-U', self.user, '-D', self.path, '-E SQL_ASCII'] + + err, code = self._call_command(init_db) + # logger -> print error + return code == 0 + + def _get_connection_string(self, database): + port = str(self.port) + driver = host_check.get_postgresql_driver_name() + password = self.password + if driver == 'pg8000' and not password: + pfilepath = os.environ.get('PGPASSFILE') + if pfilepath: + password = pgpass.get_password_from_file(pfilepath, + self.host, + port, + database, + self.user) + + extra_args = {'client_encoding': 'utf8'} + return str(URL('postgresql+' + driver, + username=self.user, + password=password, + host=self.host, + port=port, + database=database, + query=extra_args)) + + + def _wait_or_die(self): + ''' + Wait for database if the database process was stared + with a different client. No polling is possible. + ''' + + LOG.debug('Waiting for PostgreSQL') + tries_count = 0 + max_try = 20 + timeout = 5 + while not self._is_running() and tries_count < max_try: + tries_count += 1 + time.sleep(timeout) + + if tries_count >= max_try: + LOG.error('Failed to start database.') + sys.exit(1) + + + def _create_database(self): + try: + LOG.debug('Creating new database if not exists') + + db_uri = self._get_connection_string('postgres') + engine = sqlalchemy.create_engine(db_uri) + text = "SELECT 1 FROM pg_database WHERE datname='%s'" % self.database + if not bool(engine.execute(text).scalar()): + conn = engine.connect() + # From sqlalchemy documentation: + # The psycopg2 and pg8000 dialects also offer the special level AUTOCOMMIT. + conn = conn.execution_options(isolation_level="AUTOCOMMIT") + conn.execute('CREATE DATABASE "%s"' % self.database) + conn.close() + + LOG.debug('Database created: ' + self.database) + + LOG.debug('Database already exists: ' + self.database) + + except sqlalchemy.exc.SQLAlchemyError as alch_err: + LOG.error('Failed to create database!') + LOG.error(str(alch_err)) + sys.exit(1) + + + def _is_running(self): + '''Is there PostgreSQL instance running on a given host and port.''' + + LOG.debug('Checking if database is running at ' + self.host + ':' + str(self.port)) + + check_db = ['psql', '-U', self.user, '-l', '-p', str(self.port), '-h', self.host] + err, code = self._call_command(check_db,) + return code == 0 + + + def start(self, wait_for_start=True, init=False): + ''' + Start a PostgreSQL instance with given path, host and port. + Return with process instance + ''' + + if not self._is_running(): + if not util.is_localhost(self.host): + LOG.info('Database is not running yet') + sys.exit(1) + + if not self._is_database_data_exist(): + if not init: + # The database does not exists + LOG.error('Database data is missing!') + LOG.error('Please check your configuration!') + sys.exit(1) + elif not self._initialize_database_data(): + # The database does not exist and cannot create + LOG.error('Database data is missing and the initialization ' + 'of a new failed!') + LOG.error('Please check your configuration!') + sys.exit(1) + + LOG.info('Starting database') + LOG.debug('Starting database at ' + self.host + ':' + str(self.port) + ' ' + self.path) + devnull = open(os.devnull, 'wb') + + start_db = ['postgres', '-i', '-D', self.path, '-p', str(self.port), '-h', self.host] + self.proc = subprocess.Popen(start_db, + bufsize=-1, + env=self.run_env, + stdout=devnull, + stderr=subprocess.STDOUT) + + if init: + self._wait_or_die() + self._create_database() + self._create_or_update_schema() + elif wait_for_start: + self._wait_or_die() + + atexit.register(self.stop) -# ----------------------------------------------------------------------------- -def call_command(command, run_env=None): - ''' Call an external command and return with (output, return_code).''' - try: - proc = subprocess.Popen(command, bufsize=-1, env=run_env, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - stdout = proc.communicate()[0] - LOG.debug(stdout) - return stdout, proc.returncode - except OSError as oex: - LOG.error('Running command "' + ' '.join(command) + '" Failed') - LOG.error(str(oex)) - return '', 1 + def stop(self): + if self.proc: + LOG.debug('Terminating database') + self.proc.terminate() -# ----------------------------------------------------------------------------- -def is_database_exist(path): - ''' Check the postgres instance existence in a given path.''' - path = path.strip() - LOG.debug('Checking for database at ' + path) + def get_connection_string(self): + return self._get_connection_string(self.database) - return os.path.exists(path) and \ - os.path.exists(os.path.join(path, 'PG_VERSION')) and \ - os.path.exists(os.path.join(path, 'base')) +class SQLiteDatabase(SQLServer): + ''' + Handler for SQLite. + ''' -# ----------------------------------------------------------------------------- -def is_database_running(host, port, dbusername, analyzer_env=None): - ''' Is there postgres instance running on a given host and port.''' - LOG.debug('Checking if database is running at ' + host + ':' + str(port)) + def __init__(self, context, run_env=None): + super(SQLiteDatabase, self).__init__(context.migration_root) - check_db = ['psql', '-U', dbusername, '-l', '-p', str(port), '-h', host] - err, code = call_command(check_db, analyzer_env) - return code == 0 + self.dbpath = os.path.join(context.codechecker_workspace, 'codechecker.sqlite') + self.run_env = run_env -# ----------------------------------------------------------------------------- -def initialize_database(path, db_username, analyzer_env=None): - ''' Initalize a postgres instance with initdb. ''' - LOG.debug('Initializing database at ' + path) + def start(self, wait_for_start=True, init=False): + if init: + self._create_or_update_schema(use_migration=False) - init_db = ['initdb', '-U', db_username, '-D', path, '-E SQL_ASCII'] + if not os.path.exists(self.dbpath): + # The database does not exists + LOG.error('Database (%s) is missing!' % self.dbpath) + sys.exit(1) - err, code = call_command(init_db, analyzer_env) - # logger -> print error - return code == 0 + def stop(self): + pass -# ----------------------------------------------------------------------------- -def start_database(path, host, port, analyzer_env=None): - ''' Start a postgres instance with given path, host and port. - Return with process instance. ''' - LOG.debug('Starting database at ' + host + ':' + str(port) + ' ' + path) - devnull = open(os.devnull, 'wb') - proc = subprocess.Popen(['postgres', '-i', '-D', path, - '-p', str(port), '-h', host], - bufsize=-1, env=analyzer_env, - stdout=devnull, stderr=subprocess.STDOUT) - return proc + def get_connection_string(self): + return str(URL('sqlite+pysqlite', None, None, None, None, self.dbpath)) diff --git a/codechecker_lib/debug_reporter.py b/codechecker_lib/debug_reporter.py index c37e574419..16164da7b9 100644 --- a/codechecker_lib/debug_reporter.py +++ b/codechecker_lib/debug_reporter.py @@ -25,11 +25,8 @@ def get_dump_file_name(run_id, action_id): # ----------------------------------------------------------------------------- -def debug(context, dbusername, dbhost, dbport, dbname, force): +def debug(context, connection_string, force): try: - connection_string = util.create_postgresql_connection_string( - dbusername, dbhost, dbport, dbname) - engine = sqlalchemy.create_engine(connection_string) session = sqlalchemy.orm.scoped_session( sqlalchemy.orm.sessionmaker(bind=engine)) diff --git a/codechecker_lib/host_check.py b/codechecker_lib/host_check.py index ae9f8bd41e..a6b5f3bce7 100644 --- a/codechecker_lib/host_check.py +++ b/codechecker_lib/host_check.py @@ -65,6 +65,20 @@ def check_postgresql_driver(): return False +# ----------------------------------------------------------------------------- +def check_sql_driver(check_sqlite): + if check_sqlite: + # pysqlite driver (which is the same as sqlite3 module) is + # included with the Python distribution + return True + else: + try: + get_postgresql_driver_name() + return True + except Exception as ex: + return False + + # ----------------------------------------------------------------------------- def check_clang(compiler_bin, env): ''' diff --git a/codechecker_lib/util.py b/codechecker_lib/util.py index 96ff746403..03af507320 100644 --- a/codechecker_lib/util.py +++ b/codechecker_lib/util.py @@ -17,10 +17,6 @@ import shutil from codechecker_lib import logger -from codechecker_lib import host_check -from codechecker_lib import pgpass - -from sqlalchemy.engine.url import URL # WARNING! LOG should be only used in this module LOG = logger.get_new_logger('UTIL') @@ -174,15 +170,3 @@ def error_handler(*args): LOG.warning('Failed to remove directory %s' % (path)) shutil.rmtree(path, onerror=error_handler) - - -# ------------------------------------------------------------------------- -def create_postgresql_connection_string(user, host, port, database, password=None): - port = str(port) - driver = host_check.get_postgresql_driver_name() - if driver == 'pg8000' and not password: - path = os.environ.get('PGPASSFILE') - if path: - password = pgpass.get_password_from_file(path, host, port, database, user) - - return str(URL('postgresql+' + driver, user, password, host, port, database)) diff --git a/db_model/orm_model.py b/db_model/orm_model.py index b6fc3c4490..b44e28b67b 100644 --- a/db_model/orm_model.py +++ b/db_model/orm_model.py @@ -14,12 +14,13 @@ from sqlalchemy.sql.expression import true from sqlalchemy.ext.declarative import declarative_base -from datetime import * +from datetime import datetime +from math import ceil CC_META = MetaData(naming_convention={ "ix": 'ix_%(column_0_label)s', "uq": "uq_%(table_name)s_%(column_0_name)s", - "ck": "ck_%(table_name)s_%(constraint_name)s", + "ck": "ck_%(table_name)s_%(column_0_name)s", "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", "pk": "pk_%(table_name)s" }) @@ -67,7 +68,7 @@ def __init__(self, name, version, command): self.inc_count = 0 def mark_finished(self): - self.duration = (datetime.now() - self.date).total_seconds() + self.duration = ceil((datetime.now() - self.date).total_seconds()) class Config(Base): diff --git a/external-source-deps/build-logger/src/ldlogger-tool-javac.c b/external-source-deps/build-logger/src/ldlogger-tool-javac.c index 714d54405b..f3bddc87ae 100644 --- a/external-source-deps/build-logger/src/ldlogger-tool-javac.c +++ b/external-source-deps/build-logger/src/ldlogger-tool-javac.c @@ -270,7 +270,7 @@ int loggerJavacParserCollectActions( ParserData data; size_t i; - assert(prog_ == prog_); /* supress unused variable */ + assert(prog_ == prog_); /* suppress unused variable */ data.hasSourcePath = 0; data.state = Normal; diff --git a/storage_server/report_server.py b/storage_server/report_server.py index ccae4996e6..9ee6b6d26b 100644 --- a/storage_server/report_server.py +++ b/storage_server/report_server.py @@ -15,8 +15,6 @@ import ntpath import sqlalchemy -from alembic import command, config -from alembic.migration import MigrationContext from thrift.transport import TSocket from thrift.transport import TTransport @@ -498,63 +496,15 @@ def __init__(self, session, lockDB): Report.start_bugpoint) -def create_db_if_not_exists(dbusername, dbhost, dbport, db_name): - ''' True -> created, False -> already exists and do nothing. ''' - LOG.debug('Creating new database if not exists') - - db_uri = util.create_postgresql_connection_string(dbusername, dbhost, dbport, 'postgres') - engine = sqlalchemy.create_engine(db_uri, client_encoding='utf8') - - text = "SELECT 1 FROM pg_database WHERE datname='%s'" % db_name - if not bool(engine.execute(text).scalar()): - conn = engine.connect() - # From sqlalchemy documentation: - # The psycopg2 and pg8000 dialects also offer the special level AUTOCOMMIT. - conn = conn.execution_options(isolation_level="AUTOCOMMIT") - conn.execute('CREATE DATABASE "%s"' % db_name) - conn.close() - - LOG.debug('Database created: ' + db_name) - return True - - LOG.debug('Database already exists: ' + db_name) - return False - -def migrate_database(session, migration_scripts): - LOG.debug('Updating schema to HEAD...') - - connection = session.connection() - - cfg = config.Config() - cfg.set_main_option("script_location", migration_scripts) - cfg.attributes["connection"] = connection - command.upgrade(cfg, "head") - -def run_server(dbUsername, port, db_name, dbhost, dbport, db_version_info, migration_scripts, callback_event=None): +def run_server(port, db_uri, db_version_info, callback_event=None): LOG.debug('Starting codechecker server ...') try: - - ret = create_db_if_not_exists(dbUsername, dbhost, dbport, db_name) - LOG.debug('Database exists: ' + str(ret)) - except sqlalchemy.exc.SQLAlchemyError as alch_err: - LOG.error(str(alch_err)) - sys.exit(1) - - try: - db_uri = util.create_postgresql_connection_string(dbUsername, dbhost, dbport, db_name) engine = sqlalchemy.create_engine(db_uri, strategy='threadlocal') LOG.debug('Creating new database session') session = CreateSession(engine) - LOG.debug('Creating new database schema') - start = datetime.now() - migrate_database(session, migration_scripts) - end = datetime.now() - diff = end - start - LOG.debug('Creating new database schema done in ' + str(diff.seconds)) - version = session.query(DBVersion).first() if version is None: # Version is not populated yet @@ -573,8 +523,6 @@ def run_server(dbUsername, port, db_name, dbhost, dbport, db_version_info, migra except sqlalchemy.exc.SQLAlchemyError as alch_err: LOG.error(str(alch_err)) sys.exit(1) - finally: - session.commit() session.autoflush = False # autoflush is enabled by default diff --git a/test_package.py b/test_package.py index 587c00cb50..78b7d62aef 100755 --- a/test_package.py +++ b/test_package.py @@ -127,7 +127,9 @@ def __init__(self, test_proj_path, test_project_config, test_modules, - clang_version, log): + clang_version, + log, + cmd_args): self.pkg_root = pkg_root self.log = log @@ -135,6 +137,8 @@ def __init__(self, self.database = database self.start_test_client = \ partial(start_test_client, pkg_root, test_modules) + self.use_sqlite = cmd_args.sqlite + try: if test_project_config is None: with open(os.path.join(test_proj_path, 'project_info.json')) as inf_file: @@ -244,53 +248,22 @@ def run_test(self): self.env['CODECHECKER_VERBOSE'] = 'debug' # self.env['CODECHECKER_ALCHEMY_LOG'] = '2' - def first_check(suppress_file): - - check_cmd = [] - check_cmd.append('CodeChecker') - check_cmd.append('check') - check_cmd.append('--dbaddress') - check_cmd.append(db['dbaddress']) - check_cmd.append('--dbport') - check_cmd.append(str(db['dbport'])) - check_cmd.append('--dbname') - check_cmd.append(db['dbname']) - check_cmd.append('--dbusername') - check_cmd.append(db['dbusername']) - check_cmd.append('-w') - check_cmd.append(codechecker_workspace) - check_cmd.append('--suppress') - check_cmd.append(suppress_file) - unique_id = uuid.uuid4().hex - check_cmd.append('-n') - check_cmd.append(self.project_info['name'] + '_' + unique_id) - check_cmd.append('-b') - check_cmd.append(test_project_build_cmd) - - self.log.info(' '.join(check_cmd)) - - try: - subprocess.check_call(check_cmd, cwd=test_project_path, - env=self.env) - self.log.info('Checking the test project is done.') - except subprocess.CalledProcessError as perr: - self.log.error(str(perr)) - self.log.error('Failed to run command: ' + ' '.join(check_cmd)) - raise perr - - def second_check(suppress_file): + def run_check(suppress_file): check_cmd = [] check_cmd.append('CodeChecker') check_cmd.append('check') - check_cmd.append('--dbaddress') - check_cmd.append(db['dbaddress']) - check_cmd.append('--dbport') - check_cmd.append(str(db['dbport'])) - check_cmd.append('--dbname') - check_cmd.append(db['dbname']) - check_cmd.append('--dbusername') - check_cmd.append(db['dbusername']) + if self.use_sqlite: + check_cmd.append('--sqlite') + else: + check_cmd.append('--dbaddress') + check_cmd.append(db['dbaddress']) + check_cmd.append('--dbport') + check_cmd.append(str(db['dbport'])) + check_cmd.append('--dbname') + check_cmd.append(db['dbname']) + check_cmd.append('--dbusername') + check_cmd.append(db['dbusername']) check_cmd.append('-w') check_cmd.append(codechecker_workspace) check_cmd.append('--suppress') @@ -318,14 +291,17 @@ def start_server(): server_cmd.append('server') server_cmd.append('--check-port') server_cmd.append(str(test_config['CC_TEST_SERVER_PORT'])) - server_cmd.append('--dbaddress') - server_cmd.append(db['dbaddress']) - server_cmd.append('--dbport') - server_cmd.append(str(db['dbport'])) - server_cmd.append('--dbname') - server_cmd.append(db['dbname']) - server_cmd.append('--dbusername') - server_cmd.append(db['dbusername']) + if self.use_sqlite: + server_cmd.append('--sqlite') + else: + server_cmd.append('--dbaddress') + server_cmd.append(db['dbaddress']) + server_cmd.append('--dbport') + server_cmd.append(str(db['dbport'])) + server_cmd.append('--dbname') + server_cmd.append(db['dbname']) + server_cmd.append('--dbusername') + server_cmd.append(db['dbusername']) server_cmd.append('--view-port') server_cmd.append(str(test_config['CC_TEST_VIEWER_PORT'])) server_cmd.append('-w') @@ -375,14 +351,14 @@ def start_server(): self.log.info('Cleaning test project') # fist check self._clean_test_project(test_project_path, test_project_clean_cmd) - first_check(suppress_file) + run_check(suppress_file) time.sleep(5) stop_server = multiprocessing.Event() test_module_error = multiprocessing.Event() # second check self._clean_test_project(test_project_path, test_project_clean_cmd) - second_check(suppress_file) + run_check(suppress_file) time.sleep(5) stop_server = multiprocessing.Event() @@ -447,6 +423,9 @@ def main(): action='store', dest='test_project_config', help='Test project config. By default tries to use from the test_project.') + parser.add_argument('--sqlite', dest="sqlite", + action='store_true', required=False, + help='Use sqlite database.') parser.add_argument('--dbaddress', type=str, dest="dbaddress", default='localhost', help='Postgres database server address.') @@ -472,7 +451,9 @@ def main(): args.test_project, args.test_project_config, args.test_modules, - args.clang_version, LOG) + args.clang_version, + LOG, + args) LOG.info('Running tests') package_tester.run_test() diff --git a/viewer_server/client_db_access_handler.py b/viewer_server/client_db_access_handler.py index 621d2798cc..a261eb7371 100644 --- a/viewer_server/client_db_access_handler.py +++ b/viewer_server/client_db_access_handler.py @@ -102,9 +102,9 @@ def construct_report_filter(report_filters): # severity value can be 0 AND.append(Report.severity == report_filter.severity) if report_filter.suppressed: - AND.append(Report.suppressed == 'True') + AND.append(Report.suppressed == True) else: - AND.append(Report.suppressed == 'False') + AND.append(Report.suppressed == False) OR.append(and_(*AND)) @@ -583,21 +583,21 @@ def __update_suppress_storage_data(self, already_suppressed_runids = \ filter(lambda bug: bug.run_id in run_ids, set(suppressed)) - unsupress_in_these_runs = \ + unsuppress_in_these_runs = \ {bug.run_id for bug in already_suppressed_runids} LOG.debug('Already suppressed, unsuppressing now') suppressed = session.query(SuppressBug) \ .filter(and_(SuppressBug.hash == bug_id_hash, SuppressBug.file_name == source_file_name, - SuppressBug.run_id.in_(unsupress_in_these_runs))) + SuppressBug.run_id.in_(unsuppress_in_these_runs))) # delete supppress bug entries for sp in suppressed: session.delete(sp) # update report entries self.__set_report_suppress_flag(session, - unsupress_in_these_runs, + unsuppress_in_these_runs, bug_id_hash, source_file_name, suppress_flag=suppress) diff --git a/viewer_server/client_db_access_server.py b/viewer_server/client_db_access_server.py index fb8116484f..701ced4cee 100644 --- a/viewer_server/client_db_access_server.py +++ b/viewer_server/client_db_access_server.py @@ -159,7 +159,7 @@ def __init__(self, self.suppress_handler = suppress_handler self.db_version_info = db_version_info self.__engine = sqlalchemy.create_engine(db_conn_string, - client_encoding='utf8', + encoding='utf8', poolclass=sqlalchemy.pool.NullPool) Session = scoped_session(sessionmaker()) From cea9d98544f3702bbeb4d30b604ee0e9344dafb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20G=C3=A1bor=20Isp=C3=A1novics?= Date: Wed, 25 Nov 2015 16:09:37 +0100 Subject: [PATCH 2/3] Add documentation for SQLite --- README.md | 6 +++--- docs/user_guide.md | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 8eda2aafae..a794b5d904 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ CodeChecker replaces [scan-build](http://clang-analyzer.llvm.org/scan-build.html It provides * a new command line tool for analyzing projects * dynamic web based defect viewer (instead of static html) - * a Postgresql based defect storage & management + * a PostgreSQL/SQLite based defect storage & management * incremental bug reporting (show only new bugs compared to a baseline) * suppression of false positives * better integration with build systems (through the LD_PRELOAD mechanism) @@ -55,12 +55,12 @@ If your clang version does not have these features you will see warning messages ### Runtime requirements * [Clang Static analyzer](http://clang-analyzer.llvm.org/) (latest stable or [trunk](http://clang.llvm.org/get_started.html)) - * [Postgresql](http://www.postgresql.org/ "PostgreSql") (> 9.3.5) + * [PostgreSQL](http://www.postgresql.org/ "PostgreSQL") (> 9.3.5) (optional) * [Python2](https://www.python.org/) (> 2.7) * [Alembic](https://pypi.python.org/pypi/alembic) (>=0.8.2) * [SQLAlchemy](http://www.sqlalchemy.org/) (> 1.0.2) - [PyPi SQLAlchemy](https://pypi.python.org/pypi/SQLAlchemy) (> 1.0.2) - * [psycopg2](http://initd.org/psycopg/ "psycopg2") (> 2.5.4) or [pg8000](https://github.com/mfenniak/pg8000 "pg8000") (>= 1.10.0) + * [psycopg2](http://initd.org/psycopg/ "psycopg2") (> 2.5.4) or [pg8000](https://github.com/mfenniak/pg8000 "pg8000") (>= 1.10.0) (optional) - [PyPi psycopg2](https://pypi.python.org/pypi/psycopg2/2.6.1) __requires lbpq!__ - [PyPi pg8000](https://pypi.python.org/pypi/pg8000) * Thrift python modules diff --git a/docs/user_guide.md b/docs/user_guide.md index 0f78dd2331..525cb0b552 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -3,7 +3,7 @@ ##CodeChecker usage First of all, you have to setup the environment for CodeChecker. -Codechecker server uses PostgreSQL database to store the results which is also packed into the package. +Codechecker server uses PostgreSQL database (by default) to store the results which is also packed into the package. ~~~~~~~~~~~~~~~~~~~~~ cd $CODECHECKER_PACKAGE_ROOT/init @@ -40,7 +40,7 @@ optional arguments: ##Default configuration: Used ports: -* 8764 - PostgreSql +* 8764 - PostgreSQL * 11444 - CodeChecker result viewer ## 1. log mode: @@ -96,14 +96,13 @@ CodeChecker check -w ~/codechecker_wp -n myProject -l ~/codechecker_wp/build_log ### Advanced Usage ~~~~~~~~~~~~~~~~~~~~~ -CodeChecker check --help usage: CodeChecker.py check [-h] -w WORKSPACE -n NAME (-b COMMAND | -l LOGFILE) [-j JOBS] [-f CONFIGFILE] [-s SKIPFILE] [-u SUPPRESS] - [-e ENABLE] [-d DISABLE] [-c] - [--dbaddress DBADDRESS] [--dbport DBPORT] + [-e ENABLE] [-d DISABLE] [-c [DEPRECATED]] + [--keep-tmp] [--update] [--sqlite] + [--dbport DBPORT] [--dbaddress DBADDRESS] [--dbname DBNAME] [--dbusername DBUSERNAME] - [--update] optional arguments: -h, --help show this help message and exit @@ -127,14 +126,35 @@ optional arguments: Enable checker. -d DISABLE, --disable DISABLE Disable checker. + -c [DEPRECATED], --clean [DEPRECATED] + DEPRECATED argument! --keep-tmp Keep temporary report files after sending data to database storage server. + --update Incremental parsing, update the results of a previous + run. + --sqlite Use sqlite database. + --dbport DBPORT Postgres server port. --dbaddress DBADDRESS - Postgres database server address. - --dbport DBPORT Postgres database server port. + Postgres database server address --dbname DBNAME Name of the database. + --dbusername DBUSERNAME + Database user name. ~~~~~~~~~~~~~~~~~~~~~ +### Using SQLite for database: + +CodeChecker can also use SQLite for storing the results. In this case the +SQLite database will be created in the workspace directory. + +In order to use SQLite instead of PostgreSQL, use the ```--sqlite``` command +line argument for ```CodeChecker server``` and ```CodeChecker check``` +commands. In this case ```--dbport```, ```--dbaddress```, ```--dbname```, and +```--dbusername``` command line arguments are ignored. + +#### Note: +Schema migration is not supported with SQLite. This means if you upgrade your +CodeChecker to a newer version, you might need to re-check your project. + ### Suppression in the source: Suppress comments can be used in the source to suppress specific or all checker results found in a source line. From 64c6472717f737d785cb642740496050b8b9e511 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20G=C3=A1bor=20Isp=C3=A1novics?= Date: Thu, 26 Nov 2015 11:46:49 +0100 Subject: [PATCH 3/3] Refactoring according to the pull request review --- codechecker_lib/arg_handler.py | 15 +++++++-- codechecker_lib/database_handler.py | 50 ++++++++++++----------------- codechecker_lib/util.py | 18 +++++++++++ 3 files changed, 50 insertions(+), 33 deletions(-) diff --git a/codechecker_lib/arg_handler.py b/codechecker_lib/arg_handler.py index eee6bf7503..858e75abc6 100644 --- a/codechecker_lib/arg_handler.py +++ b/codechecker_lib/arg_handler.py @@ -189,7 +189,10 @@ def handle_server(args): check_env = analyzer_env.get_check_env(context.path_env_extra, context.ld_lib_path_extra) - sql_server = SQLServer.from_cmdline_args(args, context, check_env) + sql_server = SQLServer.from_cmdline_args(args, + context.codechecker_workspace, + context.migration_root, + check_env) conn_mgr = client.ConnectionManager(sql_server, args.check_address, args.check_port) if args.check_port: LOG.debug('Starting codechecker server and database server.') @@ -248,7 +251,10 @@ def handle_debug(args): check_env = analyzer_env.get_check_env(context.path_env_extra, context.ld_lib_path_extra) - sql_server = SQLServer.from_cmdline_args(args, context, check_env) + sql_server = SQLServer.from_cmdline_args(args, + context.codechecker_workspace, + context.migration_root, + check_env) sql_server.start(wait_for_start=True, init=False) debug_reporter.debug(context, sql_server.get_connection_string(), args.force) @@ -324,7 +330,10 @@ def handle_check(args): LOG.warning('There are no build actions in the log file.') sys.exit(1) - sql_server = SQLServer.from_cmdline_args(args, context, check_env) + sql_server = SQLServer.from_cmdline_args(args, + context.codechecker_workspace, + context.migration_root, + check_env) conn_mgr = client.ConnectionManager(sql_server, 'localhost', util.get_free_port()) if args.jobs <= 0: diff --git a/codechecker_lib/database_handler.py b/codechecker_lib/database_handler.py index f3a3887214..ee41c95729 100644 --- a/codechecker_lib/database_handler.py +++ b/codechecker_lib/database_handler.py @@ -136,14 +136,15 @@ def get_connection_string(self): @classmethod - def from_cmdline_args(cls, args, context, env=None): + def from_cmdline_args(cls, args, workspace, migration_root, env=None): ''' Normally only this method is called form outside of this module in order to instance the proper server implementation. Parameters: args: the command line arguments from CodeChecker.py - context: a Context object + workspace: path to the CodeChecker workspace directory + migration_root: path to the database migration scripts env: a run environment dictionary. ''' @@ -153,10 +154,11 @@ def from_cmdline_args(cls, args, context, env=None): if args.sqlite: LOG.debug("Using SQLiteDatabase") - return SQLiteDatabase(context, run_env=env) + return SQLiteDatabase(workspace, migration_root, run_env=env) else: LOG.debug("Using PostgreSQLServer") - return PostgreSQLServer(context, + return PostgreSQLServer(workspace, + migration_root, args.dbaddress, args.dbport, args.dbusername, @@ -168,10 +170,10 @@ class PostgreSQLServer(SQLServer): Handler for PostgreSQL. ''' - def __init__(self, context, host, port, user, database, password = None, run_env=None): - super(PostgreSQLServer, self).__init__(context.migration_root) + def __init__(self, workspace, migration_root, host, port, user, database, password = None, run_env=None): + super(PostgreSQLServer, self).__init__(migration_root) - self.path = os.path.join(context.codechecker_workspace, 'pgsql_data') + self.path = os.path.join(workspace, 'pgsql_data') self.host = host self.port = port self.user = user @@ -181,24 +183,6 @@ def __init__(self, context, host, port, user, database, password = None, run_env self.proc = None - def _call_command(self, command): - ''' Call an external command and return with (output, return_code).''' - - try: - proc = subprocess.Popen(command, - bufsize=-1, - env=self.run_env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - stdout = proc.communicate()[0] - LOG.debug(stdout) - return stdout, proc.returncode - except OSError as oex: - LOG.error('Running command "' + ' '.join(command) + '" Failed') - LOG.error(str(oex)) - return '', 1 - def _is_database_data_exist(self): '''Check the PostgreSQL instance existence in a given path.''' @@ -212,15 +196,21 @@ def _is_database_data_exist(self): def _initialize_database_data(self): '''Initialize a PostgreSQL instance with initdb. ''' + LOG.debug('Initializing database at ' + self.path) init_db = ['initdb', '-U', self.user, '-D', self.path, '-E SQL_ASCII'] - err, code = self._call_command(init_db) + err, code = util.call_command(init_db, self.run_env) # logger -> print error return code == 0 + def _get_connection_string(self, database): + ''' + Helper method for getting the connection string for the given database. + ''' + port = str(self.port) driver = host_check.get_postgresql_driver_name() password = self.password @@ -293,7 +283,7 @@ def _is_running(self): LOG.debug('Checking if database is running at ' + self.host + ':' + str(self.port)) check_db = ['psql', '-U', self.user, '-l', '-p', str(self.port), '-h', self.host] - err, code = self._call_command(check_db,) + err, code = util.call_command(check_db, self.run_env) return code == 0 @@ -357,10 +347,10 @@ class SQLiteDatabase(SQLServer): Handler for SQLite. ''' - def __init__(self, context, run_env=None): - super(SQLiteDatabase, self).__init__(context.migration_root) + def __init__(self, workspace, migration_root, run_env=None): + super(SQLiteDatabase, self).__init__(migration_root) - self.dbpath = os.path.join(context.codechecker_workspace, 'codechecker.sqlite') + self.dbpath = os.path.join(workspace, 'codechecker.sqlite') self.run_env = run_env diff --git a/codechecker_lib/util.py b/codechecker_lib/util.py index 03af507320..de21889718 100644 --- a/codechecker_lib/util.py +++ b/codechecker_lib/util.py @@ -15,6 +15,7 @@ import glob import socket import shutil +import subprocess from codechecker_lib import logger @@ -170,3 +171,20 @@ def error_handler(*args): LOG.warning('Failed to remove directory %s' % (path)) shutil.rmtree(path, onerror=error_handler) + + +def call_command(command, env=None): + ''' Call an external command and return with (output, return_code).''' + + try: + LOG.debug('Run ' + ' '.join(command)) + out = subprocess.check_output(command, + bufsize=-1, + env=env, + stderr=subprocess.STDOUT) + LOG.debug(out) + return out, 0 + except subprocess.CalledProcessError as ex: + LOG.error('Running command "' + ' '.join(command) + '" Failed') + LOG.error(str(ex)) + return ex.output, ex.returncode