runtests

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright (C) 2014 Canonical
#
# Authors:
#  Didier Roche
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; version 3.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

import argparse
import configparser
import logging
import logging.config
import nose
import os
import yaml
import sys

root_dir = os.path.abspath(os.path.dirname(__file__))
config_dir = os.path.join(root_dir, 'confs')
DEBUG_CONFIG_FILE = os.path.join(config_dir, "debug.nose")
COVERAGE_CONFIG_FILE = os.path.join(config_dir, "prod.nose")
TESTS_DIR = os.path.join(root_dir, 'tests')
DEBUG_LOG_CONFIG = os.path.join(config_dir, "debug.logcfg")
TESTING_LOG_CONFIG = os.path.join(config_dir, "testing.logcfg")
# subprocess need to output on stdout the logs to be monitored
# the profile is the testing one + console output in warning mode
TESTING_SUBPROCESS_LOG_CONFIG = os.path.join(config_dir, "testing.subprocess.logcfg")


def transform_nose_config_to_cmd(filename):
    """Manually transform a nose config file to a cmd parameters

    This is needed in case the same parameter is repeated multiple times

    This return the cmd line array parameters."""

    cmd_line = []
    config = configparser.ConfigParser()
    config.read(filename)
    for key in config["nosetests"]:
        value = config["nosetests"][key]
        if value == '1':
            cmd_line.append('--' + key)
        # multiple values (what the config parameter for nose doesn't support)
        elif ',' in value:
            for val in value.split(','):
                cmd_line.append('--{}={}'.format(key, val))
        else:
            cmd_line.append('--{}={}'.format(key, value))
    return cmd_line


def set_logging_profile(log_config_file):
    """Set logging profile for current process and subprocesses"""
    with open(log_config_file, 'rt') as f:
        logging_config = yaml.load(f.read())
    logging.config.dictConfig(logging_config)
    os.environ["LOG_CFG"] = log_config_file
    if log_config_file == TESTING_LOG_CONFIG:
        os.environ["LOG_CFG"] = TESTING_SUBPROCESS_LOG_CONFIG


if __name__ == '__main__':

    # setup the environment in plain english so that we standardize the test bed (if some people have some .mo
    # Ubuntu Make files installed while having a locale set to it) to avoid getting translated strings not
    # matching our expectations.
    os.environ["LANGUAGE"] = "C"

    parser = argparse.ArgumentParser(description="Run umake tests. Specified list of test run in debug mode. "
                                                 "Running a partial suite of tests is using a normal mode. "
                                                 "Running all tests is using coverage by default.")
    parser.add_argument('-s', "--system", action="store_true", help="Use system umake instead of local one")
    parser.add_argument("--publish", action="store_true", help="Publish xunit results format")

    parser.add_argument("tests", nargs='*', help="Action to perform: all (or omitted) to run all tests. "
                                                 "small/medium/large/pep8 or nose syntax: "
                                                 "tests.small.test_frameworks_loader:TestFrameworkLoaderSaveConfig.foo")

    config_group = parser.add_argument_group('Run configuration options',
                                             description="The default configuration is to use the debug profile when "
                                                         "running some manually specific list of tests. No profile is "
                                                         "selected when running some suites of tests and coverage "
                                                         "profile when selecting all tests.")\
        .add_mutually_exclusive_group()
    config_group.add_argument("-c", "--config", help="Force using a particular nose profile, disable autoselection")
    config_group.add_argument("--coverage", action="store_true", help="Force using coverage profile even when some "
                                                                      "tests or tessuite")
    config_group.add_argument("--debug", action="store_true", help="Force using debug profile even when running "
                                                                   "all tests")
    config_group.add_argument("--no-config", action="store_true", help="Disable any automated or manual config "
                                                                       "selection")

    args = parser.parse_args()

    nose_args = []
    # nosetests captured logs format
    nose_args.extend(["--logging-format", "%(asctime)s [%(name)s] %(levelname)s: %(message)s"])

    ## handle config first
    specified_config = False
    if args.config:
        nose_args.extend(["--config", args.config])
        specified_config = True
    elif args.debug:
        nose_args.extend(["--config", DEBUG_CONFIG_FILE])
        set_logging_profile(DEBUG_LOG_CONFIG)
        specified_config = True
    elif args.coverage:
        nose_args.extend(transform_nose_config_to_cmd(COVERAGE_CONFIG_FILE))
        set_logging_profile(TESTING_LOG_CONFIG)
        specified_config = True
    elif args.no_config:
        specified_config = True

    ## output xunit and json if requested
    if args.publish:
        nose_args.append("--with-xunit")
        # FIXME: disable nose-json for now, incompatible with coverage reporting when an exception is raised.
        # we are going to switch to nose2 anyway.
        #nose_args.append("--with-json")

    ## check if we want to run those tests with the system code
    if args.system:
        nose_args.append("-P")
        # let remove it from there as well
        sys.path.remove(root_dir)
    else:
        import tests
        tests.tools.set_local_umake()

    if not "all" in args.tests and len(args.tests) > 0:
        for test_type in args.tests:
            for named_test_type in ("small", "medium", "large", "pep8"):
                if test_type == named_test_type:
                    if test_type == "pep8":
                        nose_args.append(os.path.join(TESTS_DIR, "__init__.py"))
                    else:
                        nose_args.append(os.path.join(TESTS_DIR, named_test_type))
                    break
            # Not a named test_type, but a list of tests to run
            else:
                nose_args.append(test_type)
                # If no config is given, choose debug by default for partial run
                if not specified_config:
                    nose_args.extend(["--config", DEBUG_CONFIG_FILE])
                    set_logging_profile(DEBUG_LOG_CONFIG)
                    specified_config = True
    else:
        # If no config is given, run with coverage
        if not specified_config:
            nose_args.extend(transform_nose_config_to_cmd(COVERAGE_CONFIG_FILE))
            set_logging_profile(TESTING_LOG_CONFIG)
            specified_config = True

    ## need a fake $0 argument
    nose_args.insert(0, "")
    nose.main(argv=nose_args)