build_helper.py

import logging
import multiprocessing
import os
import shutil
import tarfile

from avocado.core import exceptions
from avocado.utils import download
from avocado.utils import git
from avocado.utils import path
from avocado.utils import process

from . import data_dir


def _force_copy(src, dest):
    """
    Replace dest with a new copy of src, even if it exists
    """
    if os.path.isfile(dest):
        os.remove(dest)
    if os.path.isdir(dest):
        dest = os.path.join(dest, os.path.basename(src))
    shutil.copyfile(src, dest)
    return dest


class GitRepoParamHelper(git.GitRepoHelper):

    """
    Helps to deal with git repos specified in cartersian config files

    This class attempts to make it simple to manage a git repo, by using a
    naming standard that follows this basic syntax:

    <prefix>_name_<suffix>

    <prefix> is always 'git_repo' and <suffix> sets options for this git repo.
    Example for repo named foo:

    git_repo_foo_uri = git://git.foo.org/foo.git
    git_repo_foo_base_uri = /home/user/code/foo
    git_repo_foo_branch = master
    git_repo_foo_lbranch = master
    git_repo_foo_commit = bb5fb8e678aabe286e74c4f2993dc2a9e550b627
    """

    def __init__(self, params, name, destination_dir):
        """
        Instantiates a new GitRepoParamHelper
        """
        self.params = params
        self.name = name
        self.destination_dir = destination_dir
        self._parse_params()

    def _parse_params(self):
        """
        Parses the params items for entries related to this repo

        This method currently does everything that the parent class __init__()
        method does, that is, sets all instance variables needed by other
        methods. That means it's not strictly necessary to call parent's
        __init__().
        """
        config_prefix = 'git_repo_%s' % self.name
        logging.debug('Parsing parameters for git repo %s, configuration '
                      'prefix is %s' % (self.name, config_prefix))

        self.base_uri = self.params.get('%s_base_uri' % config_prefix)
        if self.base_uri is None:
            logging.debug('Git repo %s base uri is not set' % self.name)
        else:
            logging.debug('Git repo %s base uri: %s' % (self.name,
                                                        self.base_uri))

        self.uri = self.params.get('%s_uri' % config_prefix)
        logging.debug('Git repo %s uri: %s' % (self.name, self.uri))

        self.branch = self.params.get('%s_branch' % config_prefix, 'master')
        logging.debug('Git repo %s branch: %s' % (self.name, self.branch))

        self.lbranch = self.params.get('%s_lbranch' % config_prefix)
        if self.lbranch is None:
            self.lbranch = self.branch
        logging.debug('Git repo %s lbranch: %s' % (self.name, self.lbranch))

        self.commit = self.params.get('%s_commit' % config_prefix)
        if self.commit is None:
            logging.debug('Git repo %s commit is not set' % self.name)
        else:
            logging.debug('Git repo %s commit: %s' % (self.name, self.commit))

        self.tag = self.params.get('%s_tag' % config_prefix)
        if self.tag is None:
            logging.debug('Git repo %s tag is not set' % self.name)
        else:
            logging.debug('Git repo %s tag: %s' % (self.name, self.tag))

        self.key_file = None
        tag_signed = self.params.get('%s_tag_signed' % config_prefix)
        if tag_signed is None:
            logging.warning('Git repo %s tag is not signed' % self.name)
            logging.warning('This means we will not verify if the key was '
                            'made by whomever claims to have made it '
                            '(dangerous)')
        else:
            self.key_file = os.path.join(data_dir.get_data_dir(), 'gpg',
                                         tag_signed)
            if os.path.isfile(self.key_file):
                logging.debug('Git repo %s tag %s will be verified with public '
                              'key file %s', self.name, self.tag, self.key_file)
            else:
                raise exceptions.TestError('GPG public key file %s not found, '
                                           'will not proceed with testing' %
                                           self.key_file)

        self.cmd = path.find_command('git')

        self.recursive = self.params.get('%s_recursive', 'yes')

    def execute(self):
        super(GitRepoParamHelper, self).execute()

        cwd = os.path.curdir
        os.chdir(self.destination_dir)
        process.system('git remote add origin %s' %
                       self.uri, ignore_status=True)
        if self.recursive == 'yes':
            process.system('git submodule init')
            process.system('git submodule update')

        if self.tag:
            process.system('git checkout %s' % self.tag)
            if self.key_file is not None:
                try:
                    gnupg_home = os.path.join(data_dir.get_tmp_dir(),
                                              'gnupg')
                    if not os.path.isdir(gnupg_home):
                        os.makedirs(gnupg_home)
                    os.environ['GNUPGHOME'] = gnupg_home
                    process.system('gpg --import %s' % self.key_file)
                    logging.debug('Verifying if tag is actually signed with '
                                  'GPG key ID %s' % self.key_file)
                    process.system('git tag -v %s' % self.tag)
                except process.CmdError:
                    raise exceptions.TestError("GPG signature check for git repo "
                                               "%s failed" % self.name)

        # Log the top commit message, good for quick reference
        process.system('git log -1')

        os.chdir(cwd)


class LocalSourceDirHelper(object):

    """
    Helper class to deal with source code sitting somewhere in the filesystem
    """

    def __init__(self, source_dir, destination_dir):
        """
        :param source_dir:
        :param destination_dir:
        :return: new LocalSourceDirHelper instance
        """
        self.source = source_dir
        self.destination = destination_dir

    def execute(self):
        """
        Copies the source directory to the destination directory
        """
        if os.path.isdir(self.destination):
            shutil.rmtree(self.destination)

        if os.path.isdir(self.source):
            shutil.copytree(self.source, self.destination)


class LocalSourceDirParamHelper(LocalSourceDirHelper):

    """
    Helps to deal with source dirs specified in cartersian config files

    This class attempts to make it simple to manage a source dir, by using a
    naming standard that follows this basic syntax:

    <prefix>_name_<suffix>

    <prefix> is always 'local_src' and <suffix> sets options for this source
    dir.  Example for source dir named foo:

    local_src_foo_path = /home/user/foo
    """

    def __init__(self, params, name, destination_dir):
        """
        Instantiate a new LocalSourceDirParamHelper
        """
        self.params = params
        self.name = name
        self.destination_dir = destination_dir
        self._parse_params()

    def _parse_params(self):
        """
        Parses the params items for entries related to source dir
        """
        config_prefix = 'local_src_%s' % self.name
        logging.debug('Parsing parameters for local source %s, configuration '
                      'prefix is %s' % (self.name, config_prefix))

        self.path = self.params.get('%s_path' % config_prefix)
        logging.debug('Local source directory %s path: %s' % (self.name,
                                                              self.path))
        self.source = self.path
        self.destination = self.destination_dir


class LocalTarHelper(object):

    """
    Helper class to deal with source code in a local tarball
    """

    def __init__(self, source, destination_dir):
        self.source = source
        self.destination = destination_dir

    def extract(self):
        """
        Extracts the tarball into the destination directory
        """
        if os.path.isdir(self.destination):
            shutil.rmtree(self.destination)

        if os.path.isfile(self.source) and tarfile.is_tarfile(self.source):

            name = os.path.basename(self.destination)
            temp_dir = os.path.join(os.path.dirname(self.destination),
                                    '%s.tmp' % name)
            logging.debug('Temporary directory for extracting tarball is %s' %
                          temp_dir)

            if not os.path.isdir(temp_dir):
                os.makedirs(temp_dir)

            tarball = tarfile.open(self.source)
            tarball.extractall(temp_dir)

            #
            # If there's a directory at the toplevel of the tarfile, assume
            # it's the root for the contents, usually source code
            #
            tarball_info = tarball.members[0]
            if tarball_info.isdir():
                content_path = os.path.join(temp_dir,
                                            tarball_info.name)
            else:
                content_path = temp_dir

            #
            # Now move the content directory to the final destination
            #
            shutil.move(content_path, self.destination)

        else:
            raise OSError("%s is not a file or tar file" % self.source)

    def execute(self):
        """
        Executes all action this helper is supposed to perform

        This is the main entry point method for this class, and all other
        helper classes.
        """
        self.extract()


class LocalTarParamHelper(LocalTarHelper):

    """
    Helps to deal with source tarballs specified in cartersian config files

    This class attempts to make it simple to manage a tarball with source code,
    by using a  naming standard that follows this basic syntax:

    <prefix>_name_<suffix>

    <prefix> is always 'local_tar' and <suffix> sets options for this source
    tarball.  Example for source tarball named foo:

    local_tar_foo_path = /tmp/foo-1.0.tar.gz
    """

    def __init__(self, params, name, destination_dir):
        """
        Instantiates a new LocalTarParamHelper
        """
        self.params = params
        self.name = name
        self.destination_dir = destination_dir
        self._parse_params()

    def _parse_params(self):
        """
        Parses the params items for entries related to this local tar helper
        """
        config_prefix = 'local_tar_%s' % self.name
        logging.debug('Parsing parameters for local tar %s, configuration '
                      'prefix is %s' % (self.name, config_prefix))

        self.path = self.params.get('%s_path' % config_prefix)
        logging.debug('Local source tar %s path: %s' % (self.name,
                                                        self.path))
        self.source = self.path
        self.destination = self.destination_dir


class RemoteTarHelper(LocalTarHelper):

    """
    Helper that fetches a tarball and extracts it locally
    """

    def __init__(self, source_uri, destination_dir):
        self.source = source_uri
        self.destination = destination_dir

    def execute(self):
        """
        Executes all action this helper class is supposed to perform

        This is the main entry point method for this class, and all other
        helper classes.

        This implementation fetches the remote tar file and then extracts
        it using the functionality present in the parent class.
        """
        name = os.path.basename(self.source)
        base_dest = os.path.dirname(self.destination_dir)
        dest = os.path.join(base_dest, name)
        download.get_file(self.source, dest)
        self.source = dest
        self.extract()


class RemoteTarParamHelper(RemoteTarHelper):

    """
    Helps to deal with remote source tarballs specified in cartersian config

    This class attempts to make it simple to manage a tarball with source code,
    by using a  naming standard that follows this basic syntax:

    <prefix>_name_<suffix>

    <prefix> is always 'local_tar' and <suffix> sets options for this source
    tarball.  Example for source tarball named foo:

    remote_tar_foo_uri = http://foo.org/foo-1.0.tar.gz
    """

    def __init__(self, params, name, destination_dir):
        """
        Instantiates a new RemoteTarParamHelper instance
        """
        self.params = params
        self.name = name
        self.destination_dir = destination_dir
        self._parse_params()

    def _parse_params(self):
        """
        Parses the params items for entries related to this remote tar helper
        """
        config_prefix = 'remote_tar_%s' % self.name
        logging.debug('Parsing parameters for remote tar %s, configuration '
                      'prefix is %s' % (self.name, config_prefix))

        self.uri = self.params.get('%s_uri' % config_prefix)
        logging.debug('Remote source tar %s uri: %s' % (self.name,
                                                        self.uri))
        self.source = self.uri
        self.destination = self.destination_dir


class PatchHelper(object):

    """
    Helper that encapsulates the patching of source code with patch files
    """

    def __init__(self, source_dir, patches):
        """
        Initializes a new PatchHelper
        """
        self.source_dir = source_dir
        self.patches = patches

    def download(self):
        """
        Copies patch files from remote locations to the source directory
        """
        for patch in self.patches:
            download.get_file(patch, os.path.join(self.source_dir,
                                                  os.path.basename(patch)))

    def patch(self):
        """
        Patches the source dir with all patch files
        """
        os.chdir(self.source_dir)
        for patch in self.patches:
            process.system('patch -p1 < %s' % os.path.basename(patch),
                           shell=True)

    def execute(self):
        """
        Performs all steps necessary to download patches and apply them
        """
        self.download()
        self.patch()


class PatchParamHelper(PatchHelper):

    """
    Helps to deal with patches specified in cartersian config files

    This class attempts to make it simple to patch source coude, by using a
    naming standard that follows this basic syntax:

    [<git_repo>|<local_src>|<local_tar>|<remote_tar>]_<name>_patches

    <prefix> is either a 'local_src' or 'git_repo', that, together with <name>
    specify a directory containing source code to receive the patches. That is,
    for source code coming from git repo foo, patches would be specified as:

    git_repo_foo_patches = ['http://foo/bar.patch', 'http://foo/baz.patch']

    And for for patches to be applied on local source code named also foo:

    local_src_foo_patches = ['http://foo/bar.patch', 'http://foo/baz.patch']
    """

    def __init__(self, params, prefix, source_dir):
        """
        Initializes a new PatchParamHelper instance
        """
        self.params = params
        self.prefix = prefix
        self.source_dir = source_dir
        self._parse_params()

    def _parse_params(self):
        """
        Parses the params items for entries related to this set of patches

        This method currently does everything that the parent class __init__()
        method does, that is, sets all instance variables needed by other
        methods. That means it's not strictly necessary to call parent's
        __init__().
        """
        logging.debug('Parsing patch parameters for prefix %s' % self.prefix)
        patches_param_key = '%s_patches' % self.prefix

        self.patches_str = self.params.get(patches_param_key, '[]')
        logging.debug('Patches config for prefix %s: %s' % (self.prefix,
                                                            self.patches_str))

        self.patches = eval(self.patches_str)
        logging.debug('Patches for prefix %s: %s' % (self.prefix,
                                                     ", ".join(self.patches)))


class GnuSourceBuildInvalidSource(Exception):

    """
    Exception raised when build source dir/file is not valid
    """
    pass


class SourceBuildFailed(Exception):

    """
    Exception raised when building with parallel jobs fails

    This serves as feedback for code using
    :class:`virttest.build_helper.BuildHelper`.
    """
    pass


class SourceBuildParallelFailed(Exception):

    """
    Exception raised when building with parallel jobs fails

    This serves as feedback for code using
    :class:`virttest.build_helper.BuildHelper`.
    """
    pass


class GnuSourceBuildHelper(object):

    """
    Handles software installation of GNU-like source code

    This basically means that the build will go though the classic GNU
    autotools steps: ./configure, make, make install
    """

    def __init__(self, source, build_dir, prefix,
                 configure_options=[]):
        """
        :type source: string
        :param source: source directory or tarball
        :type prefix: string
        :param prefix: installation prefix
        :type build_dir: string
        :param build_dir: temporary directory used for building the source code
        :type configure_options: builtin.list
        :param configure_options: options to pass to configure
        :throws: GnuSourceBuildInvalidSource
        """
        self.source = source
        self.build_dir = build_dir
        self.prefix = prefix
        self.configure_options = configure_options
        self.install_debug_info = True
        self.include_pkg_config_path()

    def include_pkg_config_path(self):
        """
        Adds the current prefix to the list of paths that pkg-config searches

        This is currently not optional as there is no observed adverse side
        effects of enabling this. As the "prefix" is usually only valid during
        a test run, we believe that having other pkg-config files (``*.pc``) in
        either ``<prefix>/share/pkgconfig`` or ``<prefix>/lib/pkgconfig`` is
        exactly for the purpose of using them.

        :return: None
        """
        env_var = 'PKG_CONFIG_PATH'

        include_paths = [os.path.join(self.prefix, 'share', 'pkgconfig'),
                         os.path.join(self.prefix, 'lib', 'pkgconfig')]

        if env_var in os.environ:
            paths = os.environ[env_var].split(':')
            for include_path in include_paths:
                if include_path not in paths:
                    paths.append(include_path)
            os.environ[env_var] = ':'.join(paths)
        else:
            os.environ[env_var] = ':'.join(include_paths)

        logging.debug('PKG_CONFIG_PATH is: %s' % os.environ['PKG_CONFIG_PATH'])

    def get_configure_path(self):
        """
        Checks if 'configure' exists, if not, return 'autogen.sh' as a fallback
        """
        configure_path = os.path.abspath(os.path.join(self.source,
                                                      "configure"))
        autogen_path = os.path.abspath(os.path.join(self.source,
                                                    "autogen.sh"))
        if os.path.exists(configure_path):
            return configure_path
        elif os.path.exists(autogen_path):
            return autogen_path
        else:
            raise GnuSourceBuildInvalidSource(
                'configure script does not exist')

    def get_available_configure_options(self):
        """
        Return the list of available options of a GNU like configure script

        This will run the "configure" script at the source directory

        :return: list of options accepted by configure script
        """
        help_raw = process.system_output('%s --help' % self.get_configure_path(),
                                         ignore_status=True)
        help_output = help_raw.split("\n")
        option_list = []
        for line in help_output:
            cleaned_line = line.lstrip()
            if cleaned_line.startswith("--"):
                option = cleaned_line.split()[0]
                option = option.split("=")[0]
                option_list.append(option)

        return option_list

    def enable_debug_symbols(self):
        """
        Enables option that leaves debug symbols on compiled software

        This makes debugging a lot easier.
        """
        enable_debug_option = "--disable-strip"
        if enable_debug_option in self.get_available_configure_options():
            self.configure_options.append(enable_debug_option)
            logging.debug('Enabling debug symbols with option: %s' %
                          enable_debug_option)

    def get_configure_command(self):
        """
        Formats configure script with all options set

        :return: string with all configure options, including prefix
        """
        prefix_option = "--prefix=%s" % self.prefix
        options = self.configure_options
        options.append(prefix_option)
        return "%s %s" % (self.get_configure_path(),
                          " ".join(options))

    def configure(self):
        """
        Runs the "configure" script passing appropriate command line options
        """
        configure_command = self.get_configure_command()
        logging.info('Running configure on build dir')
        os.chdir(self.build_dir)
        process.system(configure_command)

    def make_parallel(self):
        """
        Runs "make" using the correct number of parallel jobs
        """
        parallel_make_jobs = multiprocessing.cpu_count()
        make_command = "make -j %s" % parallel_make_jobs
        logging.info("Running parallel make on build dir")
        os.chdir(self.build_dir)
        process.system(make_command)

    def make_non_parallel(self):
        """
        Runs "make", using a single job
        """
        os.chdir(self.build_dir)
        process.system("make")

    def make_clean(self):
        """
        Runs "make clean"
        """
        os.chdir(self.build_dir)
        process.system("make clean")

    def make(self, failure_feedback=True):
        """
        Runs a parallel make, falling back to a single job in failure

        :param failure_feedback: return information on build failure by raising
                                 the appropriate exceptions
        :raise: SourceBuildParallelFailed if parallel build fails, or
                SourceBuildFailed if single job build fails
        """
        try:
            self.make_parallel()
        except process.CmdError:
            try:
                self.make_clean()
                self.make_non_parallel()
            except process.CmdError:
                if failure_feedback:
                    raise SourceBuildFailed
            if failure_feedback:
                raise SourceBuildParallelFailed

    def make_install(self):
        """
        Runs "make install"
        """
        os.chdir(self.build_dir)
        process.system("make install")

    install = make_install

    def execute(self):
        """
        Runs appropriate steps for *building* this source code tree
        """
        if self.install_debug_info:
            self.enable_debug_symbols()
        self.configure()
        self.make()


class LinuxKernelBuildHelper(object):

    """
    Handles Building Linux Kernel.
    """

    def __init__(self, params, prefix, source):
        """
        :type params: dict
        :param params: dictionary containing the test parameters
        :type source: string
        :param source: source directory or tarball
        :type prefix: string
        :param prefix: installation prefix
        """
        self.params = params
        self.prefix = prefix
        self.source = source
        self._parse_params()

    def _parse_params(self):
        """
        Parses the params items for entries related to guest kernel
        """
        configure_opt_key = '%s_config' % self.prefix
        self.config = self.params.get(configure_opt_key, '')

        build_image_key = '%s_build_image' % self.prefix
        self.build_image = self.params.get(build_image_key,
                                           'arch/x86/boot/bzImage')

        build_target_key = '%s_build_target' % self.prefix
        self.build_target = self.params.get(build_target_key, 'bzImage')

        kernel_path_key = '%s_kernel_path' % self.prefix
        image_dir = os.path.join(data_dir.get_data_dir(), 'images')
        default_kernel_path = os.path.join(image_dir,
                                           self.build_target)
        self.kernel_path = self.params.get(kernel_path_key,
                                           default_kernel_path)

        logging.info('Parsing Linux kernel build parameters for %s',
                     self.prefix)

    def make_guest_kernel(self):
        """
        Runs "make", using a single job
        """
        os.chdir(self.source)
        logging.info("Building guest kernel")
        logging.debug("Kernel config is %s" % self.config)
        download.get_file(self.config, '.config')

        # FIXME currently no support for builddir
        # run old config
        process.system('yes "" | make oldconfig > /dev/null', shell=True)
        parallel_make_jobs = multiprocessing.cpu_count()
        make_command = "make -j %s %s" % (
            parallel_make_jobs, self.build_target)
        logging.info("Running parallel make on src dir")
        process.system(make_command)

    def make_clean(self):
        """
        Runs "make clean"
        """
        os.chdir(self.source)
        process.system("make clean")

    def make(self, failure_feedback=True):
        """
        Runs a parallel make

        :param failure_feedback: return information on build failure by raising
                                 the appropriate exceptions
        :raise: SourceBuildParallelFailed if parallel build fails, or
        """
        try:
            self.make_clean()
            self.make_guest_kernel()
        except process.CmdError:
            if failure_feedback:
                raise SourceBuildParallelFailed

    def cp_linux_kernel(self):
        """
        Copying Linux kernel to target path
        """
        os.chdir(self.source)
        _force_copy(self.build_image, self.kernel_path)

    install = cp_linux_kernel

    def execute(self):
        """
        Runs appropriate steps for *building* this source code tree
        """
        self.make()


class GnuSourceBuildParamHelper(GnuSourceBuildHelper):

    """
    Helps to deal with gnu_autotools build helper in cartersian config files

    This class attempts to make it simple to build source coude, by using a
    naming standard that follows this basic syntax:

    [<git_repo>|<local_src>]_<name>_<option> = value

    To pass extra options to the configure script, while building foo from a
    git repo, set the following variable:

    git_repo_foo_configure_options = --enable-feature
    """

    def __init__(self, params, name, destination_dir, install_prefix):
        """
        Instantiates a new GnuSourceBuildParamHelper
        """
        self.params = params
        self.name = name
        self.destination_dir = destination_dir
        self.install_prefix = install_prefix
        self._parse_params()

    def _parse_params(self):
        """
        Parses the params items for entries related to source directory

        This method currently does everything that the parent class __init__()
        method does, that is, sets all instance variables needed by other
        methods. That means it's not strictly necessary to call parent's
        __init__().
        """
        logging.debug('Parsing gnu_autotools build parameters for %s' %
                      self.name)

        configure_opt_key = '%s_configure_options' % self.name
        configure_options = self.params.get(configure_opt_key, '').split()
        logging.debug('Configure options for %s: %s' % (self.name,
                                                        configure_options))

        self.source = self.destination_dir
        self.build_dir = self.destination_dir
        self.prefix = self.install_prefix
        self.configure_options = configure_options
        self.include_pkg_config_path()

        # Support the install_debug_info feature, that automatically
        # adds/keeps debug information on generated libraries/binaries
        install_debug_info_cfg = self.params.get("install_debug_info", "yes")
        self.install_debug_info = install_debug_info_cfg != "no"