charm_helpers_sync.py

#!/usr/bin/python

# Copyright 2014-2015 Canonical Limited.
#
# This file is part of charm-helpers.
#
# charm-helpers is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3 as
# published by the Free Software Foundation.
#
# charm-helpers is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

# Authors:
#   Adam Gandelman <adamg@ubuntu.com>

import logging
import optparse
import os
import subprocess
import shutil
import sys
import tempfile
import yaml
from fnmatch import fnmatch

import six

CHARM_HELPERS_BRANCH = 'lp:charm-helpers'


def parse_config(conf_file):
    if not os.path.isfile(conf_file):
        logging.error('Invalid config file: %s.' % conf_file)
        return False
    return yaml.load(open(conf_file).read())


def clone_helpers(work_dir, branch):
    dest = os.path.join(work_dir, 'charm-helpers')
    logging.info('Checking out %s to %s.' % (branch, dest))
    cmd = ['bzr', 'checkout', '--lightweight', branch, dest]
    subprocess.check_call(cmd)
    return dest


def _module_path(module):
    return os.path.join(*module.split('.'))


def _src_path(src, module):
    return os.path.join(src, 'charmhelpers', _module_path(module))


def _dest_path(dest, module):
    return os.path.join(dest, _module_path(module))


def _is_pyfile(path):
    return os.path.isfile(path + '.py')


def ensure_init(path):
    '''
    ensure directories leading up to path are importable, omitting
    parent directory, eg path='/hooks/helpers/foo'/:
        hooks/
        hooks/helpers/__init__.py
        hooks/helpers/foo/__init__.py
    '''
    for d, dirs, files in os.walk(os.path.join(*path.split('/')[:2])):
        _i = os.path.join(d, '__init__.py')
        if not os.path.exists(_i):
            logging.info('Adding missing __init__.py: %s' % _i)
            open(_i, 'wb').close()


def sync_pyfile(src, dest):
    src = src + '.py'
    src_dir = os.path.dirname(src)
    logging.info('Syncing pyfile: %s -> %s.' % (src, dest))
    if not os.path.exists(dest):
        os.makedirs(dest)
    shutil.copy(src, dest)
    if os.path.isfile(os.path.join(src_dir, '__init__.py')):
        shutil.copy(os.path.join(src_dir, '__init__.py'),
                    dest)
    ensure_init(dest)


def get_filter(opts=None):
    opts = opts or []
    if 'inc=*' in opts:
        # do not filter any files, include everything
        return None

    def _filter(dir, ls):
        incs = [opt.split('=').pop() for opt in opts if 'inc=' in opt]
        _filter = []
        for f in ls:
            _f = os.path.join(dir, f)

            if not os.path.isdir(_f) and not _f.endswith('.py') and incs:
                if True not in [fnmatch(_f, inc) for inc in incs]:
                    logging.debug('Not syncing %s, does not match include '
                                  'filters (%s)' % (_f, incs))
                    _filter.append(f)
                else:
                    logging.debug('Including file, which matches include '
                                  'filters (%s): %s' % (incs, _f))
            elif (os.path.isfile(_f) and not _f.endswith('.py')):
                logging.debug('Not syncing file: %s' % f)
                _filter.append(f)
            elif (os.path.isdir(_f) and not
                  os.path.isfile(os.path.join(_f, '__init__.py'))):
                logging.debug('Not syncing directory: %s' % f)
                _filter.append(f)
        return _filter
    return _filter


def sync_directory(src, dest, opts=None):
    if os.path.exists(dest):
        logging.debug('Removing existing directory: %s' % dest)
        shutil.rmtree(dest)
    logging.info('Syncing directory: %s -> %s.' % (src, dest))

    shutil.copytree(src, dest, ignore=get_filter(opts))
    ensure_init(dest)


def sync(src, dest, module, opts=None):

    # Sync charmhelpers/__init__.py for bootstrap code.
    sync_pyfile(_src_path(src, '__init__'), dest)

    # Sync other __init__.py files in the path leading to module.
    m = []
    steps = module.split('.')[:-1]
    while steps:
        m.append(steps.pop(0))
        init = '.'.join(m + ['__init__'])
        sync_pyfile(_src_path(src, init),
                    os.path.dirname(_dest_path(dest, init)))

    # Sync the module, or maybe a .py file.
    if os.path.isdir(_src_path(src, module)):
        sync_directory(_src_path(src, module), _dest_path(dest, module), opts)
    elif _is_pyfile(_src_path(src, module)):
        sync_pyfile(_src_path(src, module),
                    os.path.dirname(_dest_path(dest, module)))
    else:
        logging.warn('Could not sync: %s. Neither a pyfile or directory, '
                     'does it even exist?' % module)


def parse_sync_options(options):
    if not options:
        return []
    return options.split(',')


def extract_options(inc, global_options=None):
    global_options = global_options or []
    if global_options and isinstance(global_options, six.string_types):
        global_options = [global_options]
    if '|' not in inc:
        return (inc, global_options)
    inc, opts = inc.split('|')
    return (inc, parse_sync_options(opts) + global_options)


def sync_helpers(include, src, dest, options=None):
    if not os.path.isdir(dest):
        os.makedirs(dest)

    global_options = parse_sync_options(options)

    for inc in include:
        if isinstance(inc, str):
            inc, opts = extract_options(inc, global_options)
            sync(src, dest, inc, opts)
        elif isinstance(inc, dict):
            # could also do nested dicts here.
            for k, v in six.iteritems(inc):
                if isinstance(v, list):
                    for m in v:
                        inc, opts = extract_options(m, global_options)
                        sync(src, dest, '%s.%s' % (k, inc), opts)

if __name__ == '__main__':
    parser = optparse.OptionParser()
    parser.add_option('-c', '--config', action='store', dest='config',
                      default=None, help='helper config file')
    parser.add_option('-D', '--debug', action='store_true', dest='debug',
                      default=False, help='debug')
    parser.add_option('-b', '--branch', action='store', dest='branch',
                      help='charm-helpers bzr branch (overrides config)')
    parser.add_option('-d', '--destination', action='store', dest='dest_dir',
                      help='sync destination dir (overrides config)')
    (opts, args) = parser.parse_args()

    if opts.debug:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    if opts.config:
        logging.info('Loading charm helper config from %s.' % opts.config)
        config = parse_config(opts.config)
        if not config:
            logging.error('Could not parse config from %s.' % opts.config)
            sys.exit(1)
    else:
        config = {}

    if 'branch' not in config:
        config['branch'] = CHARM_HELPERS_BRANCH
    if opts.branch:
        config['branch'] = opts.branch
    if opts.dest_dir:
        config['destination'] = opts.dest_dir

    if 'destination' not in config:
        logging.error('No destination dir. specified as option or config.')
        sys.exit(1)

    if 'include' not in config:
        if not args:
            logging.error('No modules to sync specified as option or config.')
            sys.exit(1)
        config['include'] = []
        [config['include'].append(a) for a in args]

    sync_options = None
    if 'options' in config:
        sync_options = config['options']
    tmpd = tempfile.mkdtemp()
    try:
        checkout = clone_helpers(tmpd, config['branch'])
        sync_helpers(config['include'], checkout, config['destination'],
                     options=sync_options)
    except Exception as e:
        logging.error("Could not sync: %s" % e)
        raise e
    finally:
        logging.debug('Cleaning up %s' % tmpd)
        shutil.rmtree(tmpd)