From b94bfed1a6763069c606d50c1f57fe2e15d41a6a Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Wed, 6 Mar 2019 12:29:51 -0500 Subject: [PATCH] scan_packages: made adding package managers easier (#49079) * made adding package managers easier added portage support * moar pkg mgrs and moar info - added 'pkg' pkg manager (freebsd) - added pip - more apt info * updated clgo * Updates from feedback Co-Authored-By: bcoca * incorporated more feedback and added docstrings * moar from feedback - made manager list dynamic and names based on class - better not found msg - made abstract metaclass again - test is now init exception - module to global - better dedupe comments * more targetted errors/warnings * added strategy, reordered to conserve priority * rpm > apt * move break to top * fix trate * piping it * lines and meta * refactored common functions - moved pip into it's own module - cleaned up base clases - ensure 'lower' match in package_facts * missing license * avoid facts * update clog * addressed feedback * fix clog * cleanup * upd * removed pip as that was removed * renamed cpan * added a single line since 2 lines are needed to be readabnle instead of just 1 line, it is a huge problem otherwise * fix internal ref * not intended in this round * updated as per fb --- changelogs/fragments/scan_packages.yml | 3 + lib/ansible/module_utils/facts/packages.py | 83 ++++++ .../packaging/language/pip_package_info.py | 152 +++++++++++ .../modules/packaging/os/package_facts.py | 247 ++++++++++++------ test/sanity/validate-modules/ignore.txt | 1 - 5 files changed, 402 insertions(+), 84 deletions(-) create mode 100644 changelogs/fragments/scan_packages.yml create mode 100644 lib/ansible/module_utils/facts/packages.py create mode 100644 lib/ansible/modules/packaging/language/pip_package_info.py diff --git a/changelogs/fragments/scan_packages.yml b/changelogs/fragments/scan_packages.yml new file mode 100644 index 00000000000000..3102b63a00bb1c --- /dev/null +++ b/changelogs/fragments/scan_packages.yml @@ -0,0 +1,3 @@ +minor_changes: + - package_facts, now supports multiple package managers per system. + New systems supported include Gentoo's portage with portage-utils installed, as well as FreeBSD's pkg diff --git a/lib/ansible/module_utils/facts/packages.py b/lib/ansible/module_utils/facts/packages.py new file mode 100644 index 00000000000000..39b65f7812bf43 --- /dev/null +++ b/lib/ansible/module_utils/facts/packages.py @@ -0,0 +1,83 @@ +# (c) 2018, Ansible Project +# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +from abc import ABCMeta, abstractmethod + +from ansible.module_utils.six import with_metaclass +from ansible.module_utils.basic import get_all_subclasses +from ansible.module_utils.common.process import get_bin_path + + +def get_all_pkg_managers(): + + return dict([(obj.__name__.lower(), obj) for obj in get_all_subclasses(PkgMgr) if obj not in (CLIMgr, LibMgr)]) + + +class PkgMgr(with_metaclass(ABCMeta, object)): + + @abstractmethod + def is_available(self): + # This method is supposed to return True/False if the package manager is currently installed/usable + # It can also 'prep' the required systems in the process of detecting availability + pass + + @abstractmethod + def list_installed(self): + # This method should return a list of installed packages, each list item will be passed to get_package_details + pass + + @abstractmethod + def get_package_details(self, package): + # This takes a 'package' item and returns a dictionary with the package information, name and version are minimal requirements + pass + + def get_packages(self): + # Take all of the above and return a dictionary of lists of dictionaries (package = list of installed versions) + + installed_packages = {} + for package in self.list_installed(): + package_details = self.get_package_details(package) + if 'source' not in package_details: + package_details['source'] = self.__class__.__name__.lower() + name = package_details['name'] + if name not in installed_packages: + installed_packages[name] = [package_details] + else: + installed_packages[name].append(package_details) + return installed_packages + + +class LibMgr(PkgMgr): + + LIB = None + + def __init__(self): + + self._lib = None + super(LibMgr, self).__init__() + + def is_available(self): + found = False + try: + self._lib = __import__(self.LIB) + found = True + except ImportError: + pass + return found + + +class CLIMgr(PkgMgr): + + CLI = None + + def __init__(self): + + self._cli = None + super(CLIMgr, self).__init__() + + def is_available(self): + self._cli = get_bin_path(self.CLI, False) + return bool(self._cli) diff --git a/lib/ansible/modules/packaging/language/pip_package_info.py b/lib/ansible/modules/packaging/language/pip_package_info.py new file mode 100644 index 00000000000000..908b4affb4bae7 --- /dev/null +++ b/lib/ansible/modules/packaging/language/pip_package_info.py @@ -0,0 +1,152 @@ +#!/usr/bin/python +# (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# started out with AWX's scan_packages module + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = ''' +module: pip_package_info +short_description: pip package information +description: + - Return information about installed pip packages +version_added: "2.8" +options: + clients: + description: + - A list of the pip executables that will be used to get the packages. + They can be supplied with the full path or just the executable name, i.e `pip3.7`. + default: ['pip'] + required: False + type: list +requirements: + - The requested pip executables must be installed on the target. +author: + - Matthew Jones (@matburt) + - Brian Coca (@bcoca) + - Adam Miller (@maxamillion) +''' + +EXAMPLES = ''' +- name: Just get the list from default pip + pip_package_info: + +- name: get the facts for default pip, pip2 and pip3.6 + pip_package_info: + clients: ['pip', 'pip2', 'pip3.6'] + +- name: get from specific paths (virtualenvs?) + pip_package_info: + clients: '/home/me/projec42/python/pip3.5' +''' + +RETURN = ''' +packages: + description: a dictionary of installed package data + returned: always + type: dict + contains: + python: + description: A dictionary with each pip client which then contains a list of dicts with python package information + returned: always + type: dict + sample: + "packages": { + "pip": { + "Babel": [ + { + "name": "Babel", + "source": "pip", + "version": "2.6.0" + } + ], + "Flask": [ + { + "name": "Flask", + "source": "pip", + "version": "1.0.2" + } + ], + "Flask-SQLAlchemy": [ + { + "name": "Flask-SQLAlchemy", + "source": "pip", + "version": "2.3.2" + } + ], + "Jinja2": [ + { + "name": "Jinja2", + "source": "pip", + "version": "2.10" + } + ], + }, + } +''' +import json +import os + +from ansible.module_utils._text import to_text +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.facts.packages import CLIMgr + + +class PIP(CLIMgr): + + def __init__(self, pip): + + self.CLI = pip + + def list_installed(self): + global module + rc, out, err = module.run_command([self._cli, 'list', '-l', '--format=json']) + if rc != 0: + raise Exception("Unable to list packages rc=%s : %s" % (rc, err)) + return json.loads(out) + + def get_package_details(self, package): + package['source'] = self.CLI + return package + + +def main(): + + # start work + global module + module = AnsibleModule(argument_spec=dict(clients={'type': 'list', 'default': ['pip']},), supports_check_mode=True) + packages = {} + results = {'packages': {}} + clients = module.params['clients'] + + found = 0 + for pip in clients: + + if not os.path.basename(pip).startswith('pip'): + module.warn('Skipping invalid pip client: %s' % (pip)) + continue + try: + pip_mgr = PIP(pip) + if pip_mgr.is_available(): + found += 1 + packages[pip] = pip_mgr.get_packages() + except Exception as e: + module.warn('Failed to retrieve packages with %s: %s' % (pip, to_text(e))) + continue + + if found == 0: + module.fail_json(msg='Unable to use any of the supplied pip clients: %s' % clients) + + # return info + results['packages'] = packages + module.exit_json(**results) + + +if __name__ == '__main__': + main() diff --git a/lib/ansible/modules/packaging/os/package_facts.py b/lib/ansible/modules/packaging/os/package_facts.py index 90590a3a5a35fc..90aa208732e361 100644 --- a/lib/ansible/modules/packaging/os/package_facts.py +++ b/lib/ansible/modules/packaging/os/package_facts.py @@ -19,11 +19,24 @@ options: manager: description: - - The package manager used by the system so we can query the package information - default: auto - choices: ["auto", "rpm", "apt"] + - The package manager used by the system so we can query the package information. + - Since 2.8 this is a list and can support multiple package managers per system. + - The 'portage' and 'pkg' options were added in version 2.8. + default: ['auto'] + choices: ['auto', 'rpm', 'apt', 'portage', 'pkg'] required: False + type: list + strategy: + description: + - This option controls how the module queres the package managers on the system. + C(first) means it will return only informatino for the first supported package manager available. + C(all) will return information for all supported and available package managers on the system. + choices: ['first', 'all'] + default: 'first' + version_added: "2.8" version_added: "2.5" +requirements: + - For 'portage' support it requires the `qlist` utility, which is part of 'app-portage/portage-utils'. author: - Matthew Jones (@matburt) - Brian Coca (@bcoca) @@ -140,94 +153,162 @@ } ''' -import sys - +from ansible.module_utils._text import to_native, to_text from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils._text import to_text - - -def rpm_package_list(): - - try: - import rpm - except ImportError: - module.fail_json(msg='Unable to use the rpm python bindings, please ensure they are installed under the python the module runs under') - - trans_set = rpm.TransactionSet() - installed_packages = {} - for package in trans_set.dbMatch(): - package_details = dict(name=package[rpm.RPMTAG_NAME], - version=package[rpm.RPMTAG_VERSION], - release=package[rpm.RPMTAG_RELEASE], - epoch=package[rpm.RPMTAG_EPOCH], - arch=package[rpm.RPMTAG_ARCH], - source='rpm') - if package_details['name'] not in installed_packages: - installed_packages[package_details['name']] = [package_details] - else: - installed_packages[package_details['name']].append(package_details) - return installed_packages - - -def apt_package_list(): - - try: - import apt - except ImportError: - module.fail_json(msg='Unable to use the apt python bindings, please ensure they are installed under the python the module runs under') - - apt_cache = apt.Cache() - installed_packages = {} - apt_installed_packages = [pk for pk in apt_cache.keys() if apt_cache[pk].is_installed] - for package in apt_installed_packages: - ac_pkg = apt_cache[package].installed - package_details = dict(name=package, version=ac_pkg.version, arch=ac_pkg.architecture, source='apt') - if package_details['name'] not in installed_packages: - installed_packages[package_details['name']] = [package_details] - else: - installed_packages[package_details['name']].append(package_details) - return installed_packages - - -# FIXME: add more listing methods +from ansible.module_utils.facts.packages import LibMgr, CLIMgr, get_all_pkg_managers + + +class RPM(LibMgr): + + LIB = 'rpm' + + def list_installed(self): + return self._lib.TransactionSet().dbMatch() + + def get_package_details(self, package): + return dict(name=package[self._lib.RPMTAG_NAME], + version=package[self._lib.RPMTAG_VERSION], + release=package[self._lib.RPMTAG_RELEASE], + epoch=package[self._lib.RPMTAG_EPOCH], + arch=package[self._lib.RPMTAG_ARCH],) + + +class APT(LibMgr): + + LIB = 'apt' + + @property + def pkg_cache(self): + if self._cache: + return self._cache + + self._cache = self._lib.Cache() + return self._cache + + def list_installed(self): + return [pk for pk in self.pkg_cache.keys() if self.pkg_cache[pk].is_installed] + + def get_package_details(self, package): + ac_pkg = self.pkg_cache[package].installed + return dict(name=package, version=ac_pkg.version, arch=ac_pkg.architecture, category=ac_pkg.section, origin=ac_pkg.origins[0].origin) + + +class PKG(CLIMgr): + + CLI = 'pkg' + atoms = ['name', 'version', 'origin', 'installed', 'automatic', 'arch', 'category', 'prefix', 'vital'] + + def list_installed(self): + rc, out, err = module.run_command([self._cli, 'query', "%%%s" % '\t%'.join(['n', 'v', 'R', 't', 'a', 'q', 'o', 'p', 'V'])]) + if rc != 0 or err: + raise Exception("Unable to list packages rc=%s : %s" % (rc, err)) + return out.splitlines() + + def get_package_details(self, package): + + pkg = dict(zip(self.atoms, package.split('\t'))) + + if 'arch' in pkg: + try: + pkg['arch'] = pkg['arch'].split(':')[2] + except IndexError: + pass + + if 'automatic' in pkg: + pkg['automatic'] = bool(pkg['automatic']) + + if 'category' in pkg: + pkg['category'] = pkg['category'].split('/', 1)[0] + + if 'version' in pkg: + if ',' in pkg['version']: + pkg['version'], pkg['port_epoch'] = pkg['version'].split(',', 1) + else: + pkg['port_epoch'] = 0 + + if '_' in pkg['version']: + pkg['version'], pkg['revision'] = pkg['version'].split('_', 1) + else: + pkg['revision'] = '0' + + if 'vital' in pkg: + pkg['vital'] = bool(pkg['vital']) + + return pkg + + +class PORTAGE(CLIMgr): + + CLI = 'qlist' + atoms = ['category', 'name', 'version', 'ebuild_revision', 'slots', 'prefixes', 'sufixes'] + + def list_installed(self): + rc, out, err = module.run_command(' '.join([self._cli, '-Iv', '|', 'xargs', '-n', '1024', 'qatom']), use_unsafe_shell=True) + if rc != 0: + raise RuntimeError("Unable to list packages rc=%s : %s" % (rc, to_native(err))) + return out.splitlines() + + def get_package_details(self, package): + return dict(zip(self.atoms, package.split())) + + def main(): + + # get supported pkg managers + PKG_MANAGERS = get_all_pkg_managers() + PKG_MANAGER_NAMES = [x.lower() for x in PKG_MANAGERS.keys()] + + # start work global module - module = AnsibleModule(argument_spec=dict(manager=dict()), supports_check_mode=True) - manager = module.params['manager'] + module = AnsibleModule(argument_spec=dict(manager={'type': 'list', 'default': ['auto']}, + strategy={'choices': ['first', 'all'], 'default': 'first'}), + supports_check_mode=True) packages = {} - results = {} + results = {'ansible_facts': {}} + managers = [x.lower() for x in module.params['manager']] + strategy = module.params['strategy'] + + if 'auto' in managers: + # keep order from user, we do dedupe below + managers.extend(PKG_MANAGER_NAMES) + managers.remove('auto') + + unsupported = set(managers).difference(PKG_MANAGER_NAMES) + if unsupported: + module.fail_json(msg='Unsupported package managers requested: %s' % (', '.join(unsupported))) - if manager is None or manager == 'auto': + found = 0 + seen = set() + for pkgmgr in managers: - # detect! - for manager_lib in ('rpm', 'apt'): + if found and strategy == 'first': + break + + # dedupe as per above + if pkgmgr in seen: + continue + seen.add(pkgmgr) + try: try: - dummy = __import__(manager_lib) - manager = manager_lib - break - except ImportError: - pass + # manager throws exception on init (calls self.test) if not usable. + manager = PKG_MANAGERS[pkgmgr]() + if manager.is_available(): + found += 1 + packages.update(manager.get_packages()) + except Exception as e: + if pkgmgr in module.params['manager']: + module.warn('Requested package manager %s was not usable by this module: %s' % (pkgmgr, to_text(e))) + continue - # FIXME: add more detection methods - try: - if manager == "rpm": - packages = rpm_package_list() - elif manager == "apt": - packages = apt_package_list() - else: - if manager: - results['msg'] = 'Unsupported package manager: %s' % manager - results['skipped'] = True - else: - module.fail_json(msg='Could not detect supported package manager') - except Exception as e: - from traceback import format_tb - module.fail_json(msg='Failed to retrieve packages: %s' % to_text(e), exception=format_tb(sys.exc_info()[2])) - - results['ansible_facts'] = {} - # Set the facts, this will override the facts in ansible_facts that might - # exist from previous runs when using operating system level or distribution - # package managers + except Exception as e: + if pkgmgr in module.params['manager']: + module.warn('Failed to retrieve packages with %s: %s' % (pkgmgr, to_text(e))) + + if found == 0: + module.fail_json(msg='Could not detect a supported package manager from the following list: %s' % managers) + + # Set the facts, this will override the facts in ansible_facts that might exist from previous runs + # when using operating system level or distribution package managers results['ansible_facts']['packages'] = packages module.exit_json(**results) diff --git a/test/sanity/validate-modules/ignore.txt b/test/sanity/validate-modules/ignore.txt index 05438a93855e64..cf784e588a360d 100644 --- a/test/sanity/validate-modules/ignore.txt +++ b/test/sanity/validate-modules/ignore.txt @@ -721,7 +721,6 @@ lib/ansible/modules/packaging/os/opkg.py E322 lib/ansible/modules/packaging/os/opkg.py E324 lib/ansible/modules/packaging/os/opkg.py E326 lib/ansible/modules/packaging/os/opkg.py E336 -lib/ansible/modules/packaging/os/package_facts.py E324 lib/ansible/modules/packaging/os/package_facts.py E326 lib/ansible/modules/packaging/os/pacman.py E326 lib/ansible/modules/packaging/os/pacman.py E336