transparent downstream vendoring (ansible#69850)

* builtin downstream vendoring support

* allows downstream packagers to install packages to `ansible/_vendor` that will automatically be added to head of sys.path during `ansible` package load
* tests

* sort conflicting package names in warning text

* sanity fixes

* skip unnecessary comparison

changelogs/fragments/downstream_vendoring.yml

@@ -0,0 +1,2 @@
+minor_changes:
+- downstream packagers may install packages under ansible._vendor, which will be added to head of sys.path at ansible package load

lib/ansible/__init__.py

@@ -19,6 +19,9 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+# make vendored top-level modules accessible EARLY
+import ansible._vendor
+
 # Note: Do not add any code to this file.  The ansible module may be
 # a namespace package when using Ansible-2.1+ Anything in this file may not be
 # available if one of the other packages in the namespace is loaded first.

lib/ansible/_vendor/__init__.py

@@ -0,0 +1,46 @@
+# (c) 2020 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import pkgutil
+import sys
+import warnings
+
+# This package exists to host vendored top-level Python packages for downstream packaging. Any Python packages
+# installed beneath this one will be masked from the Ansible loader, and available from the front of sys.path.
+# It is expected that the vendored packages will be loaded very early, so a warning will be fired on import of
+# the top-level ansible package if any packages beneath this are already loaded at that point.
+#
+# Python packages may be installed here during downstream packaging using something like:
+# pip install --upgrade -t (path to this dir) cryptography pyyaml packaging jinja2
+
+# mask vendored content below this package from being accessed as an ansible subpackage
+__path__ = []
+
+
+def _ensure_vendored_path_entry():
+    """
+    Ensure that any downstream-bundled content beneath this package is available at the top of sys.path
+    """
+    # patch our vendored dir onto sys.path
+    vendored_path_entry = os.path.dirname(__file__)
+    vendored_module_names = set(m[1] for m in pkgutil.iter_modules([vendored_path_entry], ''))  # m[1] == m.name
+
+    if vendored_module_names:
+        # patch us early to load vendored deps transparently
+        if vendored_path_entry in sys.path:
+            # handle reload case by removing the existing entry, wherever it might be
+            sys.path.remove(vendored_path_entry)
+        sys.path.insert(0, vendored_path_entry)
+
+        already_loaded_vendored_modules = set(sys.modules.keys()).intersection(vendored_module_names)
+
+        if already_loaded_vendored_modules:
+            warnings.warn('One or more Python packages bundled by this ansible-base distribution were already '
+                          'loaded ({0}). This may result in undefined behavior.'.format(', '.join(sorted(already_loaded_vendored_modules))))
+
+
+_ensure_vendored_path_entry()

test/units/_vendor/test_vendor.py

@@ -0,0 +1,65 @@
+# (c) 2020 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import pkgutil
+import pytest
+import sys
+
+from units.compat.mock import MagicMock, NonCallableMagicMock, patch
+
+
+def reset_internal_vendor_package():
+    import ansible
+    ansible_vendor_path = os.path.join(os.path.dirname(ansible.__file__), '_vendor')
+
+    if ansible_vendor_path in sys.path:
+        sys.path.remove(ansible_vendor_path)
+
+    for pkg in ['ansible._vendor', 'ansible']:
+        if pkg in sys.modules:
+            del sys.modules[pkg]
+
+
+def test_package_path_masking():
+    from ansible import _vendor
+
+    assert hasattr(_vendor, '__path__') and _vendor.__path__ == []
+
+
+def test_no_vendored():
+    reset_internal_vendor_package()
+    with patch.object(pkgutil, 'iter_modules', return_value=[]):
+        previous_path = list(sys.path)
+        import ansible
+        ansible_vendor_path = os.path.join(os.path.dirname(ansible.__file__), '_vendor')
+
+        assert ansible_vendor_path not in sys.path
+        assert sys.path == previous_path
+
+
+def test_vendored(vendored_pkg_names=None):
+    if not vendored_pkg_names:
+        vendored_pkg_names = ['boguspkg']
+    reset_internal_vendor_package()
+    with patch.object(pkgutil, 'iter_modules', return_value=list((None, p, None) for p in vendored_pkg_names)):
+        previous_path = list(sys.path)
+        import ansible
+        ansible_vendor_path = os.path.join(os.path.dirname(ansible.__file__), '_vendor')
+        assert sys.path[0] == ansible_vendor_path
+
+        if ansible_vendor_path in previous_path:
+            previous_path.remove(ansible_vendor_path)
+
+        assert sys.path[1:] == previous_path
+
+
+def test_vendored_conflict():
+    with pytest.warns(UserWarning) as w:
+        import pkgutil
+        import sys
+        test_vendored(vendored_pkg_names=['sys', 'pkgutil'])  # pass a real package we know is already loaded
+        assert 'pkgutil, sys' in str(w[0].message)  # ensure both conflicting modules are listed and sorted