fix sys.modules cleanup and blacklist behavior (ansible#69898)

* fix sys.modules cleanup and blacklist behavior

* fix map-as-generator py2/py3 issue
* clear path_importer_cache between runs

* sanity fix

* don't be stupid with moving target generators

lib/ansible/utils/collection_loader/_collection_finder.py

@@ -493,7 +493,10 @@ def load_module(self, fullname):
 
         if collection_name == 'ansible.builtin':
             # ansible.builtin is a synthetic collection, get its routing config from the Ansible distro
-            raw_routing = pkgutil.get_data('ansible.config', 'ansible_builtin_runtime.yml')
+            ansible_pkg_path = os.path.dirname(import_module('ansible').__file__)
+            metadata_path = os.path.join(ansible_pkg_path, 'config/ansible_builtin_runtime.yml')
+            with open(to_bytes(metadata_path), 'rb') as fd:
+                raw_routing = fd.read()
         else:
             b_routing_meta_path = to_bytes(os.path.join(module.__path__[0], 'meta/runtime.yml'))
             if os.path.isfile(b_routing_meta_path):

test/lib/ansible_test/_data/sanity/import/importer.py

@@ -22,7 +22,7 @@ def main():
 
     ansible_path = os.path.dirname(os.path.dirname(ansible.__file__))
     temp_path = os.environ['SANITY_TEMP_PATH'] + os.path.sep
-    external_python = os.environ.get('SANITY_EXTERNAL_PYTHON')
+    external_python = os.environ.get('SANITY_EXTERNAL_PYTHON') or sys.executable
     collection_full_name = os.environ.get('SANITY_COLLECTION_FULL_NAME')
     collection_root = os.environ.get('ANSIBLE_COLLECTIONS_PATHS')
 
@@ -43,7 +43,6 @@ def import_module(name):
     if collection_full_name:
         # allow importing code from collections when testing a collection
         from ansible.module_utils.common.text.converters import to_bytes, to_text, to_native
-        from ansible.utils.collection_loader import AnsibleCollectionConfig
         from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder
         from ansible.utils.collection_loader import _collection_finder
 
@@ -76,8 +75,9 @@ def yaml_to_dict(yaml, content_id):
 
         collection_loader = _AnsibleCollectionFinder(paths=[collection_root])
         collection_loader._install()  # pylint: disable=protected-access
-        nuke_modules = list(m for m in sys.modules if m.partition('.')[0] == 'ansible')
-        map(sys.modules.pop, nuke_modules)
+
+        # remove all modules under the ansible package
+        list(map(sys.modules.pop, [m for m in sys.modules if m.partition('.')[0] == 'ansible']))
 
     else:
         # do not support collection loading when not testing a collection
@@ -374,6 +374,7 @@ def blacklist_imports(path, name, messages):
         blacklist = ImportBlacklist(path, name)
 
         sys.meta_path.insert(0, blacklist)
+        sys.path_importer_cache.clear()
 
         try:
             yield
@@ -384,6 +385,8 @@ def blacklist_imports(path, name, messages):
             while blacklist in sys.meta_path:
                 sys.meta_path.remove(blacklist)
 
+            sys.path_importer_cache.clear()
+
     @contextlib.contextmanager
     def monitor_sys_modules(path, messages):
         """Monitor sys.modules for unwanted changes, reverting any additions made to our own namespaces."""