Python 3.8 collections compatibility fixes.

Includes a new pylint blacklist plugin to prevent regressions.

contrib/inventory/ssh_config.py

@@ -43,12 +43,12 @@
 import argparse
 import os.path
 import sys
-from collections import MutableSequence
 
 import json
 
 import paramiko
 
+from ansible.module_utils.common._collections_compat import MutableSequence
 
 SSH_CONF = '~/.ssh/config'
 

contrib/inventory/vmware.py

@@ -30,7 +30,6 @@
 
 from __future__ import print_function
 
-import collections
 import json
 import logging
 import optparse
@@ -42,6 +41,8 @@
 from six import integer_types, text_type, string_types
 from six.moves import configparser
 
+from ansible.module_utils.common._collections_compat import MutableMapping
+
 # Disable logging message trigged by pSphere/suds.
 try:
     from logging import NullHandler
@@ -159,7 +160,7 @@ def _flatten_dict(self, d, parent_key='', sep='_'):
             if k.startswith('_'):
                 continue
             new_key = parent_key + sep + k if parent_key else k
-            if isinstance(v, collections.MutableMapping):
+            if isinstance(v, MutableMapping):
                 items.extend(self._flatten_dict(v, new_key, sep).items())
             elif isinstance(v, (list, tuple)):
                 if all([isinstance(x, string_types) for x in v]):
@@ -229,7 +230,7 @@ def _get_host_info(self, host, prefix='vmware'):
             except AttributeError:
                 host_info['%ss' % attr] = []
         for k, v in self._get_obj_info(host.summary, depth=0).items():
-            if isinstance(v, collections.MutableMapping):
+            if isinstance(v, MutableMapping):
                 for k2, v2 in v.items():
                     host_info[k2] = v2
             elif k != 'host':
@@ -265,7 +266,7 @@ def _get_vm_info(self, vm, prefix='vmware'):
         except AttributeError:
             vm_info['guestState'] = ''
         for k, v in self._get_obj_info(vm.summary, depth=0).items():
-            if isinstance(v, collections.MutableMapping):
+            if isinstance(v, MutableMapping):
                 for k2, v2 in v.items():
                     if k2 == 'host':
                         k2 = 'hostSystem'

lib/ansible/cli/doc.py

@@ -23,12 +23,11 @@
 import traceback
 import yaml
 
-from collections import Sequence
-
 from ansible import constants as C
 from ansible.cli import CLI
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.module_utils._text import to_native
+from ansible.module_utils.common._collections_compat import Sequence
 from ansible.module_utils.six import string_types
 from ansible.parsing.metadata import extract_metadata
 from ansible.parsing.plugin_docs import read_docstub

lib/ansible/compat/selectors/_selectors2.py

@@ -24,7 +24,8 @@
 import socket
 import sys
 import time
-from collections import namedtuple, Mapping
+from collections import namedtuple
+from ansible.module_utils.common._collections_compat import Mapping
 
 try:
     monotonic = time.monotonic

lib/ansible/errors/__init__.py

@@ -19,7 +19,6 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from collections import Sequence
 import traceback
 import sys
 
@@ -33,6 +32,7 @@
     YAML_POSITION_DETAILS,
 )
 from ansible.module_utils._text import to_native, to_text
+from ansible.module_utils.common._collections_compat import Sequence
 
 
 class AnsibleError(Exception):

lib/ansible/executor/stats.py

@@ -19,8 +19,7 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from collections import MutableMapping
-
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.utils.vars import merge_hash
 
 

lib/ansible/module_utils/common/_collections_compat.py

@@ -10,20 +10,37 @@
 from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
-
 try:
     """Python 3.3+ branch."""
     from collections.abc import (
+        MappingView,
+        ItemsView,
         KeysView,
+        ValuesView,
         Mapping, MutableMapping,
         Sequence, MutableSequence,
         Set, MutableSet,
+        Container,
+        Hashable,
+        Sized,
+        Callable,
+        Iterable,
+        Iterator,
     )
 except ImportError:
     """Use old lib location under 2.6-3.2."""
     from collections import (
+        MappingView,
+        ItemsView,
         KeysView,
+        ValuesView,
         Mapping, MutableMapping,
         Sequence, MutableSequence,
         Set, MutableSet,
+        Container,
+        Hashable,
+        Sized,
+        Callable,
+        Iterable,
+        Iterator,
     )

lib/ansible/module_utils/network/nxos/nxos.py

@@ -36,6 +36,7 @@
 from ansible.module_utils.basic import env_fallback, return_values
 from ansible.module_utils.network.common.utils import to_list, ComplexList
 from ansible.module_utils.connection import Connection, ConnectionError
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.module_utils.six import iteritems, string_types
 from ansible.module_utils.urls import fetch_url
@@ -179,7 +180,7 @@ def load_config(self, config, return_error=False, opts=None, replace=None):
         responses = []
         try:
             resp = connection.edit_config(config, replace=replace)
-            if isinstance(resp, collections.Mapping):
+            if isinstance(resp, Mapping):
                 resp = resp['response']
         except ConnectionError as e:
             code = getattr(e, 'code', 1)

lib/ansible/module_utils/oneview.py

@@ -43,6 +43,7 @@
 from ansible.module_utils import six
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils._text import to_native
+from ansible.module_utils.common._collections_compat import Mapping
 
 
 def transform_list_to_dict(list_):
@@ -59,7 +60,7 @@ def transform_list_to_dict(list_):
         return ret
 
     for value in list_:
-        if isinstance(value, collections.Mapping):
+        if isinstance(value, Mapping):
             ret.update(value)
         else:
             ret[to_native(value, errors='surrogate_or_strict')] = True
@@ -108,7 +109,7 @@ def merge_list_by_key(original_list, updated_list, key, ignore_when_null=None):
 
 
 def _str_sorted(obj):
-    if isinstance(obj, collections.Mapping):
+    if isinstance(obj, Mapping):
         return json.dumps(obj, sort_keys=True)
     else:
         return str(obj)
@@ -430,7 +431,7 @@ def compare(self, first_resource, second_resource):
             # If both values are null, empty or False it will be considered equal.
             elif not resource1[key] and not resource2[key]:
                 continue
-            elif isinstance(resource1[key], collections.Mapping):
+            elif isinstance(resource1[key], Mapping):
                 # recursive call
                 if not self.compare(resource1[key], resource2[key]):
                     self.module.log(self.MSG_DIFF_AT_KEY.format(key) + debug_resources)
@@ -482,7 +483,7 @@ def compare_list(self, first_resource, second_resource):
         resource2 = sorted(resource2, key=_str_sorted)
 
         for i, val in enumerate(resource1):
-            if isinstance(val, collections.Mapping):
+            if isinstance(val, Mapping):
                 # change comparison function to compare dictionaries
                 if not self.compare(val, resource2[i]):
                     self.module.log("resources are different. " + debug_resources)

lib/ansible/module_utils/ovirt.py

@@ -18,7 +18,6 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-import collections
 import inspect
 import os
 import time
@@ -27,6 +26,8 @@
 from datetime import datetime
 from distutils.version import LooseVersion
 
+from ansible.module_utils.common._collections_compat import Mapping
+
 try:
     from enum import Enum  # enum is a ovirtsdk4 requirement
     import ovirtsdk4 as sdk
@@ -512,7 +513,7 @@ def post_update(self, entity):
 
     def diff_update(self, after, update):
         for k, v in update.items():
-            if isinstance(v, collections.Mapping):
+            if isinstance(v, Mapping):
                 after[k] = self.diff_update(after.get(k, dict()), v)
             else:
                 after[k] = update[k]

lib/ansible/modules/files/xml.py

@@ -266,7 +266,6 @@
 import re
 import traceback
 
-from collections import MutableMapping
 from distutils.version import LooseVersion
 from io import BytesIO
 
@@ -279,6 +278,7 @@
 from ansible.module_utils.basic import AnsibleModule, json_dict_bytes_to_unicode
 from ansible.module_utils.six import iteritems, string_types
 from ansible.module_utils._text import to_bytes, to_native
+from ansible.module_utils.common._collections_compat import MutableMapping
 
 _IDENT = r"[a-zA-Z-][a-zA-Z0-9_\-\.]*"
 _NSIDENT = _IDENT + "|" + _IDENT + ":" + _IDENT

lib/ansible/modules/net_tools/basics/uri.py

@@ -270,11 +270,11 @@
 import tempfile
 import traceback
 
-from collections import Mapping, Sequence
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.six import PY2, iteritems, string_types
 from ansible.module_utils.six.moves.urllib.parse import urlencode, urlsplit
 from ansible.module_utils._text import to_native, to_text
+from ansible.module_utils.common._collections_compat import Mapping, Sequence
 from ansible.module_utils.urls import fetch_url, url_argument_spec
 
 JSON_CANDIDATES = ('text', 'json', 'javascript')

lib/ansible/parsing/ajson.py

@@ -7,10 +7,10 @@
 
 import json
 
-from collections import Mapping
 from datetime import date, datetime
 
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
 from ansible.utils.unsafe_proxy import AnsibleUnsafe, wrap_var
 from ansible.parsing.vault import VaultLib

lib/ansible/playbook/role/__init__.py

@@ -19,11 +19,11 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import os
 
 from ansible.errors import AnsibleError, AnsibleParserError, AnsibleAssertionError
 from ansible.module_utils.six import iteritems, binary_type, text_type
+from ansible.module_utils.common._collections_compat import Container, Mapping, Set, Sequence
 from ansible.playbook.attribute import FieldAttribute
 from ansible.playbook.base import Base
 from ansible.playbook.become import Become
@@ -59,8 +59,8 @@ def hash_params(params):
     # Any container is unhashable if it contains unhashable items (for
     # instance, tuple() is a Hashable subclass but if it contains a dict, it
     # cannot be hashed)
-    if isinstance(params, collections.Container) and not isinstance(params, (text_type, binary_type)):
-        if isinstance(params, collections.Mapping):
+    if isinstance(params, Container) and not isinstance(params, (text_type, binary_type)):
+        if isinstance(params, Mapping):
             try:
                 # Optimistically hope the contents are all hashable
                 new_params = frozenset(params.items())
@@ -71,7 +71,7 @@ def hash_params(params):
                     new_params.update((k, hash_params(v)))
                 new_params = frozenset(new_params)
 
-        elif isinstance(params, (collections.Set, collections.Sequence)):
+        elif isinstance(params, (Set, Sequence)):
             try:
                 # Optimistically hope the contents are all hashable
                 new_params = frozenset(params)

lib/ansible/plugins/action/synchronize.py

@@ -18,11 +18,11 @@
 __metaclass__ = type
 
 import os.path
-from collections import MutableSequence
 
 from ansible import constants as C
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import MutableSequence
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.plugins.action import ActionBase
 from ansible.plugins.loader import connection_loader

lib/ansible/plugins/cache/__init__.py

@@ -21,12 +21,12 @@
 import time
 import errno
 from abc import ABCMeta, abstractmethod
-from collections import MutableMapping
 
 from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.module_utils.six import with_metaclass
 from ansible.module_utils._text import to_bytes
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.plugins.loader import cache_loader
 
 try:

lib/ansible/plugins/cache/memcached.py

@@ -50,6 +50,7 @@
 
 from ansible import constants as C
 from ansible.errors import AnsibleError
+from ansible.module_utils.common._collections_compat import MutableSet
 from ansible.plugins.cache import BaseCacheModule
 
 try:
@@ -126,7 +127,7 @@ def _proxy_client(self, name, *args, **kwargs):
             self.release_connection(conn)
 
 
-class CacheModuleKeys(collections.MutableSet):
+class CacheModuleKeys(MutableSet):
     """
     A set subclass that keeps track of insertion time and persists
     the set in memcached.

lib/ansible/plugins/callback/__init__.py

@@ -26,9 +26,9 @@
 import warnings
 
 from copy import deepcopy
-from collections import MutableMapping
 
 from ansible import constants as C
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.parsing.ajson import AnsibleJSONEncoder
 from ansible.plugins import AnsiblePlugin, get_plugin_class
 from ansible.utils.color import stringc

lib/ansible/plugins/callback/dense.py

@@ -20,8 +20,6 @@
 - set as stdout in configuation
 '''
 
-from collections import MutableMapping, MutableSequence
-
 HAS_OD = False
 try:
     from collections import OrderedDict
@@ -30,6 +28,7 @@
     pass
 
 from ansible.module_utils.six import binary_type, text_type
+from ansible.module_utils.common._collections_compat import MutableMapping, MutableSequence
 from ansible.plugins.callback.default import CallbackModule as CallbackModule_default
 from ansible.utils.color import colorize, hostcolor
 

lib/ansible/plugins/callback/log_plays.py

@@ -21,9 +21,9 @@
 import os
 import time
 import json
-from collections import MutableMapping
 
 from ansible.module_utils._text import to_bytes
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.plugins.callback import CallbackBase