Add collections.abc backport to fix deprecation warning (pantsbuild…

…#7055)

### Problem
Python 3 moved several abstract classes like `Sequence` from `collections` to `collections.abc`. Because the `collections` members will be removed in Python 3.8, they result in deprecation warnings when running Pants with Python 3.

We first addressed this issue in pantsbuild#6747, where we had to stop using `future.moves.collections` due to issues with its source code. There, we had decided to accept the deprecation warning with the reasoning we would drop Py2 before Py3.8 is released and fix the deprecation then.

However, now that we will soon be able to run `./pants3`, the deprecation is far too annoying to not deal with.

### Solution
Add our own custom backport specifically for `collections.abc`.

Note we only use the backport with members of `collections.abc` like `Mapping`. Non-abstract base classes like `defaultdict` and `namedtuple` should continue to use `collections` in both Py2 and Py3.

build-support/bin/check_banned_imports.sh

@@ -12,7 +12,7 @@ fi
 
 bad_files="$(echo ${PYTHON_FILES} | xargs grep -l "^import future.moves.collections\|^from future.moves.collections import\|^from future.moves import .*collections")"
 if [ -n "${bad_files}" ]; then
-    echo >&2 "Found forbidden imports. \`future.moves.collections\` does not work as intended. Instead, you should use \`import collections\`. Bad files:"
+    echo >&2 "Found forbidden imports. \`future.moves.collections\` does not work as intended. Instead, you should use \`import collections\` or \`from pants.util.collections_abc_backport\`. Bad files:"
     echo >&2 "${bad_files}"
     exit 1
 fi

contrib/go/src/python/pants/contrib/go/subsystems/BUILD

@@ -3,6 +3,7 @@
 
 python_library(
   dependencies=[
+    'src/python/pants/util:collections_abc_backport',
     '3rdparty/python:future',
     '3rdparty/python:requests',
     '3rdparty/python:pyopenssl',

contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py

@@ -6,11 +6,12 @@
 
 import os
 from builtins import str
-from collections import OrderedDict, namedtuple
+from collections import namedtuple
 
 from pants.base.workunit import WorkUnit, WorkUnitLabel
 from pants.binaries.binary_tool import NativeTool
 from pants.binaries.binary_util import BinaryToolUrlGenerator
+from pants.util.collections_abc_backport import OrderedDict
 from pants.util.memo import memoized_property
 from pants.util.process_handler import subprocess
 

contrib/python/src/python/pants/contrib/python/checks/checker/common.py

@@ -11,12 +11,21 @@
 import textwrap
 import tokenize
 from builtins import object, open
-from collections import Sequence
 from io import StringIO
 
 import six
 
 
+# NB: we cannot use `pants.util.collections_abc_backport` because this library
+# must be able to be exported as universal. At the moment, Pants only is released with Py2,
+# so violates this. Even when we add Py3 support, we will want to keep this approach
+# because Pants will eventually drop internal Py2 support, yet this checker library
+# must still be universal.
+try:
+  from collections.abc import Sequence
+except ImportError:
+  from collections import Sequence
+
 __all__ = (
   'CheckstylePlugin',
   'PythonFile',

src/python/pants/backend/codegen/protobuf/java/BUILD

@@ -3,6 +3,7 @@
 
 python_library(
   dependencies = [
+    'src/python/pants/util:collections_abc_backport',
     '3rdparty/python:future',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     'src/python/pants/backend/codegen/protobuf/subsystems',

src/python/pants/backend/codegen/protobuf/java/protobuf_gen.py

@@ -6,7 +6,6 @@
 
 import os
 from builtins import open
-from collections import OrderedDict
 from hashlib import sha1
 
 from twitter.common.collections import OrderedSet
@@ -22,6 +21,7 @@
 from pants.build_graph.address import Address
 from pants.fs.archive import ZIP
 from pants.task.simple_codegen_task import SimpleCodegenTask
+from pants.util.collections_abc_backport import OrderedDict
 from pants.util.process_handler import subprocess
 
 

src/python/pants/backend/jvm/BUILD

@@ -45,6 +45,7 @@ python_library(
   name='ivy_utils',
   sources=['ivy_utils.py'],
   dependencies=[
+    'src/python/pants/util:collections_abc_backport',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     '3rdparty/python:future',
     '3rdparty/python:six',

src/python/pants/backend/jvm/ivy_utils.py

@@ -12,7 +12,7 @@
 import xml.etree.ElementTree as ET
 from abc import abstractmethod
 from builtins import object, open, str
-from collections import OrderedDict, defaultdict, namedtuple
+from collections import defaultdict, namedtuple
 from functools import total_ordering
 
 import six
@@ -31,6 +31,7 @@
 from pants.java.jar.jar_dependency import JarDependency
 from pants.java.jar.jar_dependency_utils import M2Coordinate, ResolvedJar
 from pants.java.util import execute_runner
+from pants.util.collections_abc_backport import OrderedDict
 from pants.util.dirutil import safe_concurrent_creation, safe_mkdir, safe_open
 from pants.util.fileutil import atomic_copy, safe_hardlink_or_copy
 

src/python/pants/backend/jvm/tasks/BUILD

@@ -174,6 +174,7 @@ python_library(
   name = 'classpath_util',
   sources = ['classpath_util.py'],
   dependencies = [
+    'src/python/pants/util:collections_abc_backport',
     ':classpath_entry',
     '3rdparty/python:future',
     '3rdparty/python/twitter/commons:twitter.common.collections',
@@ -322,6 +323,7 @@ python_library(
   name = 'jar_publish',
   sources = ['jar_publish.py'],
   dependencies = [
+    'src/python/pants/util:collections_abc_backport',
     '3rdparty/python:future',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     ':jar_publish_resources',
@@ -598,6 +600,7 @@ python_library(
   name = 'properties',
   sources = ['properties.py'],
   dependencies = [
+    'src/python/pants/util:collections_abc_backport',
     '3rdparty/python:future',
     '3rdparty/python:six',
   ],

src/python/pants/backend/jvm/tasks/classpath_util.py

@@ -6,11 +6,11 @@
 
 import os
 from builtins import filter, object
-from collections import OrderedDict
 
 from twitter.common.collections import OrderedSet
 
 from pants.backend.jvm.tasks.classpath_entry import ClasspathEntry
+from pants.util.collections_abc_backport import OrderedDict
 from pants.util.contextutil import open_zip
 from pants.util.dirutil import fast_relpath, safe_walk
 from pants.util.strutil import ensure_text

src/python/pants/backend/jvm/tasks/jar_publish.py

@@ -12,7 +12,7 @@
 import shutil
 import sys
 from builtins import input, next, object, open, str
-from collections import OrderedDict, defaultdict, namedtuple
+from collections import defaultdict, namedtuple
 from copy import copy
 
 from future.utils import PY3
@@ -36,6 +36,7 @@
 from pants.ivy.ivy import Ivy
 from pants.task.scm_publish_mixin import Namedver, ScmPublishMixin, Semver
 from pants.task.target_restriction_mixins import HasTransitiveOptionMixin, TransitiveOptionRegistrar
+from pants.util.collections_abc_backport import OrderedDict
 from pants.util.dirutil import safe_mkdir, safe_open, safe_rmtree
 from pants.util.strutil import ensure_text
 

src/python/pants/backend/jvm/tasks/jvmdoc_gen.py

@@ -4,11 +4,11 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import collections
 import contextlib
 import multiprocessing
 import os
 import re
+from collections import namedtuple
 from multiprocessing.pool import ThreadPool
 
 from pants.backend.jvm.tasks.jvm_task import JvmTask
@@ -22,7 +22,7 @@
 from pants.util.process_handler import subprocess
 
 
-Jvmdoc = collections.namedtuple('Jvmdoc', ['tool_name', 'product_type'])
+Jvmdoc = namedtuple('Jvmdoc', ['tool_name', 'product_type'])
 
 
 # TODO: Shouldn't this be a NailgunTask?

src/python/pants/backend/jvm/tasks/properties.py

@@ -6,10 +6,11 @@
 
 import re
 from builtins import next, object, open, str
-from collections import OrderedDict
 
 import six
 
+from pants.util.collections_abc_backport import OrderedDict
+
 
 class Properties(object):
   """A Python reader for java.util.Properties formatted data.

src/python/pants/backend/jvm/tasks/reports/junit_html_report.py

@@ -4,14 +4,14 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import collections
 import fnmatch
 import itertools
 import logging
 import os
 import xml.etree.ElementTree as ET
 from abc import abstractmethod
 from builtins import map, next, object, open
+from collections import defaultdict
 from functools import total_ordering
 
 from pants.base.mustache import MustacheRenderer
@@ -55,12 +55,12 @@ def merged(cls, report_test_suites, error_on_conflict=True, logger=None):
 
     logger = logger or _LOGGER
 
-    suites_by_name = collections.defaultdict(list)
+    suites_by_name = defaultdict(list)
     for report_test_suite in report_test_suites:
       suites_by_name[report_test_suite.name].append(report_test_suite)
 
     for suite_name, suites in suites_by_name.items():
-      cases_by_name = collections.defaultdict(list)
+      cases_by_name = defaultdict(list)
       for case in itertools.chain.from_iterable(s.testcases for s in suites):
         cases_by_name[case.name].append(case)
 

src/python/pants/backend/python/tasks/BUILD

@@ -3,6 +3,7 @@
 
 python_library(
   dependencies=[
+    'src/python/pants/util:collections_abc_backport',
     '3rdparty/python:future',
     '3rdparty/python:pex',
     '3rdparty/python/twitter/commons:twitter.common.collections',

src/python/pants/backend/python/tasks/pytest_run.py

@@ -12,7 +12,6 @@
 import traceback
 import uuid
 from builtins import open, str
-from collections import OrderedDict
 from contextlib import contextmanager
 from io import StringIO
 from textwrap import dedent
@@ -28,6 +27,7 @@
 from pants.build_graph.target import Target
 from pants.task.task import Task
 from pants.task.testrunner_task_mixin import PartitionedTestRunnerTaskMixin, TestResult
+from pants.util.collections_abc_backport import OrderedDict
 from pants.util.contextutil import environment_as, pushd, temporary_dir, temporary_file
 from pants.util.dirutil import mergetree, safe_mkdir, safe_mkdir_for
 from pants.util.memo import memoized_method, memoized_property

src/python/pants/backend/python/tasks/setup_py.py

@@ -12,7 +12,7 @@
 import textwrap
 from abc import abstractmethod
 from builtins import bytes, map, object, str, zip
-from collections import Iterable, Mapping, MutableSequence, OrderedDict, Set, defaultdict
+from collections import defaultdict
 
 from future.utils import PY2
 from pex.installer import InstallerBase, Packager
@@ -35,6 +35,7 @@
 from pants.build_graph.build_graph import sort_targets
 from pants.build_graph.resources import Resources
 from pants.task.task import Task
+from pants.util.collections_abc_backport import Iterable, Mapping, MutableSequence, OrderedDict, Set
 from pants.util.contextutil import temporary_file
 from pants.util.dirutil import safe_concurrent_creation, safe_rmtree, safe_walk
 from pants.util.memo import memoized_property

src/python/pants/base/BUILD

@@ -175,6 +175,7 @@ python_library(
   name = 'run_info',
   sources = ['run_info.py'],
   dependencies = [
+    'src/python/pants/util:collections_abc_backport',
     '3rdparty/python:future',
     ':build_environment',
     'src/python/pants/util:dirutil',

src/python/pants/base/run_info.py

@@ -10,10 +10,10 @@
 import socket
 import time
 from builtins import object, open, str
-from collections import OrderedDict
 
 from pants import version
 from pants.base.build_environment import get_buildroot, get_scm
+from pants.util.collections_abc_backport import OrderedDict
 from pants.util.dirutil import safe_mkdir_for
 
 

src/python/pants/build_graph/BUILD

@@ -4,6 +4,7 @@
 
 python_library(
   dependencies = [
+    'src/python/pants/util:collections_abc_backport',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     '3rdparty/python:future',
     '3rdparty/python:six',

src/python/pants/build_graph/app_base.py

@@ -6,7 +6,7 @@
 
 import os
 from builtins import map, object, open
-from collections import OrderedDict, namedtuple
+from collections import namedtuple
 from hashlib import sha1
 
 import six
@@ -20,6 +20,7 @@
 from pants.build_graph.target import Target
 from pants.fs import archive as Archive
 from pants.source.wrapped_globs import FilesetWithSpec
+from pants.util.collections_abc_backport import OrderedDict
 from pants.util.dirutil import fast_relpath
 from pants.util.memo import memoized_property
 

src/python/pants/build_graph/build_configuration.py

@@ -6,7 +6,7 @@
 
 import logging
 from builtins import object, str
-from collections import Iterable, namedtuple
+from collections import namedtuple
 
 from twitter.common.collections import OrderedSet
 
@@ -18,6 +18,7 @@
 from pants.engine.rules import RuleIndex
 from pants.option.optionable import Optionable
 from pants.subsystem.subsystem import Subsystem
+from pants.util.collections_abc_backport import Iterable
 from pants.util.memo import memoized_method
 
 

src/python/pants/build_graph/build_graph.py

@@ -8,14 +8,15 @@
 import logging
 from abc import abstractmethod
 from builtins import filter, object
-from collections import OrderedDict, defaultdict, deque
+from collections import defaultdict, deque
 
 from twitter.common.collections import OrderedSet
 
 from pants.build_graph.address import Address
 from pants.build_graph.address_lookup_error import AddressLookupError
 from pants.build_graph.injectables_mixin import InjectablesMixin
 from pants.build_graph.target import Target
+from pants.util.collections_abc_backport import OrderedDict
 from pants.util.meta import AbstractClass
 
 

src/python/pants/cache/BUILD

@@ -3,6 +3,7 @@
 
 python_library(
   dependencies = [
+    'src/python/pants/util:collections_abc_backport',
     '3rdparty/python:future',
     '3rdparty/python:requests',
     '3rdparty/python:pyopenssl',

src/python/pants/cache/pinger.py

@@ -5,14 +5,15 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from builtins import object, range, zip
-from collections import Counter, deque
+from collections import deque
 from contextlib import contextmanager
 from multiprocessing.pool import ThreadPool
 
 import requests
 from future.moves.urllib.parse import urlparse
 
 from pants.cache.artifact_cache import ArtifactCacheError
+from pants.util.collections_abc_backport import Counter
 from pants.util.contextutil import Timer
 from pants.util.memo import memoized_method