Update ansible-test sanity command. (ansible#31958)

* Use correct pip version in ansible-test.
* Add git fallback for validate-modules.
* Run sanity tests in a docker container.
* Use correct python version for sanity tests.
* Pin docker completion images and add default.
* Split pylint execution into multiple contexts.
* Only test .py files in use-argspec-type-path test.
* Accept identical python interpeter name or binary.
* Switch cloud tests to default container.
* Remove unused extras from pip install.
* Filter out empty pip commands.
* Don't force running of pip list.
* Support delegation for windows and network tests.
* Fix ansible-test python version usage.
* Fix ansible-test python version skipping.
* Use absolute path for log in ansible-test.
* Run vyos_command test on python 3.
* Fix windows/network instance persistence.
* Add `test/cache` dir to classification.
* Enable more python versions for network tests.
* Fix cs_router test.

.gitignore

@@ -69,6 +69,7 @@ results.xml
 coverage.xml
 /test/units/cover-html
 /test/integration/targets/*/backup/
+/test/cache/*
 # Development
 /test/develop
 venv

shippable.yml

@@ -62,11 +62,11 @@ matrix:
     - env: T=linux/ubuntu1604/3
     - env: T=linux/ubuntu1604py3/3
 
-    - env: T=cloud/ubuntu1604/1
-    - env: T=cloud/ubuntu1604py3/1
+    - env: T=cloud/default/2.7/1
+    - env: T=cloud/default/3.6/1
 
-    - env: T=cloud/ubuntu1604/2
-    - env: T=cloud/ubuntu1604py3/2
+    - env: T=cloud/default/2.7/2
+    - env: T=cloud/default/3.6/2
 
 branches:
   except:

test/integration/targets/cs_router/tasks/main.yml

@@ -49,12 +49,8 @@
     - instance.name == "instance-vm"
     - instance.state == "Running"
 
-- name: install jq
-  package:
-    name: jq
-
 - name: setup find the routers name
-  shell: cs listRouters listall=true networkid="{{ net.id }}" zone="{{ cs_common_zone_adv }}" | jq ".router[].name" | tr -d '"'
+  shell: cs listRouters listall=true networkid="{{ net.id }}" zone="{{ cs_common_zone_adv }}"
   args:
     chdir: "{{ playbook_dir }}"
   register: router
@@ -63,7 +59,10 @@
     var: router.stdout
 
 - set_fact:
-    router_name: "{{ router.stdout }}"
+    router_json: "{{ router.stdout | from_json }}"
+
+- set_fact:
+    router_name: "{{ router_json.router[0].name }}"
 
 - name: test router started
   cs_router:

test/integration/targets/vyos_command/aliases

@@ -1,2 +1 @@
 network/ci
-skip/python3

test/runner/completion/docker.txt

@@ -1,9 +1,10 @@
-centos6
-centos7
-fedora24
-fedora25
-opensuse42.2
-opensuse42.3
-ubuntu1404
-ubuntu1604
-ubuntu1604py3
+centos6@sha256:41eb4b870ce400202945ccf572d45bf5f2f5ebb50e9dee244de73b9d0278db30
+centos7@sha256:bd571611112cccefdaa951ea640177cbb77c8ee011f958d2562781d90594ea9c
+default@sha256:424161033bf1342bc463c27c5fad182c171aa3bc17b3c1fe7aac44623cc8d304
+fedora24@sha256:7b642c5d25b779a3a605fb8f70d9d92972f2004a5266fe364264809899fb1117
+fedora25@sha256:828c71d87f1636f4d09916b8e2d87fc9a615d361a9afed22e8843ffb3d2729d2
+opensuse42.2@sha256:fc22d6684910018d2e5f2e8613391b5ae5aca7760d365ac3098971b7aa41d8a2
+opensuse42.3@sha256:7f48e874367528711a1df7ff16da5667d67d2eb15902b8e5151d34546e6af04d
+ubuntu1404@sha256:ba27d23e815a4c3fb361001aea2ef70241d66f08bdf962cf5717037e882ff78a
+ubuntu1604@sha256:ff3898ac817a10ec7129f6483721a717ed0d98c6ba42c27be1472d73908568da
+ubuntu1604py3@sha256:f0b7883eb3f17ee7cb3a77f4aeea0d743101e103f93a76f4f5120aed9c44c0bc

test/runner/injector/ansible-connection

@@ -0,0 +1 @@
+injector.py

test/runner/lib/ansible_util.py

@@ -49,6 +49,9 @@ def ansible_environment(args, color=True):
     env.update(ansible)
 
     if args.debug:
-        env.update(dict(ANSIBLE_DEBUG='true'))
+        env.update(dict(
+            ANSIBLE_DEBUG='true',
+            ANSIBLE_LOG_PATH=os.path.abspath('test/results/logs/debug.log'),
+        ))
 
     return env

test/runner/lib/classification.py

@@ -367,6 +367,9 @@ def _classify(self, path):
 
             return minimal
 
+        if path.startswith('test/cache/'):
+            return minimal
+
         if path.startswith('test/compile/'):
             return {
                 'compile': 'all',

test/runner/lib/config.py

@@ -43,6 +43,7 @@ def __init__(self, args, command):
         self.docker_privileged = args.docker_privileged if 'docker_privileged' in args else False  # type: bool
         self.docker_util = docker_qualify_image(args.docker_util if 'docker_util' in args else '')  # type: str
         self.docker_pull = args.docker_pull if 'docker_pull' in args else False  # type: bool
+        self.docker_keep_git = args.docker_keep_git if 'docker_keep_git' in args else False  # type: bool
 
         self.tox_sitepackages = args.tox_sitepackages  # type: bool
 
@@ -53,7 +54,7 @@ def __init__(self, args, command):
         self.requirements = args.requirements  # type: bool
 
         if self.python == 'default':
-            self.python = '.'.join(str(i) for i in sys.version_info[:2])
+            self.python = None
 
         self.python_version = self.python or '.'.join(str(i) for i in sys.version_info[:2])
 

test/runner/lib/core_ci.py

@@ -8,6 +8,7 @@
 import uuid
 import errno
 import time
+import shutil
 
 from lib.http import (
     HttpClient,
@@ -35,13 +36,14 @@
 
 class AnsibleCoreCI(object):
     """Client for Ansible Core CI services."""
-    def __init__(self, args, platform, version, stage='prod', persist=True, name=None):
+    def __init__(self, args, platform, version, stage='prod', persist=True, load=True, name=None):
         """
         :type args: EnvironmentConfig
         :type platform: str
         :type version: str
         :type stage: str
         :type persist: bool
+        :type load: bool
         :type name: str
         """
         self.args = args
@@ -106,7 +108,7 @@ def __init__(self, args, platform, version, stage='prod', persist=True, name=Non
 
         self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s' % (self.name, self.stage))
 
-        if persist and self._load():
+        if persist and load and self._load():
             try:
                 display.info('Checking existing %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
                              verbosity=1)
@@ -125,7 +127,7 @@ def __init__(self, args, platform, version, stage='prod', persist=True, name=Non
 
                 self.instance_id = None
                 self.endpoint = None
-        else:
+        elif not persist:
             self.instance_id = None
             self.endpoint = None
             self._clear()
@@ -160,6 +162,11 @@ def _get_parallels_endpoints(self):
 
     def start(self):
         """Start instance."""
+        if self.started:
+            display.info('Skipping started %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
+                         verbosity=1)
+            return
+
         if is_shippable():
             return self.start_shippable()
 
@@ -289,11 +296,6 @@ def _uri(self):
 
     def _start(self, auth):
         """Start instance."""
-        if self.started:
-            display.info('Skipping started %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
-                         verbosity=1)
-            return
-
         display.info('Initializing new %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1)
 
         if self.platform == 'windows':
@@ -413,6 +415,13 @@ def _load(self):
 
         config = json.loads(data)
 
+        return self.load(config)
+
+    def load(self, config):
+        """
+        :type config: dict[str, str]
+        :rtype: bool
+        """
         self.instance_id = config['instance_id']
         self.endpoint = config['endpoint']
         self.started = True
@@ -424,16 +433,23 @@ def _save(self):
         if self.args.explain:
             return
 
+        config = self.save()
+
         make_dirs(os.path.dirname(self.path))
 
         with open(self.path, 'w') as instance_fd:
-            config = dict(
-                instance_id=self.instance_id,
-                endpoint=self.endpoint,
-            )
-
             instance_fd.write(json.dumps(config, indent=4, sort_keys=True))
 
+    def save(self):
+        """
+        :rtype: dict[str, str]
+        """
+        return dict(
+            platform_version='%s/%s' % (self.platform, self.version),
+            instance_id=self.instance_id,
+            endpoint=self.endpoint,
+        )
+
     @staticmethod
     def _create_http_error(response):
         """
@@ -472,20 +488,33 @@ def __init__(self, status, remote_message, remote_stack_trace):
 
 class SshKey(object):
     """Container for SSH key used to connect to remote instances."""
+    KEY_NAME = 'id_rsa'
+    PUB_NAME = 'id_rsa.pub'
+
     def __init__(self, args):
         """
         :type args: EnvironmentConfig
         """
-        tmp = os.path.expanduser('~/.ansible/test/')
+        cache_dir = 'test/cache'
+
+        self.key = os.path.join(cache_dir, self.KEY_NAME)
+        self.pub = os.path.join(cache_dir, self.PUB_NAME)
 
-        self.key = os.path.join(tmp, 'id_rsa')
-        self.pub = os.path.join(tmp, 'id_rsa.pub')
+        if not os.path.isfile(self.key) or not os.path.isfile(self.pub):
+            base_dir = os.path.expanduser('~/.ansible/test/')
+
+            key = os.path.join(base_dir, self.KEY_NAME)
+            pub = os.path.join(base_dir, self.PUB_NAME)
 
-        if not os.path.isfile(self.pub):
             if not args.explain:
-                make_dirs(tmp)
+                make_dirs(base_dir)
+
+            if not os.path.isfile(key) or not os.path.isfile(pub):
+                run_command(args, ['ssh-keygen', '-q', '-t', 'rsa', '-N', '', '-f', key])
 
-            run_command(args, ['ssh-keygen', '-q', '-t', 'rsa', '-N', '', '-f', self.key])
+            if not args.explain:
+                shutil.copy2(key, self.key)
+                shutil.copy2(pub, self.pub)
 
         if args.explain:
             self.pub_contents = None

test/runner/lib/delegation.py

@@ -102,10 +102,10 @@ def delegate_tox(args, exclude, require):
     :type require: list[str]
     """
     if args.python:
-        versions = args.python,
+        versions = args.python_version,
 
-        if args.python not in SUPPORTED_PYTHON_VERSIONS:
-            raise ApplicationError('tox does not support Python version %s' % args.python)
+        if args.python_version not in SUPPORTED_PYTHON_VERSIONS:
+            raise ApplicationError('tox does not support Python version %s' % args.python_version)
     else:
         versions = SUPPORTED_PYTHON_VERSIONS
 
@@ -189,7 +189,12 @@ def delegate_docker(args, exclude, require):
     with tempfile.NamedTemporaryFile(prefix='ansible-source-', suffix='.tgz') as local_source_fd:
         try:
             if not args.explain:
-                lib.pytar.create_tarfile(local_source_fd.name, '.', lib.pytar.ignore)
+                if args.docker_keep_git:
+                    tar_filter = lib.pytar.AllowGitTarFilter()
+                else:
+                    tar_filter = lib.pytar.DefaultTarFilter()
+
+                lib.pytar.create_tarfile(local_source_fd.name, '.', tar_filter)
 
             if util_image:
                 util_options = [