Add support for cliconf and netconf plugin (ansible#25093)

* ansible-connection refactor and action plugin changes
* Add cliconf plugin for eos, ios, iosxr, junos, nxos, vyos
* Add netconf plugin for junos
* Add jsonrpc support
* Modify network_cli and netconf connection plugin
* Fix py3 unit test failure
* Fix review comment
* Minor fixes
* Fix ansible-connection review comments
* Fix CI issue
* platform_agnostic related changes

lib/ansible/constants.py

@@ -394,6 +394,7 @@ def load_config_file():
 PERSISTENT_CONNECT_TIMEOUT = get_config(p, 'persistent_connection', 'connect_timeout', 'ANSIBLE_PERSISTENT_CONNECT_TIMEOUT', 30, value_type='integer')
 PERSISTENT_CONNECT_RETRIES = get_config(p, 'persistent_connection', 'connect_retries', 'ANSIBLE_PERSISTENT_CONNECT_RETRIES', 30, value_type='integer')
 PERSISTENT_CONNECT_INTERVAL = get_config(p, 'persistent_connection', 'connect_interval', 'ANSIBLE_PERSISTENT_CONNECT_INTERVAL', 1, value_type='integer')
+PERSISTENT_CONTROL_PATH_DIR = get_config(p, 'persistent_connection', 'control_path_dir', 'ANSIBLE_PERSISTENT_CONTROL_PATH_DIR', u'~/.ansible/pc')
 
 # obsolete -- will be formally removed
 ACCELERATE_PORT = get_config(p, 'accelerate', 'accelerate_port', 'ACCELERATE_PORT', 5099, value_type='integer')

lib/ansible/module_utils/connection.py

@@ -29,9 +29,13 @@
 import signal
 import socket
 import struct
+import os
+import uuid
+
+from functools import partial
 
 from ansible.module_utils.basic import get_exception
-from ansible.module_utils._text import to_bytes, to_native
+from ansible.module_utils._text import to_bytes, to_native, to_text
 
 
 def send_data(s, data):
@@ -75,4 +79,63 @@ def exec_command(module, command):
 
     sf.close()
 
-    return (rc, to_native(stdout), to_native(stderr))
+    return rc, to_native(stdout), to_native(stderr)
+
+
+class Connection:
+
+    def __init__(self, module):
+        self._module = module
+
+    def __getattr__(self, name):
+        try:
+            return self.__dict__[name]
+        except KeyError:
+            if name.startswith('_'):
+                raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name))
+            return partial(self.__rpc__, name)
+
+    def __rpc__(self, name, *args, **kwargs):
+        """Executes the json-rpc and returns the output received
+           from remote device.
+           :name: rpc method to be executed over connection plugin that implements jsonrpc 2.0
+           :args: Ordered list of params passed as arguments to rpc method
+           :kwargs: Dict of valid key, value pairs passed as arguments to rpc method
+
+           For usage refer the respective connection plugin docs.
+        """
+
+        reqid = str(uuid.uuid4())
+        req = {'jsonrpc': '2.0', 'method': name, 'id': reqid}
+
+        params = list(args) or kwargs or None
+        if params:
+            req['params'] = params
+
+        if not self._module._socket_path:
+            self._module.fail_json(msg='provider support not available for this host')
+
+        if not os.path.exists(self._module._socket_path):
+            self._module.fail_json(msg='provider socket does not exist, is the provider running?')
+
+        try:
+            data = self._module.jsonify(req)
+            rc, out, err = exec_command(self._module, data)
+
+        except socket.error:
+            exc = get_exception()
+            self._module.fail_json(msg='unable to connect to socket', err=str(exc))
+
+        try:
+            response = self._module.from_json(to_text(out, errors='surrogate_then_replace'))
+        except ValueError as exc:
+            self._module.fail_json(msg=to_text(exc, errors='surrogate_then_replace'))
+
+        if response['id'] != reqid:
+            self._module.fail_json(msg='invalid id received')
+
+        if 'error' in response:
+            msg = response['error'].get('data') or response['error']['message']
+            self._module.fail_json(msg=to_text(msg, errors='surrogate_then_replace'))
+
+        return response['result']

lib/ansible/plugins/__init__.py

@@ -550,3 +550,19 @@ def all(self, *args, **kwargs):
     C.DEFAULT_VARS_PLUGIN_PATH,
     'vars_plugins',
 )
+
+cliconf_loader = PluginLoader(
+    'Cliconf',
+    'ansible.plugins.cliconf',
+    'cliconf_plugins',
+    'cliconf_plugins',
+    required_base_class='CliconfBase'
+)
+
+netconf_loader = PluginLoader(
+    'Netconf',
+    'ansible.plugins.netconf',
+    'netconf_plugins',
+    'netconf_plugins',
+    required_base_class='NetconfBase'
+)

lib/ansible/plugins/action/ce.py

@@ -19,17 +19,13 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import os
 import sys
 import copy
 
 from ansible.plugins.action.normal import ActionModule as _ActionModule
-from ansible.utils.path import unfrackpath
-from ansible.plugins import connection_loader
 from ansible.module_utils.six import iteritems
 from ansible.module_utils.ce import ce_argument_spec
 from ansible.module_utils.basic import AnsibleFallbackNotFound
-from ansible.module_utils._text import to_bytes
 
 try:
     from __main__ import display
@@ -71,26 +67,21 @@ def run(self, tmp=None, task_vars=None):
             )
             display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
             connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
-            socket_path = self._get_socket_path(pc)
-            display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
 
-            if not os.path.exists(socket_path):
-                # start the connection if it isn't started
-                rc, out, err = connection.exec_command('open_shell()')
-                display.vvvv('open_shell() returned %s %s %s' % (rc, out, err))
-                if rc != 0:
-                    return {'failed': True,
-                            'msg': 'unable to open shell. Please see: ' +
-                            'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell',
-                            'rc': rc}
-            else:
-                # make sure we are in the right cli context which should be
-                # enable mode and not config module
+            socket_path = connection.run()
+            display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
+            if not socket_path:
+                return {'failed': True,
+                        'msg': 'unable to open shell. Please see: ' +
+                               'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
+
+            # make sure we are in the right cli context which should be
+            # enable mode and not config module
+            rc, out, err = connection.exec_command('prompt()')
+            while str(out).strip().endswith(']'):
+                display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
+                connection.exec_command('return')
                 rc, out, err = connection.exec_command('prompt()')
-                while str(out).strip().endswith(']'):
-                    display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
-                    connection.exec_command('return')
-                    rc, out, err = connection.exec_command('prompt()')
 
             task_vars['ansible_socket'] = socket_path
 
@@ -100,12 +91,6 @@ def run(self, tmp=None, task_vars=None):
         result = super(ActionModule, self).run(tmp, task_vars)
         return result
 
-    def _get_socket_path(self, play_context):
-        ssh = connection_loader.get('ssh', class_only=True)
-        cp = ssh._create_control_path(play_context.remote_addr, play_context.port, play_context.remote_user)
-        path = unfrackpath("$HOME/.ansible/pc")
-        return cp % dict(directory=path)
-
     def load_provider(self):
         provider = self._task.args.get('provider', {})
         for key, value in iteritems(ce_argument_spec):

lib/ansible/plugins/action/dellos10.py

@@ -21,17 +21,13 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import os
 import sys
 import copy
 
 from ansible.plugins.action.normal import ActionModule as _ActionModule
-from ansible.utils.path import unfrackpath
-from ansible.plugins import connection_loader
 from ansible.module_utils.six import iteritems
 from ansible.module_utils.dellos10 import dellos10_argument_spec
 from ansible.module_utils.basic import AnsibleFallbackNotFound
-from ansible.module_utils._text import to_bytes
 
 try:
     from __main__ import display
@@ -67,26 +63,20 @@ def run(self, tmp=None, task_vars=None):
         display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
         connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
 
-        socket_path = self._get_socket_path(pc)
+        socket_path = connection.run()
         display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
-
-        if not os.path.exists(socket_path):
-            # start the connection if it isn't started
-            rc, out, err = connection.exec_command('open_shell()')
-            display.vvvv('open_shell() returned %s %s %s' % (rc, out, err))
-            if not rc == 0:
-                return {'failed': True,
-                        'msg': 'unable to open shell. Please see: ' +
-                               'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell',
-                        'rc': rc}
-        else:
-            # make sure we are in the right cli context which should be
-            # enable mode and not config module
+        if not socket_path:
+            return {'failed': True,
+                    'msg': 'unable to open shell. Please see: ' +
+                           'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
+
+        # make sure we are in the right cli context which should be
+        # enable mode and not config module
+        rc, out, err = connection.exec_command('prompt()')
+        while str(out).strip().endswith(')#'):
+            display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
+            connection.exec_command('exit')
             rc, out, err = connection.exec_command('prompt()')
-            while str(out).strip().endswith(')#'):
-                display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
-                connection.exec_command('exit')
-                rc, out, err = connection.exec_command('prompt()')
 
         task_vars['ansible_socket'] = socket_path
 
@@ -97,12 +87,6 @@ def run(self, tmp=None, task_vars=None):
         result = super(ActionModule, self).run(tmp, task_vars)
         return result
 
-    def _get_socket_path(self, play_context):
-        ssh = connection_loader.get('ssh', class_only=True)
-        cp = ssh._create_control_path(play_context.remote_addr, play_context.port, play_context.remote_user)
-        path = unfrackpath("$HOME/.ansible/pc")
-        return cp % dict(directory=path)
-
     def load_provider(self):
         provider = self._task.args.get('provider', {})
         for key, value in iteritems(dellos10_argument_spec):

lib/ansible/plugins/action/dellos6.py

@@ -18,17 +18,13 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import os
 import sys
 import copy
 
 from ansible.plugins.action.normal import ActionModule as _ActionModule
-from ansible.utils.path import unfrackpath
-from ansible.plugins import connection_loader
 from ansible.module_utils.six import iteritems
 from ansible.module_utils.dellos6 import dellos6_argument_spec
 from ansible.module_utils.basic import AnsibleFallbackNotFound
-from ansible.module_utils._text import to_bytes
 
 try:
     from __main__ import display
@@ -63,26 +59,20 @@ def run(self, tmp=None, task_vars=None):
         display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
         connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
 
-        socket_path = self._get_socket_path(pc)
+        socket_path = connection.run()
         display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
-
-        if not os.path.exists(socket_path):
-            # start the connection if it isn't started
-            rc, out, err = connection.exec_command('open_shell()')
-            display.vvvv('open_shell() returned %s %s %s' % (rc, out, err))
-            if not rc == 0:
-                return {'failed': True,
-                        'msg': 'unable to open shell. Please see: ' +
-                               'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell',
-                        'rc': rc}
-        else:
-            # make sure we are in the right cli context which should be
-            # enable mode and not config module
+        if not socket_path:
+            return {'failed': True,
+                    'msg': 'unable to open shell. Please see: ' +
+                           'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
+
+        # make sure we are in the right cli context which should be
+        # enable mode and not config module
+        rc, out, err = connection.exec_command('prompt()')
+        while str(out).strip().endswith(')#'):
+            display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
+            connection.exec_command('exit')
             rc, out, err = connection.exec_command('prompt()')
-            while str(out).strip().endswith(')#'):
-                display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
-                connection.exec_command('exit')
-                rc, out, err = connection.exec_command('prompt()')
 
         task_vars['ansible_socket'] = socket_path
 
@@ -93,12 +83,6 @@ def run(self, tmp=None, task_vars=None):
         result = super(ActionModule, self).run(tmp, task_vars)
         return result
 
-    def _get_socket_path(self, play_context):
-        ssh = connection_loader.get('ssh', class_only=True)
-        cp = ssh._create_control_path(play_context.remote_addr, play_context.port, play_context.remote_user)
-        path = unfrackpath("$HOME/.ansible/pc")
-        return cp % dict(directory=path)
-
     def load_provider(self):
         provider = self._task.args.get('provider', {})
         for key, value in iteritems(dellos6_argument_spec):

lib/ansible/plugins/action/dellos9.py

@@ -21,17 +21,13 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import os
 import sys
 import copy
 
 from ansible.plugins.action.normal import ActionModule as _ActionModule
-from ansible.utils.path import unfrackpath
-from ansible.plugins import connection_loader
 from ansible.module_utils.six import iteritems
 from ansible.module_utils.dellos9 import dellos9_argument_spec
 from ansible.module_utils.basic import AnsibleFallbackNotFound
-from ansible.module_utils._text import to_bytes
 
 try:
     from __main__ import display
@@ -67,26 +63,20 @@ def run(self, tmp=None, task_vars=None):
         display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
         connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
 
-        socket_path = self._get_socket_path(pc)
+        socket_path = connection.run()
         display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
-
-        if not os.path.exists(socket_path):
-            # start the connection if it isn't started
-            rc, out, err = connection.exec_command('open_shell()')
-            display.vvvv('open_shell() returned %s %s %s' % (rc, out, err))
-            if not rc == 0:
-                return {'failed': True,
-                        'msg': 'unable to open shell. Please see: ' +
-                               'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell',
-                        'rc': rc}
-        else:
-            # make sure we are in the right cli context which should be
-            # enable mode and not config module
+        if not socket_path:
+            return {'failed': True,
+                    'msg': 'unable to open shell. Please see: ' +
+                           'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
+
+        # make sure we are in the right cli context which should be
+        # enable mode and not config module
+        rc, out, err = connection.exec_command('prompt()')
+        while str(out).strip().endswith(')#'):
+            display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
+            connection.exec_command('exit')
             rc, out, err = connection.exec_command('prompt()')
-            while str(out).strip().endswith(')#'):
-                display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
-                connection.exec_command('exit')
-                rc, out, err = connection.exec_command('prompt()')
 
         task_vars['ansible_socket'] = socket_path
 
@@ -97,12 +87,6 @@ def run(self, tmp=None, task_vars=None):
         result = super(ActionModule, self).run(tmp, task_vars)
         return result
 
-    def _get_socket_path(self, play_context):
-        ssh = connection_loader.get('ssh', class_only=True)
-        cp = ssh._create_control_path(play_context.remote_addr, play_context.port, play_context.remote_user)
-        path = unfrackpath("$HOME/.ansible/pc")
-        return cp % dict(directory=path)
-
     def load_provider(self):
         provider = self._task.args.get('provider', {})
         for key, value in iteritems(dellos9_argument_spec):