Junos fixes (ansible#22423)

* Fixes for junos_config errors

* Check transport settings for core Junos

* Don't pop from the same list you iterate over

* use of persistent connections are now explicitly enabled in junos

* modules must now explicitly enable persistent connections
* adds rpc support to junos_command

fixes ansible#22166

lib/ansible/module_utils/junos.py

@@ -18,7 +18,7 @@
 #
 from contextlib import contextmanager
 
-from ncclient.xml_ import new_ele, sub_ele, to_xml
+from xml.etree.ElementTree import Element, SubElement, tostring
 
 from ansible.module_utils.basic import env_fallback
 from ansible.module_utils.netconf import send_request
@@ -41,6 +41,7 @@
     'ssh_keyfile': dict(fallback=(env_fallback, ['ANSIBLE_NET_SSH_KEYFILE']), type='path'),
     'timeout': dict(type='int', default=10),
     'provider': dict(type='dict'),
+    'transport': dict(choices=['cli', 'netconf'])
 }
 
 def check_args(module, warnings):
@@ -81,17 +82,17 @@ def load_configuration(module, candidate=None, action='merge', rollback=None, fo
     else:
         xattrs = {'action': action, 'format': format}
 
-    obj = new_ele('load-configuration', xattrs)
+    obj = Element('load-configuration', xattrs)
 
     if candidate is not None:
         lookup = {'xml': 'configuration', 'text': 'configuration-text',
                   'set': 'configuration-set', 'json': 'configuration-json'}
 
         if action == 'set':
-            cfg = sub_ele(obj, 'configuration-set')
+            cfg = SubElement(obj, 'configuration-set')
             cfg.text = '\n'.join(candidate)
         else:
-            cfg = sub_ele(obj, lookup[format])
+            cfg = SubElement(obj, lookup[format])
             cfg.append(candidate)
 
     return send_request(module, obj)
@@ -104,22 +105,22 @@ def get_configuration(module, compare=False, format='xml', rollback='0'):
         validate_rollback_id(rollback)
         xattrs['compare'] = 'rollback'
         xattrs['rollback'] = str(rollback)
-    return send_request(module, new_ele('get-configuration', xattrs))
+    return send_request(module, Element('get-configuration', xattrs))
 
 def commit_configuration(module, confirm=False, check=False, comment=None, confirm_timeout=None):
-    obj = new_ele('commit-configuration')
+    obj = Element('commit-configuration')
     if confirm:
-        sub_ele(obj, 'confirmed')
+        SubElement(obj, 'confirmed')
     if check:
-        sub_ele(obj, 'check')
+        SubElement(obj, 'check')
     if comment:
         children(obj, ('log', str(comment)))
     if confirm_timeout:
         children(obj, ('confirm-timeout', int(confirm_timeout)))
     return send_request(module, obj)
 
-lock_configuration = lambda x: send_request(x, new_ele('lock-configuration'))
-unlock_configuration = lambda x: send_request(x, new_ele('unlock-configuration'))
+lock_configuration = lambda x: send_request(x, Element('lock-configuration'))
+unlock_configuration = lambda x: send_request(x, Element('unlock-configuration'))
 
 @contextmanager
 def locked_config(module):
@@ -131,7 +132,7 @@ def locked_config(module):
 
 def get_diff(module):
     reply = get_configuration(module, compare=True, format='text')
-    output = reply.xpath('//configuration-output')
+    output = reply.find('.//configuration-output')
     if output:
         return output[0].text
 

lib/ansible/module_utils/netconf.py

@@ -26,50 +26,50 @@
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 from contextlib import contextmanager
-
-from ncclient.xml_ import new_ele, sub_ele, to_xml, to_ele
+from xml.etree.ElementTree import Element, SubElement
+from xml.etree.ElementTree import tostring, fromstring
 
 from ansible.module_utils.connection import exec_command
 
 def send_request(module, obj, check_rc=True):
-    request = to_xml(obj)
+    request = tostring(obj)
     rc, out, err = exec_command(module, request)
     if rc != 0:
         if check_rc:
             module.fail_json(msg=str(err))
-        return to_ele(err)
-    return to_ele(out)
+        return fromstring(out)
+    return fromstring(out)
 
 def children(root, iterable):
     for item in iterable:
         try:
-            ele = sub_ele(ele, item)
+            ele = SubElement(ele, item)
         except NameError:
-            ele = sub_ele(root, item)
+            ele = SubElement(root, item)
 
 def lock(module, target='candidate'):
-    obj = new_ele('lock')
+    obj = Element('lock')
     children(obj, ('target', target))
     return send_request(module, obj)
 
 def unlock(module, target='candidate'):
-    obj = new_ele('unlock')
+    obj = Element('unlock')
     children(obj, ('target', target))
     return send_request(module, obj)
 
 def commit(module):
-    return send_request(module, new_ele('commit'))
+    return send_request(module, Element('commit'))
 
 def discard_changes(module):
-    return send_request(module, new_ele('discard-changes'))
+    return send_request(module, Element('discard-changes'))
 
 def validate(module):
-    obj = new_ele('validate')
+    obj = Element('validate')
     children(obj, ('source', 'candidate'))
     return send_request(module, obj)
 
 def get_config(module, source='running', filter=None):
-    obj = new_ele('get-config')
+    obj = Element('get-config')
     children(obj, ('source', source))
     children(obj, ('filter', filter))
     return send_request(module, obj)

lib/ansible/modules/network/junos/_junos_template.py

@@ -116,8 +116,16 @@
 from ansible.module_utils.junos import get_configuration, load
 from ansible.module_utils.six import text_type
 
+USE_PERSISTENT_CONNECTION = True
 DEFAULT_COMMENT = 'configured by junos_template'
 
+def check_transport(module):
+    transport = (module.params['provider'] or {}).get('transport')
+
+    if transport == 'netconf':
+        module.fail_json(msg='junos_template module is only supported over cli transport')
+
+
 def main():
 
     argument_spec = dict(
@@ -127,14 +135,15 @@ def main():
         action=dict(default='merge', choices=['merge', 'overwrite', 'replace']),
         config_format=dict(choices=['text', 'set', 'xml']),
         backup=dict(default=False, type='bool'),
-        transport=dict(default='netconf', choices=['netconf'])
     )
 
     argument_spec.update(junos_argument_spec)
 
     module = AnsibleModule(argument_spec=argument_spec,
                            supports_check_mode=True)
 
+    check_transport(module)
+
     warnings = list()
     check_args(module, warnings)
 

lib/ansible/modules/network/junos/junos_command.py

@@ -127,30 +127,44 @@
   sample: ['...', '...']
 """
 import time
+import re
+import shlex
 
 from functools import partial
 from xml.etree import ElementTree as etree
+from xml.etree.ElementTree import Element, SubElement, tostring
+
 
 from ansible.module_utils.junos import run_commands
-from ansible.module_utils.junos import junos_argument_spec
-from ansible.module_utils.junos import check_args as junos_check_args
+from ansible.module_utils.junos import junos_argument_spec, check_args
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.six import string_types
 from ansible.module_utils.netcli import Conditional, FailedConditionalError
-from ansible.module_utils.network_common import ComplexList
+from ansible.module_utils.netconf import send_request
+from ansible.module_utils.network_common import ComplexList, to_list
+from ansible.module_utils.six import string_types, iteritems
 
 try:
     import jxmlease
     HAS_JXMLEASE = True
 except ImportError:
     HAS_JXMLEASE = False
 
-def check_args(module, warnings):
-    junos_check_args(module, warnings)
+USE_PERSISTENT_CONNECTION = True
+
+
+VALID_KEYS = {
+    'cli': frozenset(['command', 'output', 'prompt', 'response']),
+    'rpc': frozenset(['command', 'output'])
+}
+
+def check_transport(module):
+    transport = (module.params['provider'] or {}).get('transport')
 
-    if module.params['rpcs']:
-        module.fail_json(msg='argument rpcs has been deprecated, please use '
-                             'junos_rpc instead')
+    if transport == 'netconf' and not module.params['rpcs']:
+        module.fail_json(msg='argument commands is only supported over cli transport')
+
+    elif transport == 'cli' and not module.params['commands']:
+        module.fail_json(msg='argument rpcs is only supported over netconf transport')
 
 def to_lines(stdout):
     lines = list()
@@ -160,7 +174,78 @@ def to_lines(stdout):
         lines.append(item)
     return lines
 
-def parse_commands(module, warnings):
+def run_rpcs(module, items):
+
+    responses = list()
+
+    for item in items:
+        name = item['name']
+        args = item['args']
+
+        name = str(name).replace('_', '-')
+
+        if all((module.check_mode, not name.startswith('get'))):
+            module.fail_json(msg='invalid rpc for running in check_mode')
+
+        xattrs = {'format': item['output']}
+
+        element = Element(name, xattrs)
+
+        for key, value in iteritems(args):
+            key = str(key).replace('_', '-')
+            if isinstance(value, list):
+                for item in value:
+                    child = SubElement(element, key)
+                    if item is not True:
+                        child.text = item
+            else:
+                child = SubElement(element, key)
+                if value is not True:
+                    child.text = value
+
+        reply = send_request(module, element)
+
+        if module.params['display'] == 'text':
+            data = reply.find('.//output')
+            responses.append(data.text.strip())
+        elif module.params['display'] == 'json':
+            responses.append(module.from_json(reply.text.strip()))
+        else:
+            responses.append(tostring(reply))
+
+    return responses
+
+def split(value):
+    lex = shlex.shlex(value)
+    lex.quotes = '"'
+    lex.whitespace_split = True
+    lex.commenters = ''
+    return list(lex)
+
+def parse_rpcs(module):
+    items = list()
+    for rpc in module.params['rpcs']:
+        parts = split(rpc)
+
+        name = parts.pop(0)
+        args = dict()
+
+        for item in parts:
+            key, value = item.split('=')
+            if str(value).upper() in ['TRUE', 'FALSE']:
+                args[key] = bool(value)
+            elif re.match(r'^[0-9]+$', value):
+                args[key] = int(value)
+            else:
+                args[key] = str(value)
+
+        output = module.params['display'] or 'xml'
+        items.append({'name': name, 'args': args, 'output': output})
+
+    return items
+
+
+def parse_commands(module):
     spec = dict(
         command=dict(key=True),
         output=dict(default=module.params['display'], choices=['text', 'json', 'xml']),
@@ -178,6 +263,8 @@ def parse_commands(module, warnings):
                 'executing %s' % item['command']
             )
 
+        if item['command'].startswith('show configuration'):
+            item['output'] = 'text'
         if item['output'] == 'json' and 'display json' not in item['command']:
             item['command'] += '| display json'
         elif item['output'] == 'xml' and 'display xml' not in item['command']:
@@ -195,12 +282,11 @@ def main():
     """entry point for module execution
     """
     argument_spec = dict(
-        commands=dict(type='list', required=True),
-        display=dict(choices=['text', 'json', 'xml'], default='text', aliases=['format', 'output']),
-
-        # deprecated (Ansible 2.3) - use junos_rpc
+        commands=dict(type='list'),
         rpcs=dict(type='list'),
 
+        display=dict(choices=['text', 'json', 'xml'], aliases=['format', 'output']),
+
         wait_for=dict(type='list', aliases=['waitfor']),
         match=dict(default='all', choices=['all', 'any']),
 
@@ -210,14 +296,25 @@ def main():
 
     argument_spec.update(junos_argument_spec)
 
+    mutually_exclusive = [('commands', 'rpcs')]
+
+    required_one_of = [('commands', 'rpcs')]
+
     module = AnsibleModule(argument_spec=argument_spec,
+                           mutually_exclusive=mutually_exclusive,
+                           required_one_of=required_one_of,
                            supports_check_mode=True)
 
+    check_transport(module)
 
     warnings = list()
     check_args(module, warnings)
 
-    commands = parse_commands(module, warnings)
+    if module.params['commands']:
+        items = parse_commands(module)
+    else:
+        items = parse_rpcs(module)
+
 
     wait_for = module.params['wait_for'] or list()
     display = module.params['display']
@@ -228,21 +325,29 @@ def main():
     match = module.params['match']
 
     while retries > 0:
-        responses = run_commands(module, commands)
+        if module.params['commands']:
+            responses = run_commands(module, items)
+        else:
+            responses = run_rpcs(module, items)
 
-        for index, (resp, cmd) in enumerate(zip(responses, commands)):
-            if cmd['output'] == 'xml':
+        transformed = list()
+
+        for item, resp in zip(items, responses):
+            if item['output'] == 'xml':
                 if not HAS_JXMLEASE:
                     module.fail_json(msg='jxmlease is required but does not appear to '
                         'be installed.  It can be installed using `pip install jxmlease`')
+
                 try:
-                    responses[index] = jxmlease.parse(resp)
+                    transformed.append(jxmlease.parse(resp))
                 except:
                     raise ValueError(resp)
+            else:
+                transformed.append(resp)
 
         for item in list(conditionals):
             try:
-                if item(responses):
+                if item(transformed):
                     if match == 'any':
                         conditionals = list()
                         break