Allow persistent connection plugins to queue messages back to ansible…

…-connection (ansible#49977)

* Connections can queue messages to be returned from ansible-connection

* Provide fallback for invalid display level

* Strip display from plugins

* Route messages through helper method to try to avoid improper appends

bin/ansible-connection

@@ -90,7 +90,7 @@ class ConnectionProcess(object):
             messages = list()
             result = {}
 
-            messages.append('control socket path is %s' % self.socket_path)
+            messages.append(('vvvv', 'control socket path is %s' % self.socket_path))
 
             # If this is a relative path (~ gets expanded later) then plug the
             # key's path on to the directory we originally came from, so we can
@@ -100,17 +100,18 @@ class ConnectionProcess(object):
             self.connection = connection_loader.get(self.play_context.connection, self.play_context, '/dev/null',
                                                     ansible_playbook_pid=self._ansible_playbook_pid)
             self.connection.set_options(var_options=variables)
+
             self.connection._connect()
 
             self.connection._socket_path = self.socket_path
             self.srv.register(self.connection)
-            messages.extend(sys.stdout.getvalue().splitlines())
-            messages.append('connection to remote device started successfully')
+            messages.extend([('vvvv', msg) for msg in sys.stdout.getvalue().splitlines()])
+            messages.append(('vvvv', 'connection to remote device started successfully'))
 
             self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
             self.sock.bind(self.socket_path)
             self.sock.listen(1)
-            messages.append('local domain socket listeners started successfully')
+            messages.append(('vvvv', 'local domain socket listeners started successfully'))
         except Exception as exc:
             result['error'] = to_text(exc)
             result['exception'] = traceback.format_exc()
@@ -256,7 +257,7 @@ def main():
 
         with file_lock(lock_path):
             if not os.path.exists(socket_path):
-                messages.append('local domain socket does not exist, starting it')
+                messages.append(('vvvv', 'local domain socket does not exist, starting it'))
                 original_path = os.getcwd()
                 r, w = os.pipe()
                 pid = fork_process()
@@ -268,7 +269,7 @@ def main():
                         process = ConnectionProcess(wfd, play_context, socket_path, original_path, ansible_playbook_pid)
                         process.start(variables)
                     except Exception:
-                        messages.append(traceback.format_exc())
+                        messages.append(('error', traceback.format_exc()))
                         rc = 1
 
                     if rc == 0:
@@ -286,12 +287,12 @@ def main():
                     result.update(data)
 
             else:
-                messages.append('found existing local domain socket, using it!')
+                messages.append(('vvvv', 'found existing local domain socket, using it!'))
                 conn = Connection(socket_path)
                 conn.set_options(var_options=variables)
                 pc_data = to_text(init_data)
                 try:
-                    messages.extend(conn.update_play_context(pc_data))
+                    conn.update_play_context(pc_data)
                 except Exception as exc:
                     # Only network_cli has update_play context, so missing this is
                     # not fatal e.g. netconf
@@ -303,7 +304,8 @@ def main():
                             'exception': traceback.format_exc()
                         })
 
-    messages.append(sys.stdout.getvalue())
+    messages.extend(Connection(socket_path).pop_messages())
+    messages.append(('vvvv', sys.stdout.getvalue()))
     result.update({
         'messages': messages,
         'socket_path': socket_path

lib/ansible/executor/task_executor.py

@@ -1027,8 +1027,16 @@ def _start_connection(self, variables):
                 result = {'error': to_text(stderr, errors='surrogate_then_replace')}
 
         if 'messages' in result:
-            for msg in result.get('messages'):
-                display.vvvv('%s' % msg, host=self._play_context.remote_addr)
+            for level, message in result['messages']:
+                if level == 'log':
+                    display.display(message, log_only=True)
+                elif level in ('debug', 'v', 'vv', 'vvv', 'vvvv', 'vvvvv', 'vvvvvv'):
+                    getattr(display, level)(message, host=self._play_context.remote_addr)
+                else:
+                    if hasattr(display, level):
+                        getattr(display, level)(message)
+                    else:
+                        display.vvvv(message, host=self._play_context.remote_addr)
 
         if 'error' in result:
             if self._play_context.verbosity > 2:

lib/ansible/plugins/cliconf/__init__.py

@@ -25,16 +25,13 @@
 from ansible.plugins import AnsiblePlugin
 from ansible.errors import AnsibleError, AnsibleConnectionFailure
 from ansible.module_utils._text import to_bytes, to_text
-from ansible.utils.display import Display
 
 try:
     from scp import SCPClient
     HAS_SCP = True
 except ImportError:
     HAS_SCP = False
 
-display = Display()
-
 
 def enable_mode(func):
     @wraps(func)
@@ -88,7 +85,7 @@ def __init__(self, connection):
 
     def _alarm_handler(self, signum, frame):
         """Alarm handler raised in case of command timeout """
-        display.display('closing shell due to command timeout (%s seconds).' % self._connection._play_context.timeout, log_only=True)
+        self._connection.queue_message('log', 'closing shell due to command timeout (%s seconds).' % self._connection._play_context.timeout)
         self.close()
 
     def send_command(self, command=None, prompt=None, answer=None, sendonly=False, newline=True, prompt_retry_check=False, check_all=False):

lib/ansible/plugins/cliconf/routeros.py

@@ -27,9 +27,6 @@
 from ansible.module_utils._text import to_bytes, to_text
 from ansible.module_utils.network.common.utils import to_list
 from ansible.plugins.cliconf import CliconfBase, enable_mode
-from ansible.utils.display import Display
-
-display = Display()
 
 
 class Cliconf(CliconfBase):

lib/ansible/plugins/connection/__init__.py

@@ -291,6 +291,7 @@ class NetworkConnectionBase(ConnectionBase):
 
     def __init__(self, play_context, new_stdin, *args, **kwargs):
         super(NetworkConnectionBase, self).__init__(play_context, new_stdin, *args, **kwargs)
+        self._messages = []
 
         self._network_os = self._play_context.network_os
 
@@ -319,6 +320,20 @@ def __getattr__(self, name):
     def exec_command(self, cmd, in_data=None, sudoable=True):
         return self._local.exec_command(cmd, in_data, sudoable)
 
+    def queue_message(self, level, message):
+        """
+        Adds a message to the queue of messages waiting to be pushed back to the controller process.
+
+        :arg level: A string which can either be the name of a method in display, or 'log'. When
+            the messages are returned to task_executor, a value of log will correspond to
+            ``display.display(message, log_only=True)``, while another value will call ``display.[level](message)``
+        """
+        self._messages.append((level, message))
+
+    def pop_messages(self):
+        messages, self._messages = self._messages, []
+        return messages
+
     def put_file(self, in_path, out_path):
         """Transfer a file from local to remote"""
         return self._local.put_file(in_path, out_path)
@@ -332,9 +347,9 @@ def reset(self):
         Reset the connection
         '''
         if self._socket_path:
-            display.vvvv('resetting persistent connection for socket_path %s' % self._socket_path, host=self._play_context.remote_addr)
+            self.queue_message('vvvv', 'resetting persistent connection for socket_path %s' % self._socket_path)
             self.close()
-        display.vvvv('reset call on connection instance', host=self._play_context.remote_addr)
+        self.queue_message('vvvv', 'reset call on connection instance')
 
     def close(self):
         if self._connected:

lib/ansible/plugins/connection/httpapi.py

@@ -156,9 +156,6 @@
 from ansible.playbook.play_context import PlayContext
 from ansible.plugins.loader import httpapi_loader
 from ansible.plugins.connection import NetworkConnectionBase
-from ansible.utils.display import Display
-
-display = Display()
 
 
 class Connection(NetworkConnectionBase):
@@ -178,7 +175,7 @@ def __init__(self, play_context, new_stdin, *args, **kwargs):
             self.httpapi = httpapi_loader.get(self._network_os, self)
             if self.httpapi:
                 self._sub_plugin = {'type': 'httpapi', 'name': self._network_os, 'obj': self.httpapi}
-                display.vvvv('loaded API plugin for network_os %s' % self._network_os)
+                self.queue_message('vvvv', 'loaded API plugin for network_os %s' % self._network_os)
             else:
                 raise AnsibleConnectionFailure('unable to load API plugin for network_os %s' % self._network_os)
 
@@ -187,7 +184,7 @@ def __init__(self, play_context, new_stdin, *args, **kwargs):
                 'Unable to automatically determine host network os. Please '
                 'manually configure ansible_network_os value for this host'
             )
-        display.display('network_os is set to %s' % self._network_os, log_only=True)
+        self.queue_message('log', 'network_os is set to %s' % self._network_os)
 
     def update_play_context(self, pc_data):
         """Updates the play context information for the connection"""
@@ -199,16 +196,15 @@ def update_play_context(self, pc_data):
         play_context = PlayContext()
         play_context.deserialize(pc_data)
 
-        messages = ['updating play_context for connection']
+        self.queue_message('vvvv', 'updating play_context for connection')
         if self._play_context.become ^ play_context.become:
             self.set_become(play_context)
             if play_context.become is True:
-                messages.append('authorizing connection')
+                self.queue_message('vvvv', 'authorizing connection')
             else:
-                messages.append('deauthorizing connection')
+                self.queue_message('vvvv', 'deauthorizing connection')
 
         self._play_context = play_context
-        return messages
 
     def _connect(self):
         if not self.connected:
@@ -228,7 +224,7 @@ def close(self):
         '''
         # only close the connection if its connected.
         if self._connected:
-            display.vvvv("closing http(s) connection to device", host=self._play_context.remote_addr)
+            self.queue_message('vvvv', "closing http(s) connection to device")
             self.logout()
 
         super(Connection, self).close()

lib/ansible/plugins/connection/napalm.py

@@ -131,7 +131,6 @@
 
 from ansible.errors import AnsibleConnectionFailure, AnsibleError
 from ansible.plugins.connection import NetworkConnectionBase
-from ansible.utils.display import Display
 
 try:
     from napalm import get_network_driver
@@ -140,8 +139,6 @@
 except ImportError:
     HAS_NAPALM = False
 
-display = Display()
-
 
 class Connection(NetworkConnectionBase):
     """Napalm connections"""
@@ -168,7 +165,7 @@ def _connect(self):
                     'Unable to automatically determine host network os. Please '
                     'manually configure ansible_network_os value for this host'
                 )
-            display.display('network_os is set to %s' % self._network_os, log_only=True)
+            self.queue_message('log', 'network_os is set to %s' % self._network_os)
 
             try:
                 driver = get_network_driver(self._network_os)
@@ -186,7 +183,7 @@ def _connect(self):
             self.napalm.open()
 
             self._sub_plugin = {'type': 'external', 'name': 'napalm', 'obj': self.napalm}
-            display.vvvv('created napalm device for network_os %s' % self._network_os, host=host)
+            self.queue_message('vvvv', 'created napalm device for network_os %s' % self._network_os)
             self._connected = True
 
     def close(self):

lib/ansible/plugins/connection/netconf.py

@@ -181,7 +181,6 @@
 from ansible.module_utils.parsing.convert_bool import BOOLEANS_TRUE, BOOLEANS_FALSE
 from ansible.plugins.loader import netconf_loader
 from ansible.plugins.connection import NetworkConnectionBase
-from ansible.utils.display import Display
 
 try:
     from ncclient import manager
@@ -192,8 +191,6 @@
 except ImportError:
     HAS_NCCLIENT = False
 
-display = Display()
-
 logging.getLogger('ncclient').setLevel(logging.INFO)
 
 NETWORK_OS_DEVICE_PARAM_MAP = {
@@ -219,12 +216,12 @@ def __init__(self, play_context, new_stdin, *args, **kwargs):
         netconf = netconf_loader.get(self._network_os, self)
         if netconf:
             self._sub_plugin = {'type': 'netconf', 'name': self._network_os, 'obj': netconf}
-            display.display('loaded netconf plugin for network_os %s' % self._network_os, log_only=True)
+            self.queue_message('log', 'loaded netconf plugin for network_os %s' % self._network_os)
         else:
             netconf = netconf_loader.get("default", self)
             self._sub_plugin = {'type': 'netconf', 'name': 'default', 'obj': netconf}
-            display.display('unable to load netconf plugin for network_os %s, falling back to default plugin' % self._network_os)
-        display.display('network_os is set to %s' % self._network_os, log_only=True)
+            self.queue_message('display', 'unable to load netconf plugin for network_os %s, falling back to default plugin' % self._network_os)
+        self.queue_message('log', 'network_os is set to %s' % self._network_os)
 
         self._manager = None
         self.key_filename = None
@@ -259,7 +256,7 @@ def _connect(self):
                 'Please run pip install ncclient'
             )
 
-        display.display('ssh connection done, starting ncclient', log_only=True)
+        self.queue_message('log', 'ssh connection done, starting ncclient')
 
         allow_agent = True
         if self._play_context.password is not None:
@@ -274,7 +271,7 @@ def _connect(self):
             for cls in netconf_loader.all(class_only=True):
                 network_os = cls.guess_network_os(self)
                 if network_os:
-                    display.display('discovered network_os %s' % network_os, log_only=True)
+                    self.queue_message('log', 'discovered network_os %s' % network_os)
                     self._network_os = network_os
 
         device_params = {'name': NETWORK_OS_DEVICE_PARAM_MAP.get(self._network_os) or self._network_os}
@@ -307,7 +304,7 @@ def _connect(self):
         if not self._manager.connected:
             return 1, b'', b'not connected'
 
-        display.display('ncclient manager object created successfully', log_only=True)
+        self.queue_message('log', 'ncclient manager object created successfully')
 
         self._connected = True