Config continued (ansible#31024)

* included inventory and callback in new config

allow inventory to be configurable
updated connection options settings
also updated winrm to work with new configs
removed now obsolete set_host_overrides
added notes for future bcoca, current one is just punting, it's future's problem
updated docs per feedback
added remove group/host methods to inv data
moved fact cache from data to constructed
cleaner/better options
fix when vars are added
extended ignore list to config dicts
updated paramiko connection docs
removed options from base that paramiko already handles
left the look option as it is used by other plugin types
resolve delegation
updated cache doc options
fixed test_script
better fragment merge for options
fixed proxy command
restore ini for proxy
normalized options
moved pipelining to class
updates for host_key_checking
restructured mixins

* fix typo

bin/ansible-connection

@@ -65,6 +65,7 @@ class ConnectionProcess(object):
                 self.play_context.private_key_file = os.path.join(self.original_path, self.play_context.private_key_file)
 
             self.connection = connection_loader.get(self.play_context.connection, self.play_context, '/dev/null')
+            self.connection.set_options()
             self.connection._connect()
             self.srv.register(self.connection)
             messages.append('connection to remote device started successfully')
@@ -143,7 +144,7 @@ class ConnectionProcess(object):
             if self.connection:
                 self.connection.close()
 
-        except:
+        except Exception:
             pass
 
         finally:
@@ -271,7 +272,7 @@ def main():
                     wfd = os.fdopen(w, 'w')
                     process = ConnectionProcess(wfd, play_context, socket_path, original_path)
                     process.start()
-                except Exception as exc:
+                except Exception:
                     messages.append(traceback.format_exc())
                     rc = 1
 

lib/ansible/cli/doc.py

@@ -44,6 +44,9 @@ class DocCLI(CLI):
         provides a printout of their DOCUMENTATION strings,
         and it can create a short "snippet" which can be pasted into a playbook.  '''
 
+    # default ignore list for detailed views
+    IGNORE = ('module', 'docuri', 'version_added', 'short_description', 'now_date', 'plainexamples', 'returndocs')
+
     def __init__(self, args):
 
         super(DocCLI, self).__init__(args)
@@ -394,6 +397,10 @@ def add_fields(self, text, fields, limit, opt_indent):
             for config in ('env', 'ini', 'yaml', 'vars'):
                 if config in opt and opt[config]:
                     conf[config] = opt.pop(config)
+                    for ignore in self.IGNORE:
+                        for item in conf[config]:
+                            if ignore in item:
+                                del item[ignore]
 
             if conf:
                 text.append(self._dump_yaml({'set_via': conf}, opt_indent))
@@ -441,7 +448,7 @@ def get_metadata_block(doc):
 
     def get_man_text(self, doc):
 
-        IGNORE = frozenset(['module', 'docuri', 'version_added', 'short_description', 'now_date', 'plainexamples', 'returndocs', self.options.type])
+        self.IGNORE = self.IGNORE + (self.options.type,)
         opt_indent = "        "
         text = []
         pad = display.columns * 0.20
@@ -492,7 +499,7 @@ def get_man_text(self, doc):
 
         # Generic handler
         for k in sorted(doc):
-            if k in IGNORE or not doc[k]:
+            if k in self.IGNORE or not doc[k]:
                 continue
             if isinstance(doc[k], string_types):
                 text.append('%s: %s' % (k.upper(), textwrap.fill(CLI.tty_ify(doc[k]), limit - (len(k) + 2), subsequent_indent=opt_indent)))

lib/ansible/config/base.yml

@@ -1319,36 +1319,13 @@ PARAMIKO_HOST_KEY_AUTO_ADD:
   - {key: host_key_auto_add, section: paramiko_connection}
   type: boolean
 PARAMIKO_LOOK_FOR_KEYS:
-  # TODO: move to plugin
+  name: look for keys
   default: True
   description: 'TODO: write it'
   env: [{name: ANSIBLE_PARAMIKO_LOOK_FOR_KEYS}]
   ini:
   - {key: look_for_keys, section: paramiko_connection}
   type: boolean
-PARAMIKO_PROXY_COMMAND:
-  # TODO: move to plugin
-  default:
-  description: 'TODO: write it'
-  env: [{name: ANSIBLE_PARAMIKO_PROXY_COMMAND}]
-  ini:
-  - {key: proxy_command, section: paramiko_connection}
-PARAMIKO_PTY:
-  # TODO: move to plugin
-  default: True
-  description: 'TODO: write it'
-  env: [{name: ANSIBLE_PARAMIKO_PTY}]
-  ini:
-  - {key: pty, section: paramiko_connection}
-  type: boolean
-PARAMIKO_RECORD_HOST_KEYS:
-  # TODO: move to plugin
-  default: True
-  description: 'TODO: write it'
-  env: [{name: ANSIBLE_PARAMIKO_RECORD_HOST_KEYS}]
-  ini:
-  - {key: record_host_keys, section: paramiko_connection}
-  type: boolean
 PERSISTENT_CONTROL_PATH_DIR:
   name: Persistence socket path
   default: ~/.ansible/pc

lib/ansible/config/manager.py

@@ -128,7 +128,7 @@ def get_ini_config_value(p, entry):
     if p is not None:
         try:
             value = p.get(entry.get('section', 'defaults'), entry.get('key', ''), raw=True)
-        except:  # FIXME: actually report issues here
+        except Exception:  # FIXME: actually report issues here
             pass
     return value
 
@@ -224,15 +224,24 @@ def _find_yaml_config_files(self):
         ''' Load YAML Config Files in order, check merge flags, keep origin of settings'''
         pass
 
-    def get_plugin_options(self, plugin_type, name, variables=None):
+    def get_plugin_options(self, plugin_type, name, keys=None, variables=None):
 
         options = {}
         defs = self.get_configuration_definitions(plugin_type, name)
         for option in defs:
-            options[option] = self.get_config_value(option, plugin_type=plugin_type, plugin_name=name, variables=variables)
+            options[option] = self.get_config_value(option, plugin_type=plugin_type, plugin_name=name, keys=keys, variables=variables)
 
         return options
 
+    def get_plugin_vars(self, plugin_type, name):
+
+        pvars = []
+        for pdef in self.get_configuration_definitions(plugin_type, name).values():
+            if 'vars' in pdef and pdef['vars']:
+                for var_entry in pdef['vars']:
+                    pvars.append(var_entry['name'])
+        return pvars
+
     def get_configuration_definitions(self, plugin_type=None, name=None):
         ''' just list the possible settings, either base or for specific plugins or plugin '''
 
@@ -264,12 +273,12 @@ def _loop_entries(self, container, entry_list):
 
         return value, origin
 
-    def get_config_value(self, config, cfile=None, plugin_type=None, plugin_name=None, variables=None):
+    def get_config_value(self, config, cfile=None, plugin_type=None, plugin_name=None, keys=None, variables=None):
         ''' wrapper '''
-        value, _drop = self.get_config_value_and_origin(config, cfile=cfile, plugin_type=plugin_type, plugin_name=plugin_name, variables=variables)
+        value, _drop = self.get_config_value_and_origin(config, cfile=cfile, plugin_type=plugin_type, plugin_name=plugin_name, keys=keys, variables=variables)
         return value
 
-    def get_config_value_and_origin(self, config, cfile=None, plugin_type=None, plugin_name=None, variables=None):
+    def get_config_value_and_origin(self, config, cfile=None, plugin_type=None, plugin_name=None, keys=None, variables=None):
         ''' Given a config key figure out the actual value and report on the origin of the settings '''
 
         if cfile is None:
@@ -290,10 +299,15 @@ def get_config_value_and_origin(self, config, cfile=None, plugin_type=None, plug
 
         if config in defs:
             # Use 'variable overrides' if present, highest precedence, but only present when querying running play
-            if variables:
+            if variables and defs[config].get('vars'):
                 value, origin = self._loop_entries(variables, defs[config]['vars'])
                 origin = 'var: %s' % origin
 
+            # use playbook keywords if you have em
+            if value is None and keys:
+                value, origin = self._loop_entries(keys, defs[config]['keywords'])
+                origin = 'keyword: %s' % origin
+
             # env vars are next precedence
             if value is None and defs[config].get('env'):
                 value, origin = self._loop_entries(os.environ, defs[config]['env'])
@@ -319,13 +333,6 @@ def get_config_value_and_origin(self, config, cfile=None, plugin_type=None, plug
                         # FIXME: implement, also , break down key from defs (. notation???)
                         origin = cfile
 
-            '''
-            # for plugins, try using existing constants, this is for backwards compatiblity
-            if plugin_name and defs[config].get('constants'):
-                value, origin = self._loop_entries(self.data, defs[config]['constants'])
-                origin = 'constant: %s' % origin
-            '''
-
             # set default if we got here w/o a value
             if value is None:
                 value = defs[config].get('default')

lib/ansible/executor/task_executor.py

@@ -19,11 +19,11 @@
 from ansible.module_utils._text import to_text
 from ansible.playbook.conditional import Conditional
 from ansible.playbook.task import Task
-from ansible.plugins.connection import ConnectionBase
 from ansible.template import Templar
 from ansible.utils.listify import listify_lookup_plugin_terms
 from ansible.utils.unsafe_proxy import UnsafeProxy, wrap_var
 from ansible.vars.clean import namespace_facts, clean_facts
+from ansible.utils.vars import combine_vars
 
 try:
     from __main__ import display
@@ -480,18 +480,14 @@ def _execute(self, variables=None):
                 not getattr(self._connection, 'connected', False) or
                 self._play_context.remote_addr != self._connection._play_context.remote_addr):
             self._connection = self._get_connection(variables=variables, templar=templar)
-            if getattr(self._connection, '_socket_path'):
-                variables['ansible_socket'] = self._connection._socket_path
-            # only template the vars if the connection actually implements set_host_overrides
-            # NB: this is expensive, and should be removed once connection-specific vars are being handled by play_context
-            sho_impl = getattr(type(self._connection), 'set_host_overrides', None)
-            if sho_impl and sho_impl != ConnectionBase.set_host_overrides:
-                self._connection.set_host_overrides(self._host, variables, templar)
         else:
             # if connection is reused, its _play_context is no longer valid and needs
             # to be replaced with the one templated above, in case other data changed
             self._connection._play_context = self._play_context
 
+        self._set_connection_options(variables, templar)
+
+        # get handler
         self._handler = self._get_action_handler(connection=self._connection, templar=templar)
 
         # And filter out any fields which were set to default(omit), and got the omit token value
@@ -734,6 +730,7 @@ def _get_connection(self, variables, templar):
         if not connection:
             raise AnsibleError("the connection plugin '%s' was not found" % conn_type)
 
+        # FIXME: remove once all plugins pull all data from self._options
         self._play_context.set_options_from_plugin(connection)
 
         if any(((connection.supports_persistence and C.USE_PERSISTENT_CONNECTIONS), connection.force_persistence)):
@@ -745,6 +742,29 @@ def _get_connection(self, variables, templar):
 
         return connection
 
+    def _set_connection_options(self, variables, templar):
+
+        # create copy with delegation built in
+        final_vars = combine_vars(variables, variables.get('ansible_delegated_vars', dict()).get(self._task.delegate_to, dict()))
+
+        # grab list of usable vars for this plugin
+        option_vars = C.config.get_plugin_vars('connection', self._connection._load_name)
+
+        # create dict of 'templated vars'
+        options = {'_extras': {}}
+        for k in option_vars:
+            if k in final_vars:
+                options[k] = templar.template(final_vars[k])
+
+        # add extras if plugin supports them
+        if getattr(self._connection, 'allow_extras', False):
+            for k in final_vars:
+                if k.startswith('ansible_%s_' % self._connection._load_name) and k not in options:
+                    options['_extras'][k] = templar.template(final_vars[k])
+
+        # set options with 'templated vars' specific to this plugin
+        self._connection.set_options(var_options=options)
+
     def _get_action_handler(self, connection, templar):
         '''
         Returns the correct action plugin to handle the requestion task action

lib/ansible/executor/task_queue_manager.py

@@ -178,7 +178,7 @@ def load_callbacks(self):
             else:
                 self._stdout_callback = callback_loader.get(self._stdout_callback)
                 try:
-                    self._stdout_callback.set_options(C.config.get_plugin_options('callback', self._stdout_callback._load_name))
+                    self._stdout_callback.set_options()
                 except AttributeError:
                     display.deprecated("%s stdout callback, does not support setting 'options', it will work for now, "
                                        " but this will be required in the future and should be updated,"
@@ -207,7 +207,7 @@ def load_callbacks(self):
 
             callback_obj = callback_plugin()
             try:
-                callback_obj .set_options(C.config.get_plugin_options('callback', callback_plugin._load_name))
+                callback_obj.set_options()
             except AttributeError:
                     display.deprecated("%s callback, does not support setting 'options', it will work for now, "
                                        " but this will be required in the future and should be updated, "

lib/ansible/inventory/data.py

@@ -173,6 +173,17 @@ def add_group(self, group):
         else:
             display.debug("group %s already in inventory" % group)
 
+    def remove_group(self, group):
+
+        if group in self.groups:
+            del self.groups[group]
+            display.debug("Removed group %s from inventory" % group)
+            self._groups_dict_cache = {}
+
+        for host in self.hosts:
+            h = self.hosts[host]
+            h.remove_group(group)
+
     def add_host(self, host, group=None, port=None):
         ''' adds a host to inventory and possibly a group if not there already '''
 
@@ -209,6 +220,15 @@ def add_host(self, host, group=None, port=None):
             self._groups_dict_cache = {}
             display.debug("Added host %s to group %s" % (host, group))
 
+    def remove_host(self, host):
+
+        if host in self.hosts:
+            del self.hosts[host]
+
+        for group in self.groups:
+            g = self.groups[group]
+            g.remove_host(host)
+
     def set_variable(self, entity, varname, value):
         ''' sets a varible for an inventory object '''
 

lib/ansible/inventory/manager.py

@@ -183,6 +183,7 @@ def _setup_inventory_plugins(self):
         for name in C.INVENTORY_ENABLED:
             plugin = inventory_loader.get(name)
             if plugin:
+                plugin.set_options()
                 self._inventory_plugins.append(plugin)
             else:
                 display.warning('Failed to load inventory plugin, skipping %s' % name)
@@ -282,7 +283,8 @@ def parse_source(self, source, cache=False):
                     else:
                         for fail in failures:
                             display.warning(u'\n* Failed to parse %s with %s plugin: %s' % (to_text(fail['src']), fail['plugin'], to_text(fail['exc'])))
-                            display.vvv(to_text(fail['exc'].tb))
+                            if hasattr(fail['exc'], 'tb'):
+                                display.vvv(to_text(fail['exc'].tb))
         if not parsed:
             display.warning("Unable to parse %s as an inventory source" % to_text(source))
 

lib/ansible/plugins/__init__.py

@@ -47,6 +47,9 @@ def get_plugin_class(obj):
 
 class AnsiblePlugin(with_metaclass(ABCMeta, object)):
 
+    # allow extra passthrough parameters
+    allow_extras = False
+
     def __init__(self):
         self._options = {}
 
@@ -59,8 +62,28 @@ def get_option(self, option, hostvars=None):
     def set_option(self, option, value):
         self._options[option] = value
 
-    def set_options(self, options):
-        self._options = options
+    def set_options(self, task_keys=None, var_options=None, direct=None):
+        '''
+        Sets the _options attribute with the configuration/keyword information for this plugin
+
+        :arg task_keys: Dict with playbook keywords that affect this option
+        :arg var_options: Dict with either 'conneciton variables'
+        :arg direct: Dict with 'direct assignment'
+        '''
+
+        if not self._options:
+            # load config options if we have not done so already, if vars provided we should be mostly done
+            self._options = C.config.get_plugin_options(get_plugin_class(self), self._load_name, keys=task_keys, variables=var_options)
+
+        # they can be direct options overriding config
+        if direct:
+            for k in self._options:
+                if k in direct:
+                    self.set_option(k, direct[k])
+
+        # allow extras/wildcards from vars that are not directly consumed in configuration
+        if self.allow_extras and var_options and '_extras' in var_options:
+            self.set_option('_extras', var_options['_extras'])
 
     def _check_required(self):
         # FIXME: standarize required check based on config

lib/ansible/plugins/action/__init__.py

@@ -612,6 +612,21 @@ def _update_module_args(self, module_name, module_args, task_vars):
         # make sure all commands use the designated shell executable
         module_args['_ansible_shell_executable'] = self._play_context.executable
 
+    def _update_connection_options(self, options, variables=None):
+        ''' ensures connections have the appropriate information '''
+        update = {}
+
+        if getattr(self.connection, 'glob_option_vars', False):
+            # if the connection allows for it, pass any variables matching it.
+            if variables is not None:
+                for varname in variables:
+                    if varname.match('ansible_%s_' % self.connection._load_name):
+                        update[varname] = variables[varname]
+
+        # always override existing with options
+        update.update(options)
+        self.connection.set_options(update)
+
     def _execute_module(self, module_name=None, module_args=None, tmp=None, task_vars=None, persist_files=False, delete_remote_tmp=True, wrap_async=False):
         '''
         Transfer and run a module along with its arguments.