Generate plugin rst (ansible#28901)

Generate rst docs for plugins Based on rst generated for modules. But generated plugin docs go into docs/docsite/rst/plugins/$PLUGIN_TYPE/plugin_name.rst ( docs/docsite/rst/plugins/connection/ssh.py for ex) * move plugins docs to rst/*_plugins/ subdirs for namespace * Only gen support pages for modules for now. * Add generated plugin docs to gitignore* add list_*_plugins templates * support MODULES/PLUGINS filters for make htmldocs Add a 'PLUGINS=ssh' filter env var like MODULES to filter plugins to build docs for. * fixup 'historical' version_added, skip plugins/loader.py * Fix plugins_by_support ref link to new plugins/*/ location * use :ref: for common_return_values, allow empty version_added * warnings on missing doc info * add a prefix to _random_choice It was colliding with the target for random_choice plugin
jabowles · Sep 19, 2017 · da15cf1 · da15cf1
1 parent 76aaaf1
commit da15cf1
Show file tree

Hide file tree

Showing 13 changed files with 264 additions and 54 deletions.
diff --git a/.gitignore b/.gitignore
@@ -33,17 +33,20 @@ docs/docsite/rst/core_maintained.rst
 docs/docsite/rst/list_of_*.rst
 docs/docsite/rst/*_module.rst
 docs/docsite/rst/modules_by_category.rst
+docs/docsite/rst/plugins_by_category.rst
 docs/docsite/rst/network_maintained.rst
 docs/docsite/rst/partner_maintained.rst
 docs/docsite/rst/playbook_keywords.rst
 docs/docsite/rst/playbooks_directives.rst
+docs/docsite/rst/plugins/
 docs/docsite/*.html
 docs/docsite/_static/*.gif
 docs/docsite/_static/*.png
 docs/docsite/_static/websupport.js
 docs/docsite/searchindex.js
 docs/docsite/htmlout
 docs/docsite/_build
+docs/docsite/rst_warnings
 docs/api/rst/
 docs/api/_build/
 # deb building stuff...

diff --git a/docs/bin/plugin_formatter.py b/docs/bin/plugin_formatter.py
@@ -26,6 +26,7 @@
 import glob
 import optparse
 import os
+import pprint
 import re
 import sys
 import warnings
@@ -41,7 +42,7 @@ def html_escape(text, quote=True):
 
 import yaml
 from jinja2 import Environment, FileSystemLoader
-from six import iteritems
+from six import iteritems, string_types
 
 from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_bytes
@@ -120,6 +121,9 @@ def write_data(text, output_dir, outputname, module=None):
     if output_dir is not None:
         if module:
             outputname = outputname % module
+
+        if not os.path.exists(output_dir):
+            os.mkdir(output_dir)
         fname = os.path.join(output_dir, outputname)
         fname = fname.replace(".py", "")
         with open(fname, 'wb') as f:
@@ -152,6 +156,7 @@ def get_module_info(module_dir, limit_to_modules=None, verbose=False):
             least one key, '_modules' which contains a list of module names in
             that category.  Any other keys in the dict are subcategories with
             the same structure.
+
     '''
 
     categories = dict()
@@ -207,15 +212,21 @@ def get_module_info(module_dir, limit_to_modules=None, verbose=False):
         # Start at the second directory because we don't want the "vendor"
         mod_path_only = os.path.dirname(module_path[len(module_dir):])
 
+        module_categories = []
         # build up the categories that this module belongs to
         for new_cat in mod_path_only.split('/')[1:]:
             if new_cat not in category:
                 category[new_cat] = dict()
                 category[new_cat]['_modules'] = []
+            module_categories.append(new_cat)
             category = category[new_cat]
 
         category['_modules'].append(module)
 
+        # the category we will use in links (so list_of_all_plugins can point to plugins/action_plugins/*'
+        if module_categories:
+            primary_category = module_categories[0]
+
         # use ansible core library to parse out doc metadata YAML and plaintext examples
         doc, examples, returndocs, metadata = plugin_docs.get_docstring(module_path, verbose=verbose)
 
@@ -227,6 +238,8 @@ def get_module_info(module_dir, limit_to_modules=None, verbose=False):
                                'doc': doc,
                                'examples': examples,
                                'returndocs': returndocs,
+                               'categories': module_categories,
+                               'primary_category': primary_category,
                                }
 
     # keep module tests out of becoming module docs
@@ -247,6 +260,7 @@ def generate_parser():
 
     p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number")
     p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path")
+    p.add_option("-P", "--plugin-type", action="store", dest="plugin_type", default='modules', help="The type of plugin (plugins, modules)")
     p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="hacking/templates", help="directory containing Jinja2 templates")
     p.add_option("-t", "--type", action='store', dest='type', choices=['rst'], default='rst', help="Document type")
     p.add_option("-v", "--verbose", action='store_true', default=False, help="Verbose")
@@ -258,7 +272,7 @@ def generate_parser():
     return p
 
 
-def jinja2_environment(template_dir, typ):
+def jinja2_environment(template_dir, typ, plugin_type):
 
     env = Environment(loader=FileSystemLoader(template_dir),
                       variable_start_string="@{",
@@ -273,14 +287,13 @@ def jinja2_environment(template_dir, typ):
         env.filters['fmt'] = rst_fmt
         env.filters['xline'] = rst_xline
         templates['plugin'] = env.get_template('plugin.rst.j2')
-        templates['category_list'] = env.get_template('modules_by_category.rst.j2')
-        templates['support_list'] = env.get_template('modules_by_support.rst.j2')
-        templates['list_of_CATEGORY_modules'] = env.get_template('list_of_CATEGORY_modules.rst.j2')
-        outputname = "%s_module.rst"
+        templates['category_list'] = env.get_template('%s_by_category.rst.j2' % plugin_type)
+        templates['support_list'] = env.get_template('%s_by_support.rst.j2' % plugin_type)
+        templates['list_of_CATEGORY_modules'] = env.get_template('list_of_CATEGORY_%s.rst.j2' % plugin_type)
     else:
         raise Exception("unknown module format type: %s" % typ)
 
-    return templates, outputname
+    return templates
 
 
 def too_old(added):
@@ -296,24 +309,55 @@ def too_old(added):
     return added_float < TO_OLD_TO_BE_NOTABLE
 
 
-def process_modules(module_map, templates, outputname, output_dir, ansible_version):
+def process_modules(module_map, templates, outputname,
+                    output_dir, ansible_version, plugin_type):
     for module in module_map:
-        print("rendering: %s" % module)
+        # print("rendering: %s" % module)
+
+        # pprint.pprint(('process_modules module:', module))
 
         fname = module_map[module]['path']
 
+        # pprint.pprint(('process_modules module_info: ', module_map[module]))
+
+        module_categories = module_map[module].get('categories', [])
+
         # crash if module is missing documentation and not explicitly hidden from docs index
         if module_map[module]['doc'] is None:
-            sys.exit("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
+            print("%s: ERROR: MODULE MISSING DOCUMENTATION" % (fname,))
+            _doc = {'module': module,
+                    'version_added': '2.4',
+                    'filename': fname}
+            module_map[module]['doc'] = _doc
+            # continue
 
         # Going to reference this heavily so make a short name to reference it by
         doc = module_map[module]['doc']
 
+        # pprint.pprint(('process_modules doc: ', doc))
+
+        # add some defaults for plugins that dont have most of the info
+        doc['module'] = doc.get('module', module)
+        doc['version_added'] = doc.get('version_added', 'historical')
+
+        doc['plugin_type'] = plugin_type
+
         if module_map[module]['deprecated'] and 'deprecated' not in doc:
-            sys.exit("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
+            print("%s: WARNING: MODULE MISSING DEPRECATION DOCUMENTATION: %s" % (fname, 'deprecated'))
+
+        required_fields = ('short_description',)
+        for field in required_fields:
+            if field not in doc:
+                print("%s: WARNING: MODULE MISSING field '%s'" % (fname, field))
+
+        not_nullable_fields = ('short_description',)
+        for field in not_nullable_fields:
+            if field in doc and doc[field] in (None, ''):
+                print("%s: WARNING: MODULE field '%s' DOCUMENTATION is null/empty value=%s" % (fname, field, doc[field]))
 
         if 'version_added' not in doc:
-            sys.exit("*** ERROR: missing version_added in: %s ***\n" % module)
+            pprint.pprint(doc)
+            # sys.exit("*** ERROR: missing version_added in: %s ***\n" % module)
 
         #
         # The present template gets everything from doc so we spend most of this
@@ -366,47 +410,78 @@ def process_modules(module_map, templates, outputname, output_dir, ansible_versi
         doc['docuri'] = doc['module'].replace('_', '-')
         doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d')
         doc['ansible_version'] = ansible_version
-        doc['plainexamples'] = module_map[module]['examples']  # plain text
+
+        # check the 'deprecated' field in doc. We expect a dict potentially with 'why', 'version', and 'alternative' fields
+        # examples = module_map[module]['examples']
+        # print('\n\n%s: type of examples: %s\n' % (module, type(examples)))
+        # if examples and not isinstance(examples, (str, unicode, list)):
+        #    raise TypeError('module %s examples is wrong type (%s): %s' % (module, type(examples), examples))
+
+        # use 'examples' for 'plainexamples' if 'examples' is a string
+        if isinstance(module_map[module]['examples'], string_types):
+            doc['plainexamples'] = module_map[module]['examples']  # plain text
+        else:
+            doc['plainexamples'] = ''
+
         doc['metadata'] = module_map[module]['metadata']
 
+        # pprint.pprint(module_map[module]
         if module_map[module]['returndocs']:
             try:
                 doc['returndocs'] = yaml.safe_load(module_map[module]['returndocs'])
-            except:
-                print("could not load yaml: %s" % module_map[module]['returndocs'])
-                raise
+            except Exception as e:
+                print("%s:%s:yaml error:%s:returndocs=%s" % (fname, module, e, module_map[module]['returndocs']))
+                doc['returndocs'] = None
         else:
             doc['returndocs'] = None
 
+        doc['author'] = doc.get('author', ['UNKNOWN'])
+        if isinstance(doc['author'], string_types):
+            doc['author'] = [doc['author']]
+
+        # print('about to template')
+        # pprint.pprint(doc)
         text = templates['plugin'].render(doc)
 
-        write_data(text, output_dir, outputname, module)
+        # plugins get namespace dirs but modules do not
+        if plugin_type == 'plugins':
+            for module_category in module_categories:
+                category_output_dir = os.path.join(output_dir, 'plugins', '%s' % module_category)
+                write_data(text, category_output_dir, outputname, module)
+        else:
+                write_data(text, output_dir, outputname, module)
 
 
-def process_categories(mod_info, categories, templates, output_dir, output_name):
+def process_categories(mod_info, categories, templates,
+                       output_dir, output_name, plugin_type):
     for category in sorted(categories.keys()):
+        if (plugin_type, category) == ('plugins', ''):
+            print('skipping unknown cat: %s' % category)
+            continue
         module_map = categories[category]
         category_filename = output_name % category
 
         print("*** recording category %s in %s ***" % (category, category_filename))
 
         # start a new category file
 
-        category = category.replace("_", " ")
-        category = category.title()
+        category_name = category.replace("_", " ")
+        category_title = category_name.title()
 
         subcategories = dict((k, v) for k, v in module_map.items() if k != '_modules')
-        template_data = {'title': category,
+        template_data = {'title': category_title,
+                         'category_name': category_name,
                          'category': module_map,
                          'subcategories': subcategories,
                          'module_info': mod_info,
+                         'plugin_type': plugin_type
                          }
 
         text = templates['list_of_CATEGORY_modules'].render(template_data)
         write_data(text, output_dir, category_filename)
 
 
-def process_support_levels(mod_info, templates, output_dir):
+def process_support_levels(mod_info, templates, output_dir, plugin_type):
     supported_by = {'Ansible Core Team': {'slug': 'core_supported',
                                           'modules': [],
                                           'output': 'core_maintained.rst',
@@ -450,8 +525,14 @@ def process_support_levels(mod_info, templates, output_dir):
                                           """},
                     }
 
+    # only gen support pages for modules for now, need to split and namespace templates and generated docs
+    if plugin_type == 'plugins':
+        return
     # Separate the modules by support_level
     for module, info in mod_info.items():
+        if not info.get('metadata', None):
+            print('no metadata for %s' % module)
+            continue
         if info['metadata']['supported_by'] == 'core':
             supported_by['Ansible Core Team']['modules'].append(module)
         elif info['metadata']['supported_by'] == 'network':
@@ -492,7 +573,15 @@ def main():
     (options, args) = p.parse_args()
     validate_options(options)
 
-    templates, outputname = jinja2_environment(options.template_dir, options.type)
+    plugin_type = options.plugin_type
+    templates = jinja2_environment(options.template_dir, options.type,
+                                   plugin_type)
+
+    # for plugins, just use the short name 'ssh.rst' vs 'ssh_module.rst'
+    outputname = '%s.rst'
+    # trim trailing s off of plugin_type for plugin_type=='modules'. ie 'copy_module.rst'
+    if plugin_type == 'modules':
+        outputname = '%s_' + '%s.rst' % plugin_type[:-1]
 
     # Convert passed-in limit_to_modules to None or list of modules.
     if options.limit_to_modules is not None:
@@ -503,23 +592,36 @@ def main():
 
     categories['all'] = {'_modules': mod_info.keys()}
 
+    # pprint.pprint(categories)
+    # pprint.pprint(mod_info)
+    # pprint.pprint(dict(mod_info))
     # Transform the data
     if options.type == 'rst':
-        for record in mod_info.values():
-            record['doc']['short_description'] = rst_ify(record['doc']['short_description'])
+        for key, record in mod_info.items():
+            # pprint.pprint(('record', record))
+            if record.get('doc', None):
+                short_desc = record['doc']['short_description']
+                if short_desc is None:
+                    print('WARNING: short_description for %s is None' % key)
+                    short_desc = ''
+                record['doc']['short_description'] = rst_ify(short_desc)
 
     # Write master category list
     category_list_text = templates['category_list'].render(categories=sorted(categories.keys()))
-    write_data(category_list_text, options.output_dir, 'modules_by_category.rst')
+    category_index_name = '%s_by_category.rst' % plugin_type
+    write_data(category_list_text, options.output_dir, category_index_name)
 
     # Render all the individual module pages
-    process_modules(mod_info, templates, outputname, options.output_dir, options.ansible_version)
+    process_modules(mod_info, templates, outputname,
+                    options.output_dir, options.ansible_version, plugin_type)
 
     # Render all the categories for modules
-    process_categories(mod_info, categories, templates, options.output_dir, "list_of_%s_modules.rst")
+    category_list_name_template = 'list_of_%s_' + '%s.rst' % plugin_type
+    process_categories(mod_info, categories, templates, options.output_dir,
+                       category_list_name_template, plugin_type)
 
     # Render all the categories for modules
-    process_support_levels(mod_info, templates, options.output_dir)
+    process_support_levels(mod_info, templates, options.output_dir, plugin_type)
 
 
 if __name__ == '__main__':

diff --git a/docs/docsite/Makefile b/docs/docsite/Makefile
@@ -11,6 +11,16 @@ else
 CPUS ?= $(shell nproc)
 endif
 
+MODULE_ARGS=
+ifdef MODULES
+	MODULE_ARGS = -l $(MODULES)
+endif
+
+PLUGIN_ARGS=
+ifdef PLUGINS
+	PLUGIN_ARGS = -l $(PLUGINS)
+endif
+
 assertrst:
 ifndef rst
 	$(error specify document or pattern with rst=somefile.rst)
@@ -20,7 +30,7 @@ all: docs
 
 docs: clean htmldocs
 
-generate_rst: testing keywords modules staticmin cli config
+generate_rst: testing keywords modules plugins staticmin cli config
 
 htmldocs: generate_rst
 	CPUS=$(CPUS) $(MAKE) -f Makefile.sphinx html
@@ -48,6 +58,8 @@ clean:
 	-rm rst/list_of_*.rst
 	-rm rst/*_by_category.rst
 	-rm rst/*_module.rst
+	-rm rst/*_plugin.rst
+	-rm -rf rst/plugins/*
 	-rm rst/*_maintained.rst
 	-rm rst/playbooks_directives.rst
 	-rm rst/playbooks_keywords.rst
@@ -67,12 +79,10 @@ config:
 	PYTHONPATH=../../lib $(CONFIG_DUMPER) --template-file=../templates/config.rst.j2 --output-dir=rst/ -d ../../lib/ansible/config/base.yml
 
 modules: $(FORMATTER) ../templates/plugin.rst.j2
-# Limit building of module documentation if requested.
-ifdef MODULES
-	PYTHONPATH=../../lib $(FORMATTER) -t rst --template-dir=../templates --module-dir=../../lib/ansible/modules -o rst/ -l $(MODULES)
-else
-	PYTHONPATH=../../lib $(FORMATTER) -t rst --template-dir=../templates --module-dir=../../lib/ansible/modules -o rst/
-endif
+	PYTHONPATH=../../lib $(FORMATTER) -t rst --template-dir=../templates --module-dir=../../lib/ansible/modules -o rst/ $(MODULE_ARGS)
+
+plugins: $(FORMATTER) ../templates/plugin.rst.j2
+	PYTHONPATH=../../lib $(FORMATTER) -t rst --plugin-type plugins --template-dir=../templates --module-dir=../../lib/ansible/plugins -o rst/ $(PLUGIN_ARGS)
 
 testing:
 	$(TESTING_FORMATTER)