[Docs]Auto generate pages for pulsar-admin (apache#5940)

### Motivation

After implementing this tool apache#5738, we need to use this tool to automatically generate the documentation of pulsar-admin.

### Modifications

* Add a script to automatically generate document pages of pulsar-admin
* Add em tag for markdown file of pulsar-admin

### Verifying this change

![image](https://user-images.githubusercontent.com/1907867/71462733-40707400-27ef-11ea-8dae-dce70783f22f.png)

.gitignore

@@ -77,6 +77,9 @@ docker.debug-info
 
 **/website/i18n/*
 **/website/translated_docs*
+**/website/brodocs/index.html
+**/website/brodocs/navData.js
+**/website/brodocs/documents/*.md
 
 # Avro
 examples/flink/src/main/java/org/apache/flink/avro/generated

pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java

@@ -105,7 +105,7 @@ private void generateDocument(StringBuilder sb, String module, JCommander obj) {
                 sb.append("* `").append(s).append("`\n");
             }
             cmdObj.jcommander.getCommands().forEach((subK, subV) -> {
-                sb.append("\n\n## ").append(subK).append("\n\n");
+                sb.append("\n\n## <em>").append(subK).append("</em>\n\n");
                 sb.append(cmdObj.jcommander.getCommandDescription(subK)).append("\n\n");
                 sb.append("### Usage\n\n");
                 sb.append("------------\n\n\n");

site2/tools/generate-api-docs.sh

@@ -39,3 +39,6 @@ cd $SCRIPT_DIR
 ./javadoc-gen.sh
 
 ./python-doc-gen.sh
+
+# Generate document for command line tools.
+./pulsar-admin-doc-gen.sh

site2/tools/pulsar-admin-doc-gen.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+ROOT_DIR=$(git rev-parse --show-toplevel)
+VERSION=`${ROOT_DIR}/src/get-project-version.py`
+DEST_DIR=$ROOT_DIR/generated-site
+
+mkdir -p $DEST_DIR/tools/pulsar-admin/${VERSION}
+mkdir -p $DEST_DIR/tools/pulsar-admin/${VERSION}/node_modules
+
+$ROOT_DIR/bin/pulsar-admin documents generate broker-stats > $ROOT_DIR/site2/website/brodocs/documents/broker-stats.md
+$ROOT_DIR/bin/pulsar-admin documents generate brokers > $ROOT_DIR/site2/website/brodocs/documents/brokers.md
+$ROOT_DIR/bin/pulsar-admin documents generate clusters > $ROOT_DIR/site2/website/brodocs/documents/clusters.md
+$ROOT_DIR/bin/pulsar-admin documents generate functions > $ROOT_DIR/site2/website/brodocs/documents/functions.md
+$ROOT_DIR/bin/pulsar-admin documents generate namespaces > $ROOT_DIR/site2/website/brodocs/documents/namespaces.md
+$ROOT_DIR/bin/pulsar-admin documents generate ns-isolation-policy > $ROOT_DIR/site2/website/brodocs/documents/ns-isolation-policy.md
+$ROOT_DIR/bin/pulsar-admin documents generate sources > $ROOT_DIR/site2/website/brodocs/documents/sources.md
+$ROOT_DIR/bin/pulsar-admin documents generate sinks > $ROOT_DIR/site2/website/brodocs/documents/sinks.md
+$ROOT_DIR/bin/pulsar-admin documents generate topics > $ROOT_DIR/site2/website/brodocs/documents/topics.md
+$ROOT_DIR/bin/pulsar-admin documents generate tenants > $ROOT_DIR/site2/website/brodocs/documents/tenants.md
+$ROOT_DIR/bin/pulsar-admin documents generate resource-quotas > $ROOT_DIR/site2/website/brodocs/documents/resource-quotas.md
+$ROOT_DIR/bin/pulsar-admin documents generate schemas > $ROOT_DIR/site2/website/brodocs/documents/schemas.md
+
+cd $ROOT_DIR/site2/website/brodocs
+node brodoc.js
+
+cp index.html $DEST_DIR/tools/pulsar-admin/${VERSION}/
+cp navData.js stylesheet.css $DEST_DIR/tools/pulsar-admin/${VERSION}/
+cp scroll.js tabvisibility.js $DEST_DIR/tools/pulsar-admin/${VERSION}/
+cp favicon.ico $DEST_DIR/tools/pulsar-admin/${VERSION}/
+mkdir -p $DEST_DIR/tools/pulsar-admin/${VERSION}/node_modules/bootstrap/dist/css
+cp -r $ROOT_DIR/site2/website/node_modules/bootstrap/dist/css/bootstrap.min.css $DEST_DIR/tools/pulsar-admin/${VERSION}/node_modules/bootstrap/dist/css
+mkdir -p $DEST_DIR/tools/pulsar-admin/${VERSION}/node_modules/font-awesome/css
+cp -r $ROOT_DIR/site2/website/node_modules/font-awesome/css/font-awesome.min.css $DEST_DIR/tools/pulsar-admin/${VERSION}/node_modules/font-awesome/css
+mkdir -p $DEST_DIR/tools/pulsar-admin/${VERSION}/node_modules/highlight.js/styles
+cp -r $ROOT_DIR/site2/website/node_modules/highlight.js/styles/default.css $DEST_DIR/tools/pulsar-admin/${VERSION}/node_modules/highlight.js/styles
+mkdir -p $DEST_DIR/tools/pulsar-admin/${VERSION}/node_modules/jquery/dist
+cp -r $ROOT_DIR/site2/website/node_modules/jquery/dist/jquery.min.js $DEST_DIR/tools/pulsar-admin/${VERSION}/node_modules/jquery/dist/
+mkdir -p $DEST_DIR/tools/pulsar-admin/${VERSION}/node_modules/jquery.scrollto
+cp -r $ROOT_DIR/site2/website/node_modules/jquery.scrollto/jquery.scrollTo.min.js $DEST_DIR/tools/pulsar-admin/${VERSION}/node_modules/jquery.scrollto
+
+

site2/website/brodocs/actions.js

@@ -0,0 +1,58 @@
+// https://jsfiddle.net/upqwhou2/
+
+$(document).ready(function() {
+    var navigationLinks = $('#sidebar-wrapper > ul li a');
+    var navigationSections = $('#sidebar-wrapper > ul > ul');
+    var sectionIdTonavigationLink = {};
+    var sections = $('#page-content-wrapper').find('h1, h2').map(function(index, node) {
+        if (node.id) {
+            sectionIdTonavigationLink[node.id] = $('#sidebar-wrapper > ul li a[href="#' + node.id + '"]');
+            return node;
+        }
+    });
+    var sectionIdToNavContainerLink = {};
+    var topLevelSections = $('#page-content-wrapper').find('h1').map(function(index, node) {
+        if (node.id) {
+            sectionIdToNavContainerLink[node.id] = $('#sidebar-wrapper > ul > ul[id="' + node.id + '-nav' +'"]');
+            return node;
+        }
+    });
+
+    var firstLevelNavs = $('#sidebar-wrapper > li');
+    var secondLevelNavs = $('#sidebar-wrapper > ul > ul');
+    var secondLevelNavContents = $('#sidebar-wrapper > ul > ul > li');
+    var thirdLevelNavs = null; // TODO: When compile provides 3 level nav, implement
+
+    var sectionsReversed = $(sections.get().reverse());
+
+    function checkScroll(event) {
+        var scrollPosition = $(window).scrollTop();
+        var offset = 50;
+        scrollPosition += offset;
+        sections.each(function() {
+            var currentSection = $(this);
+            var sectionTop = $(this).offset().top;
+            var id = $(this).attr('id');
+            if (scrollPosition >= sectionTop) {
+                navigationLinks.removeClass('selected');
+                sectionIdTonavigationLink[id].addClass('selected');
+                var sectionNavContainer = sectionIdToNavContainerLink[id];
+                var sectionNavContainerDisplay;
+                if (sectionNavContainer) {
+                    sectionNavContainerDisplay = sectionNavContainer.css('display');
+                }
+                if (sectionNavContainer && sectionNavContainerDisplay === 'none') {
+                    navigationSections.toggle(false);
+                    sectionNavContainer.toggle(true);
+                }
+            }
+            if (($(this).offset().top < window.pageYOffset + 50) && $(this).offset().top + $(this).height() > window.pageYOffset) {
+                window.location.hash = id;
+            }
+        });
+    }
+    checkScroll();
+    $(window).on('scroll', function(event) {
+        checkScroll(event);
+    });
+});

site2/website/brodocs/brodoc.js

@@ -0,0 +1,214 @@
+const docFolder = './documents/';
+const fs = require('fs');
+const marked = require('marked');
+const highlight = require('highlight.js');
+const renderer = new marked.Renderer();
+const brodocDec = require('./markedDecorations.js');
+
+
+marked.setOptions({
+    renderer: renderer,
+    gfm: true,
+    tables: true,
+    breaks: false,
+    pedantic: false,
+    sanitize: false,
+    smartLists: true,
+    smartypants: false,
+    highlight: function (code, lang) {
+        return highlight.highlightAuto(code).value;
+    }
+});
+brodocDec.decorateMarked(renderer);
+
+var config = require('./manifest');
+var docs = config.docs;
+
+var files = [];
+var fileArray = [];
+docs.forEach(file => {
+    files.push(file.filename);
+    fileArray.push(file);
+});
+
+var bodyContent = '';
+var navIds = brodocDec.navIds;
+var codeTabs = brodocDec.codeTabs;
+
+
+// const lexer = new marked.Lexer();
+// lexer.rules.bdoc = /^(\/{4} )(\w+).*$/;
+
+var path = docFolder;
+var fIndex = 0;
+var rIndex = 0;
+var fileObj = {toc: [], content: [], tabs: []};
+fileArray.forEach((file, index) => {
+    fs.readFile(path + file.filename, 'utf8', (err, data) => {
+        rIndex++;
+        file.content = data;
+
+        if (rIndex >= files.length) {
+            // do the things
+            parseFileContent(fileArray);
+            var navData = generateNavItems(navIds);
+            var navContent = navData.content;
+            var navDataArray = navData.navDataArray;
+            var codeTabContent = generateCodeTabItems(codeTabs);
+            var bodyContent = flattenContent(parsedContentArray);
+            generateDoc(navContent, bodyContent, codeTabContent);
+            generateNavJson(navDataArray);
+        }
+    });
+});
+
+function flattenContent(content) {
+    var flattenedContent = content.reduce(function(accum, val) {
+        return accum + val;
+    });
+    return flattenedContent;
+}
+
+var parsedContentArray = [];
+function parseFileContent(files) {
+    files.forEach((file, index) => {
+        parsedContentArray[index] = parseDoc(file.content);
+    });
+}
+function parseDoc(doc) {
+    return marked(doc, { renderer: renderer });
+}
+
+function generateNavItems(navObjs) {
+    var reversedNavs = navObjs.reverse();
+    var currentNestArray = [];
+    var currentStrongArray = [];
+    var flattenedNest = '';
+    var nestedNavArray = []; // Array containing generated html menu items - is flattened into a string.
+    var navArrayInvert = []; // Deals with data layer of navigation;
+    var navSectionArray = [];
+    var navStrongSectionArray = [];
+    var navSectionArrayClone;
+    var flatNavArrayInvert = [];
+    reversedNavs.forEach(obj => {
+        flatNavArrayInvert.push(obj.id);
+        var strong = (obj.id.indexOf('-strong-') !== -1);
+        if (obj.level !== 1) {
+            if (strong && currentNestArray.length !== 0) {
+                flattenedNest = flattenContent(currentNestArray.reverse());
+                currentStrongArray.push(generateNestedNav(obj, flattenedNest));
+                currentNestArray.length = 0;
+
+                navSectionArrayClone = Object.assign([], navSectionArray);
+                navStrongSectionArray.push({section: obj.id, subsections: navSectionArrayClone});
+                navSectionArray.length = 0;
+            } else {
+                currentNestArray.push(generateNav(obj));
+                navSectionArray.push({section: obj.id});
+            }
+        } else if (obj.level === 1) {
+            if (currentStrongArray.length !== 0) {
+                currentNestArray.forEach(obj => {
+                    currentStrongArray.push(obj);
+                });
+                flattenedNest = flattenContent(currentStrongArray.reverse());
+            } else if (currentNestArray.length !== 0) {
+                flattenedNest = flattenContent(currentNestArray.reverse());
+            }
+            nestedNavArray.push(generateNestedNav(obj, flattenedNest));
+            currentNestArray.length = 0;
+            currentStrongArray.length = 0;
+            flattenedNest = '';
+
+            navSectionArray.forEach(obj => {
+                navStrongSectionArray.push(obj);
+            });
+            navSectionArrayClone = Object.assign([], navStrongSectionArray);
+            navStrongSectionArray.length = 0;
+            navArrayInvert.push({section: obj.id, subsections: navSectionArrayClone});
+            navSectionArray.length = 0;
+        }
+    });
+
+    var navContent = flattenContent(nestedNavArray.reverse());
+    return {content: navContent, navDataArray: {toc: navArrayInvert, flatToc: flatNavArrayInvert}};
+}
+
+function generateNav(obj) {
+    var classString = 'nav-level-' + obj.level;
+    var isStrong = obj.id.indexOf('-strong-') !== -1;
+    if (isStrong) {
+        classString += ' strong-nav';
+    }
+    if (obj.level === 2) {
+        return '<li class="' + classString + '">' + '<a href="#' + obj.id + '" class="nav-item">' + obj.text + '</a></li>';
+    }
+    return '<li class="' + classString + '">' + '<a href="#' + obj.id + '" class="nav-item">' + obj.text + '</a></li>';
+}
+
+function generateNestedNav(parent, nest) {
+    var nestContent = '';
+    if (nest.length > 0) {
+        nestContent = nest ? '<ul id="' + parent.id + '-nav" style="display: none;">' + nest + '</ul>' : '';
+    }
+    return '<ul>' + generateNav(parent) + nestContent + '</ul>';
+}
+
+function generateNavJson(data) {
+    var navJson = JSON.stringify(data);
+    navScript = `(function(){navData = ${navJson};})();`;
+    fs.writeFile('./navData.js', navScript, function(err) {
+        if (err) {
+            return console.log(err);
+        }
+        console.log("navData.js saved!");
+    });
+}
+
+function generateCodeTabItems(tabs) {
+    var codeTabList = '';
+    tabs.forEach(tab => {
+        codeTabList += generateCodeTab(tab);
+    });
+    return codeTabList;
+}
+
+function generateCodeTab(tab) {
+    return '<li class="code-tab" id="' + tab + '">' + tab + '</li>';
+}
+
+function generateDoc(navContent, bodyContent, codeTabContent) {
+    var doc = 
+`<!DOCTYPE html>
+<html>
+<head>
+<meta charset="UTF-8">
+<title>${config.title}</title>
+<link rel="shortcut icon" href="favicon.ico" type="image/vnd.microsoft.icon">
+<!-- Latest compiled and minified CSS -->
+<link rel="stylesheet" href="node_modules/bootstrap/dist/css/bootstrap.min.css">
+<link rel="stylesheet" href="node_modules/font-awesome/css/font-awesome.min.css" type="text/css">
+<link rel="stylesheet" href="node_modules/highlight.js/styles/default.css" type="text/css">
+<link rel="stylesheet" href="stylesheet.css" type="text/css">
+</head>
+<body>
+<div id="sidebar-wrapper" class="side-nav side-bar-nav">${navContent}<br/><div class="copyright">${config.copyright}</div></div>
+<div id="wrapper">
+<div id="code-tabs-wrapper" class="code-tabs"><ul class="code-tab-list">${codeTabContent}</ul></div>
+<div id="page-content-wrapper" class="body-content container-fluid">${bodyContent}</div>
+</div>
+<script src="node_modules/jquery/dist/jquery.min.js"></script>
+<script src="node_modules/jquery.scrollto/jquery.scrollTo.min.js"></script>
+<script src="navData.js"></script>
+<script src="scroll.js"></script>
+<!--<script src="actions.js"></script>-->
+<script src="tabvisibility.js"></script>
+</body>
+</html>`;
+    fs.writeFile('./index.html', doc, function (err) {
+        if (err) {
+            return console.log(err);
+        }
+        console.log("index.html saved!");
+    });
+}