Enable generation of swagger definitions for functions/sources/sinks (a…

…pache#3871)

* Enable generation of swagger definitions for functions/sources/sinks

* Added more changes to make it backwards compatible

* Add logic to server functions rest api

* Deleted swagger file since its supposed to be generated.
Also added some missing python packages for building

* Added back deleted file

* Fix comments

* Seperated out sources and sinks

* More changes to collapse all rest apis into one dropdown

pulsar-broker/pom.xml

@@ -365,7 +365,7 @@
               <apiSources>
                 <apiSource>
                   <springmvc>false</springmvc>
-                  <locations>org.apache.pulsar.broker.admin</locations>
+                  <locations>org.apache.pulsar.broker.admin.v2</locations>
                   <schemes>http,https</schemes>
                   <basePath>/admin/v2</basePath>
                   <info>
@@ -378,6 +378,58 @@
                     </license>
                   </info>
                   <swaggerDirectory>${basedir}/target/docs</swaggerDirectory>
+                  <swaggerFileName>swagger</swaggerFileName>
+                </apiSource>
+                <apiSource>
+                  <springmvc>false</springmvc>
+                  <locations>org.apache.pulsar.broker.admin.v3.Functions</locations>
+                  <schemes>http,https</schemes>
+                  <basePath>/admin/v3</basePath>
+                  <info>
+                    <title>Pulsar Functions REST API</title>
+                    <version>v3</version>
+                    <description>This provides the REST API for Pulsar Functions operations</description>
+                    <license>
+                      <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
+                      <name>Apache 2.0</name>
+                    </license>
+                  </info>
+                  <swaggerDirectory>${basedir}/target/docs</swaggerDirectory>
+                  <swaggerFileName>swaggerfunctions</swaggerFileName>
+                </apiSource>
+                <apiSource>
+                  <springmvc>false</springmvc>
+                  <locations>org.apache.pulsar.broker.admin.v3.Source</locations>
+                  <schemes>http,https</schemes>
+                  <basePath>/admin/v3</basePath>
+                  <info>
+                    <title>Pulsar Source REST API</title>
+                    <version>v3</version>
+                    <description>This provides the REST API for Pulsar Source operations</description>
+                    <license>
+                      <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
+                      <name>Apache 2.0</name>
+                    </license>
+                  </info>
+                  <swaggerDirectory>${basedir}/target/docs</swaggerDirectory>
+                  <swaggerFileName>swaggersource</swaggerFileName>
+                </apiSource>
+                <apiSource>
+                  <springmvc>false</springmvc>
+                  <locations>org.apache.pulsar.broker.admin.v3.Sink</locations>
+                  <schemes>http,https</schemes>
+                  <basePath>/admin/v3</basePath>
+                  <info>
+                    <title>Pulsar Sink REST API</title>
+                    <version>v3</version>
+                    <description>This provides the REST API for Pulsar Sink operations</description>
+                    <license>
+                      <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
+                      <name>Apache 2.0</name>
+                    </license>
+                  </info>
+                  <swaggerDirectory>${basedir}/target/docs</swaggerDirectory>
+                  <swaggerFileName>swaggersink</swaggerFileName>
                 </apiSource>
               </apiSources>
             </configuration>

pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java

@@ -326,4 +326,4 @@ public Response downloadFunction(final @QueryParam("path") String path) {
     public List<ConnectorDefinition> getConnectorsList() throws IOException {
         return functions.getListOfConnectors();
     }
-}
+}

pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Functions.java

@@ -27,7 +27,7 @@
 import javax.ws.rs.core.MediaType;
 
 @Path("/functions")
-@Api(value = "/functions", description = "Functions admin apis", tags = "functions", hidden = true)
+@Api(value = "/functions", description = "Functions admin apis", tags = "functions")
 @Produces(MediaType.APPLICATION_JSON)
 @Consumes(MediaType.APPLICATION_JSON)
 public class Functions extends FunctionsBase {

pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Sink.java

@@ -27,7 +27,7 @@
 import javax.ws.rs.core.MediaType;
 
 @Path("/sink")
-@Api(value = "/sink", description = "Sink admin apis", tags = "sink", hidden = true)
+@Api(value = "/sink", description = "Sink admin apis", tags = "sink")
 @Produces(MediaType.APPLICATION_JSON)
 @Consumes(MediaType.APPLICATION_JSON)
 public class Sink extends SinkBase {

pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Source.java

@@ -27,7 +27,7 @@
 import javax.ws.rs.core.MediaType;
 
 @Path("/source")
-@Api(value = "/source", description = "Source admin apis", tags = "source", hidden = true)
+@Api(value = "/source", description = "Source admin apis", tags = "source")
 @Produces(MediaType.APPLICATION_JSON)
 @Consumes(MediaType.APPLICATION_JSON)
 public class Source extends SourceBase {

pulsar-client-cpp/docker/Dockerfile

@@ -150,6 +150,9 @@ RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.3.7/zstd-1.
     rm -rf /zstd-1.3.7 /zstd-1.3.7.tar.gz
 
 RUN pip install twine
+RUN pip install fastavro
+RUN pip install six
+RUN pip install enum34
 
 
 ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include

site2/tools/python-doc-gen.sh

@@ -24,7 +24,14 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
 
 # Make sure the Python client lib is installed
 # so that Pdoc can import the module
-pip install pulsar-client
+find $ROOT_DIR -name CMakeCache.txt | xargs rm -f
+find $ROOT_DIR -name CMakeFiles | xargs rm -rf
+cd $ROOT_DIR/pulsar-client-cpp
+cmake . 
+make -j8 _pulsar
+pip install enum34
+pip install six
+pip install fastavro
 
 DESTINATION=$ROOT_DIR/generated-site/api/python
 rm -fr $DESTINATION/{index.html,functions,pulsar}

site2/website/pages/en/admin-rest-api.js

@@ -12,7 +12,7 @@ class AdminRestApi extends React.Component {
       <div className="pageContainer">
         <Container className="mainContainer documentContainer postContainer" >
           <redoc spec-url={`${swaggerUrl}`} lazy-rendering="true"></redoc>
-          <script src="//cdn.jsdelivr.net/npm/redoc/bundles/redoc.standalone.js"/>
+          <script src="https://rebilly.github.io/ReDoc/releases/latest/redoc.min.js"></script>
         </Container>
       </div>
     );

site2/website/pages/en/functions-rest-api.js

@@ -0,0 +1,22 @@
+const React = require('react');
+const CompLibrary = require('../../core/CompLibrary.js');
+
+const Container = CompLibrary.Container;
+const siteConfig = require(`${process.cwd()}/siteConfig.js`);
+
+class FunctionsRestApi extends React.Component {
+  render() {
+    const swaggerUrl = `${siteConfig.baseUrl}swagger/swaggerfunctions.json`
+
+    return (
+      <div className="pageContainer">
+        <Container className="mainContainer documentContainer postContainer" >
+          <redoc spec-url={`${swaggerUrl}`} lazy-rendering="true"></redoc>
+          <script src="https://rebilly.github.io/ReDoc/releases/latest/redoc.min.js"></script>
+        </Container>
+      </div>
+    );
+  }
+}
+
+module.exports = FunctionsRestApi;

site2/website/pages/en/sink-rest-api.js

@@ -0,0 +1,22 @@
+const React = require('react');
+const CompLibrary = require('../../core/CompLibrary.js');
+
+const Container = CompLibrary.Container;
+const siteConfig = require(`${process.cwd()}/siteConfig.js`);
+
+class SinkRestApi extends React.Component {
+  render() {
+    const swaggerUrl = `${siteConfig.baseUrl}swagger/swaggersink.json`
+
+    return (
+      <div className="pageContainer">
+        <Container className="mainContainer documentContainer postContainer" >
+          <redoc spec-url={`${swaggerUrl}`} lazy-rendering="true"></redoc>
+          <script src="https://rebilly.github.io/ReDoc/releases/latest/redoc.min.js"></script>
+        </Container>
+      </div>
+    );
+  }
+}
+
+module.exports = SinkRestApi;

site2/website/pages/en/source-rest-api.js

@@ -0,0 +1,22 @@
+const React = require('react');
+const CompLibrary = require('../../core/CompLibrary.js');
+
+const Container = CompLibrary.Container;
+const siteConfig = require(`${process.cwd()}/siteConfig.js`);
+
+class SourceRestApi extends React.Component {
+  render() {
+    const swaggerUrl = `${siteConfig.baseUrl}swagger/swaggersource.json`
+
+    return (
+      <div className="pageContainer">
+        <Container className="mainContainer documentContainer postContainer" >
+          <redoc spec-url={`${swaggerUrl}`} lazy-rendering="true"></redoc>
+          <script src="https://rebilly.github.io/ReDoc/releases/latest/redoc.min.js"></script>
+        </Container>
+      </div>
+    );
+  }
+}
+
+module.exports = SourceRestApi;

site2/website/siteConfig.js

@@ -23,6 +23,12 @@ const createVariableInjectionPlugin = variables => {
       // rest api: rest:<name>:<path>
       } else if (keyparts[0] == 'rest') {
           return renderUrl(initializedPlugin, restApiUrl + "#", keyparts);
+      } else if (keyparts[0] == 'functions') {
+          return renderUrl(initializedPlugin, functionsApiUrl + "#", keyparts);
+      } else if (keyparts[0] == 'source') {
+          return renderUrl(initializedPlugin, sourceApiUrl + "#", keyparts);
+      } else if (keyparts[0] == 'sink') {
+          return renderUrl(initializedPlugin, sinkApiUrl + "#", keyparts);
       } else {
         keyparts = key.split("|");
         // endpoint api: endpoint|<op>
@@ -65,6 +71,9 @@ const renderEndpoint = (initializedPlugin, baseUrl, keyparts) => {
 const url = 'https://pulsar.incubator.apache.org';
 const javadocUrl = url + '/api';
 const restApiUrl = url + '/en' + "/admin-rest-api";
+const functionsApiUrl = url + '/en' + "/functions-rest-api";
+const sourceApiUrl = url + '/en' + "/source-rest-api";
+const sinkApiUrl = url + '/en' + "/sink-rest-api";
 const githubUrl = 'https://github.com/apache/incubator-pulsar';
 const baseUrl = '/';
 
@@ -94,7 +103,7 @@ const siteConfig = {
     {doc: 'standalone', label: 'Docs'},
     {page: 'download', label: 'Download'},
     {doc: 'client-libraries', label: 'Clients'},
-    {page: 'admin-rest-api', label: 'REST API'},
+    {href: '#restapis', label: 'REST APIs'},
     {blog: true, label: 'Blog'},
     {href: '#community', label: 'Community'},
     {href: '#apache', label: 'Apache'},

site2/website/static/css/custom.css

@@ -63,19 +63,22 @@
 /* for drop down menus */
 #community-dropdown.hide,
 #apache-dropdown.hide,
+#restapis-dropdown.hide,
 #languages-dropdown.hide {
   display: none;
 }
 
 
 #community-dropdown.visible,
 #apache-dropdown.visible,
+#restapis-dropdown.visible,
 #languages-dropdown.visible {
   display: flex;
 }
 
 #community-dropdown,
 #apache-dropdown,
+#restapis-dropdown,
 #languages-dropdown {
   pointer-events: none;
   position: absolute;
@@ -84,6 +87,7 @@
 
 #community-dropdown-items,
 #apache-dropdown-items,
+#restapis-dropdown-items,
 #languages-dropdown-items {
   background-color: white;
   display: flex;

site2/website/static/js/custom.js

@@ -64,6 +64,35 @@ window.addEventListener('load', function() {
     }
   });
 
+  // setup rest api menu items in nav bar
+  const restapis = document.querySelector("a[href='#restapis']").parentNode;
+  const restapisMenu =
+    '<li>' +
+    '<a id="restapis-menu" href="#">REST APIs <span style="font-size: 0.75em">&nbsp;▼</span></a>' +
+    '<div id="restapis-dropdown" class="hide">' +
+      '<ul id="restapis-dropdown-items">' +
+        '<li><a href="/admin-rest-api">Admin REST API </a></li>' +
+        '<li><a href="/functions-rest-api">Functions </a></li>' +
+        '<li><a href="/source-rest-api">Sources </a></li>' +
+        '<li><a href="/sink-rest-api">Sinks </a></li>' +
+      '</ul>' +
+    '</div>' +
+    '</li>';
+
+  restapis.innerHTML = restapisMenu;
+
+  const restapisMenuItem = document.getElementById("restapis-menu");
+  const restapisDropDown = document.getElementById("restapis-dropdown");
+  restapisMenuItem.addEventListener("click", function(event) {
+    event.preventDefault();
+
+    if (restapisDropDown.className == 'hide') {
+      restapisDropDown.className = 'visible';
+    } else {
+      restapisDropDown.className = 'hide';
+    }
+  });
+
 
   function button(label, ariaLabel, icon, className) {
     const btn = document.createElement('button');

src/gen-swagger.sh

@@ -26,6 +26,6 @@ echo "Generating swagger json file ..."
 mvn -am -pl pulsar-broker install -DskipTests -Pswagger
 echo "Swagger json file is generated."
 
-cp pulsar-broker/target/docs/swagger.json site2/website/static/swagger/
+cp pulsar-broker/target/docs/swagger*.json site2/website/static/swagger/
 echo "Copied swagger json file."